From 14e32af136ce5220b5483e06a6acfeaf3a393f20 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 17:06:42 -0700 Subject: [PATCH 01/32] feat(network): support trusted private endpoints --- docs/inference/custom-endpoint-security.mdx | 6 +- .../set-up-openai-compatible-endpoint.mdx | 7 +- docs/manage-sandboxes/add-mcp-server.mdx | 86 +++- docs/manage-sandboxes/manage-mcp-servers.mdx | 37 +- .../create-custom-policy-presets.mdx | 37 ++ docs/reference/commands.mdx | 42 +- docs/reference/troubleshoot-mcp-servers.mdx | 53 ++- src/commands/sandbox/policy/add.ts | 8 +- src/commands/sandbox/policy/mutate.test.ts | 26 ++ .../actions/sandbox/mcp-bridge-add-restart.ts | 101 ++++- .../actions/sandbox/mcp-bridge-contracts.ts | 8 + .../sandbox/mcp-bridge-destroy-preflight.ts | 9 +- .../sandbox/mcp-bridge-input-targets.test.ts | 87 ++++ .../mcp-bridge-input-validation.test.ts | 60 ++- src/lib/actions/sandbox/mcp-bridge-policy.ts | 56 ++- .../mcp-bridge-private-lifecycle.test.ts | 139 +++++++ .../sandbox/mcp-bridge-provider-inspection.ts | 48 ++- .../mcp-bridge-rebuild-exec-unavailable.ts | 20 +- src/lib/actions/sandbox/mcp-bridge-rebuild.ts | 2 +- src/lib/actions/sandbox/mcp-bridge-render.ts | 4 + src/lib/actions/sandbox/mcp-bridge-restart.ts | 17 +- src/lib/actions/sandbox/mcp-bridge-status.ts | 46 ++- .../sandbox/mcp-bridge-url-validation.ts | 167 +++++++- .../actions/sandbox/mcp-bridge-validation.ts | 68 +++- src/lib/actions/sandbox/mcp-bridge.ts | 6 +- src/lib/actions/sandbox/policy-channel.ts | 88 ++++- src/lib/actions/sandbox/rebuild-pipeline.ts | 20 +- .../sandbox/rebuild-restore-phase.test.ts | 19 +- .../actions/sandbox/rebuild-restore-phase.ts | 9 +- ...store-observability-reconciliation.test.ts | 38 ++ src/lib/actions/sandbox/snapshot.ts | 20 +- src/lib/cli/public-display-defaults.ts | 3 +- src/lib/domain/policy-channel.test.ts | 38 ++ src/lib/domain/policy-channel.ts | 22 +- .../compatible-endpoint-context.test.ts | 2 +- .../inference/compatible-endpoint-context.ts | 6 +- .../inference/endpoint-ssrf-preflight.test.ts | 10 + src/lib/inference/endpoint-ssrf-preflight.ts | 286 +------------- .../inference-selection-validation.test.ts | 2 +- .../onboard/inference-selection-validation.ts | 7 +- .../setup-inference-route-containment.test.ts | 2 +- src/lib/onboard/setup-inference.ts | 6 +- src/lib/policy/index.ts | 31 +- src/lib/policy/preset-allowed-ips.test.ts | 28 ++ src/lib/policy/preset-scope-render.test.ts | 15 + src/lib/policy/preset-scope-render.ts | 10 +- .../policy/trusted-private-endpoints.test.ts | 212 ++++++++++ src/lib/policy/trusted-private-endpoints.ts | 351 +++++++++++++++++ .../security/trusted-private-endpoint.test.ts | 159 ++++++++ src/lib/security/trusted-private-endpoint.ts | 370 ++++++++++++++++++ src/lib/state/registry-mcp.ts | 62 ++- src/lib/state/registry-normalization.test.ts | 56 +++ src/lib/state/registry-normalization.ts | 54 ++- src/lib/state/registry.ts | 3 +- src/lib/state/registry/persistence.ts | 7 + src/lib/state/registry/types.ts | 3 + src/lib/state/sandbox.ts | 19 +- test/registry.test.ts | 30 ++ 58 files changed, 2701 insertions(+), 427 deletions(-) create mode 100644 src/lib/actions/sandbox/mcp-bridge-private-lifecycle.test.ts create mode 100644 src/lib/policy/trusted-private-endpoints.test.ts create mode 100644 src/lib/policy/trusted-private-endpoints.ts create mode 100644 src/lib/security/trusted-private-endpoint.test.ts create mode 100644 src/lib/security/trusted-private-endpoint.ts diff --git a/docs/inference/custom-endpoint-security.mdx b/docs/inference/custom-endpoint-security.mdx index 8fdc60e373..2646fa49ee 100644 --- a/docs/inference/custom-endpoint-security.mdx +++ b/docs/inference/custom-endpoint-security.mdx @@ -33,10 +33,14 @@ It also rejects public hostnames that resolve to a private address. Managed provider defaults that do not provide an explicit custom endpoint through these paths are unaffected. Custom endpoint onboarding has one narrower operator-controlled exception for corporate inference gateways. -Set `NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` to a comma-separated list of exact hostnames or IP literals to admit an endpoint on RFC1918, CGNAT, or IPv6 ULA space. +Set `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` to a comma-separated list of exact hostnames or IP literals to admit an endpoint on RFC1918, carrier-grade network address translation (CGNAT), or IPv6 unique local address space. NemoClaw still resolves DNS, pins the validation connection, and rejects wildcard or suffix matches, link-local metadata, reserved destinations, and resolver failures. This allowlist does not relax direct blueprint, `config set`, or unrelated persisted-URL validation. +`NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` remains an inference-only compatibility alias. +Inference onboarding combines exact entries from the generic variable and the compatibility alias. +New configurations should use `NEMOCLAW_TRUSTED_PRIVATE_HOSTS`. + After onboarding records an admitted custom endpoint, `inference set` accepts that same canonical URL for a model change without resolving it again. The registry must record onboarding as the endpoint source, and the supplied URL must match exactly after normalization. Legacy entries without a source, endpoints recorded by `inference set`, and different URLs still pass through the full server-side request forgery validation path. diff --git a/docs/inference/set-up-openai-compatible-endpoint.mdx b/docs/inference/set-up-openai-compatible-endpoint.mdx index da600f30e4..6663f2370b 100644 --- a/docs/inference/set-up-openai-compatible-endpoint.mdx +++ b/docs/inference/set-up-openai-compatible-endpoint.mdx @@ -123,7 +123,8 @@ NEMOCLAW_PROVIDER=custom \ | `NEMOCLAW_MODEL` | Model ID reported by the server. | | `NEMOCLAW_COMPATIBLE_AUTH_MODE` | Set to `none` to explicitly select no authentication for an HTTP endpoint using `localhost`, `127.0.0.1`, or `[::1]` and port `8000`, `11434`, or `11435`. | | `NEMOCLAW_REASONING` | Enables reasoning-only validation with the case-insensitive true values `true`, `1`, `yes`, and `y`. | -| `NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` | Optional comma-separated exact hostnames or IP literals for operator-owned private inference endpoints. Wildcards are not supported. | +| `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` | Optional comma-separated exact hostnames or IP literals for operator-owned private endpoints. Wildcards are not supported. | +| `NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` | Inference-only compatibility alias for `NEMOCLAW_TRUSTED_PRIVATE_HOSTS`. Inference onboarding combines entries from both variables. | | `COMPATIBLE_API_KEY` | Endpoint API key. Required unless loopback no-auth mode is selected. | @@ -141,7 +142,7 @@ Private and reserved addresses are blocked by default. To use an inference gateway on a trusted corporate network, list only its exact host and keep the endpoint URL on that host: ```bash -NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS=llm.corp.example \ +NEMOCLAW_TRUSTED_PRIVATE_HOSTS=llm.corp.example \ NEMOCLAW_PROVIDER=custom \ NEMOCLAW_ENDPOINT_URL=https://llm.corp.example/v1 \ NEMOCLAW_MODEL=your-model \ @@ -150,7 +151,7 @@ NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS=llm.corp.example \ ``` NemoClaw still resolves the host before probing and pins the probe to the resolved address. -Only RFC1918, CGNAT, and IPv6 ULA destinations can be admitted; link-local metadata and other reserved ranges remain blocked. +Only RFC1918, carrier-grade network address translation (CGNAT), and IPv6 unique local address (ULA) destinations can be admitted; link-local metadata and other reserved ranges remain blocked. An unlisted private host, a hostname suffix match, or a DNS failure also remains blocked. ## Related Topics diff --git a/docs/manage-sandboxes/add-mcp-server.mdx b/docs/manage-sandboxes/add-mcp-server.mdx index 017dd1e90b..785f10f11c 100644 --- a/docs/manage-sandboxes/add-mcp-server.mdx +++ b/docs/manage-sandboxes/add-mcp-server.mdx @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 title: "Add an MCP Server" sidebar-title: "Add an MCP Server" -description: "Register an authenticated Streamable HTTP MCP server with a NemoClaw sandbox." -description-agent: "Explains mcp add inputs, credential-name restrictions, URL and DNS validation, policy pinning, and agent capability checks. Use when adding a managed MCP server." -keywords: ["nemoclaw mcp add", "streamable http mcp", "mcp bearer credential"] +description: "Register an authenticated public or trusted private Streamable HTTP MCP server with a NemoClaw sandbox." +description-agent: "Explains mcp add inputs, trusted private endpoints, credential-name restrictions, URL and DNS validation, policy pinning, and agent capability checks. Use when adding a managed MCP server." +keywords: ["nemoclaw mcp add", "streamable http mcp", "mcp bearer credential", "trusted private mcp"] content: type: "how_to" skill: @@ -48,6 +48,74 @@ NemoClaw persists only the variable name, writes `openshell:resolve:env:KEY` int After `mcp add` commits, NemoClaw performs a fresh status inspection and runs the gated credential-resolution probe once unless you pass `--no-probe`. If readiness is inconclusive, the command reports a probe skip without failing the committed add. +## Add a Trusted Private Server + +Use an exact trusted private host when the MCP endpoint must remain on an operator-controlled private network. +This flow is the same for OpenClaw, Hermes, and Deep Agents Code. + +Before registration, configure an HTTPS endpoint that meets these requirements: + +- The endpoint hostname resolves to stable RFC1918, carrier-grade network address translation (CGNAT), or IPv6 unique local addresses, or the URL uses one exact private IPv4 literal. +- The OpenShell gateway can route to every resolved address. +- The Transport Layer Security (TLS) certificate matches the endpoint hostname and chains to a trust root available to the managed runtime. +- A host firewall limits the listener to the OpenShell gateway or the required deployment subnet. +- The endpoint exposes only the required MCP path and methods. + +For a host-local MCP process, keep the process bound to loopback when practical. +Place an operator-managed HTTPS reverse proxy on one stable, routed private address. +Bind the proxy to the exact private interface instead of every host interface. +NemoClaw preserves the URL hostname for TLS Server Name Indication and certificate validation. +A direct private IPv4 URL requires a certificate with the matching IP subject alternative name. +Use a DNS hostname for an IPv6 unique local address. +OpenShell `v0.0.85` cannot safely represent a direct IPv6 literal as a proxy target, so NemoClaw rejects that URL form. + +Direct `127.0.0.1`, `::1`, and hostnames that resolve to loopback remain rejected. +Sandbox loopback is not the host service, and trusted-private admission does not create a route to it. + + +The `--trusted-private-host` option and `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` admit private network access to the matching exact host for this registration. +Confirm that you operate the endpoint and its network before you run the command. + + +Export one dedicated bearer credential, then register the endpoint with its exact URL hostname: + +```bash +export LOCAL_MCP_TOKEN='replace-with-secret-manager-value' +$$nemoclaw my-sandbox mcp add local-tools \ + --url https://mcp-host.corp.example/mcp \ + --env LOCAL_MCP_TOKEN \ + --trusted-private-host mcp-host.corp.example +unset LOCAL_MCP_TOKEN +``` + +The declaration must equal the normalized host from `--url`. +NemoClaw rejects unused, unrelated, wildcard, suffix, CIDR, URL-shaped, duplicate, or malformed `--trusted-private-host` declarations before mutation. +It also rejects a hostname when any answer is outside the admitted private ranges or otherwise disallowed. + +As an alternative, set `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` to a comma-separated list of exact hosts for the current command. +NemoClaw combines the environment list with any `--trusted-private-host` options. +It normalizes and deduplicates environment entries, ignores entries unrelated to this URL, and persists only the matching private host. +The command records the resulting exact trust intent, so later lifecycle commands do not depend on the ambient environment. + +NemoClaw records the normalized trust intent and every validated address as exact policy pins. +The raw bearer value passes transiently to the OpenShell provider and remains absent from NemoClaw state, sandbox configuration, command arguments, and logs. +The sandbox configuration contains only the `openshell:resolve:env:LOCAL_MCP_TOKEN` credential placeholder. +You can unset the host variable after `mcp add` returns because OpenShell retains the credential. +An exported replacement updates it during restart, and `mcp remove` or sandbox destroy deletes the registry-owned provider. + +Inspect the registration after the add returns: + +```bash +$$nemoclaw my-sandbox mcp status local-tools --json +``` + +The JSON field `trustedPrivateTarget.state` must report `match`. +The ordinary provider, policy, and adapter checks must also report readiness. + + +NemoClaw does not start, configure, monitor, or retain the reverse proxy, MCP process, certificate, DNS record, or host firewall state. + + ## Choose a Dedicated Credential Name Do not reuse OpenShell Google Cloud compatibility names such as `GCP_PROJECT_ID`, `GOOGLE_CLOUD_PROJECT`, `CLOUD_ML_REGION`, `GCP_LOCATION`, `GCP_SERVICE_ACCOUNT_EMAIL`, `GOOSE_PROVIDER`, `ANTHROPIC_VERTEX_PROJECT_ID`, or `VERTEX_LOCATION`. @@ -84,14 +152,22 @@ Deep Agents Code supports at most 64 managed MCP servers in one sandbox. Endpoint paths cannot contain percent escapes, backslashes, semicolons, or OpenShell glob metacharacters. Endpoint URLs cannot use port `0`. -NemoClaw resolves public hostnames before registration, rejects private, local, and special-use targets, and pins the resolved addresses in the generated policy. +NemoClaw resolves hostnames before registration and pins the validated addresses in the generated policy. +Public endpoints need no trusted-private declaration. +Private endpoints require an exact trust declaration and can use only RFC1918, CGNAT, or IPv6 unique local addresses. +Supply the declaration with `--trusted-private-host` or the generic `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` environment variable. +NemoClaw continues to reject loopback, link-local, metadata, unspecified, multicast, documentation, translation, benchmarking, and other reserved ranges. OpenShell re-resolves the hostname for each new connection, requires every current answer to match the pinned `allowed_ips`, and connects to the validated socket addresses. A DNS change to a private, special-use, or otherwise unpinned address fails closed instead of widening the route. +Cloudflare Quick Tunnels and other endpoints with rotating address sets are not durable managed MCP deployments. +A named tunnel can stabilize the hostname without stabilizing its exact address set. +Use stable private DNS and a routed private HTTPS endpoint when you operate the MCP server locally. + Authenticated MCP rejects `host.openshell.internal`, `host.docker.internal`, and `host.containers.internal` on stable OpenShell `v0.0.85`. That release has a trusted-gateway branch for one narrow link-local topology, but it does not expose an attested driver gateway address that NemoClaw can pin. -Use a normal HTTPS DNS endpoint with public address records until OpenShell exposes attested gateway state for exact policy pinning. +Use a routed HTTPS endpoint on a stable private address instead. ## Understand the Generated Method Profile diff --git a/docs/manage-sandboxes/manage-mcp-servers.mdx b/docs/manage-sandboxes/manage-mcp-servers.mdx index a51646f954..b5c6abb0e8 100644 --- a/docs/manage-sandboxes/manage-mcp-servers.mdx +++ b/docs/manage-sandboxes/manage-mcp-servers.mdx @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 title: "Manage MCP Servers" sidebar-title: "Manage MCP Servers" -description: "Inspect advertised tools, probe, rotate, restart, remove, rebuild, and destroy NemoClaw-managed MCP servers." -description-agent: "Explains managed MCP status, advertised tool discovery, credential-resolution probes, credential rotation, restart, removal, rebuild restoration, destroy recovery, and lifecycle locking. Use after an MCP server is registered." -keywords: ["nemoclaw mcp status", "mcp tool discovery", "nemoclaw mcp restart", "nemoclaw mcp remove", "mcp credential rotation"] +description: "Inspect advertised tools and DNS pins, probe, rotate, restart, remove, rebuild, and destroy NemoClaw-managed MCP servers." +description-agent: "Explains managed MCP status, DNS pin drift, advertised tool discovery, credential-resolution probes, credential rotation, restart, removal, rebuild restoration, destroy recovery, and lifecycle locking. Use after an MCP server is registered." +keywords: ["nemoclaw mcp status", "mcp dns pin drift", "mcp tool discovery", "nemoclaw mcp restart", "nemoclaw mcp remove", "mcp credential rotation"] content: type: "how_to" skill: @@ -29,6 +29,11 @@ The `env.missing` field lists recorded host variable names that are currently un An existing valid provider can remain ready when a host variable is unset because OpenShell retains the credential. The JSON value `support.mode: "bridge"` identifies the agent's config-adapter capability, not a host-side traffic bridge. +For a trusted private server, status resolves the endpoint without changing managed state. +Text output reports `private address pins: match`, `drift`, or `unresolved`. +JSON output reports the same value in `trustedPrivateTarget.state` and includes the recorded pins. +Status never adds a new address to the policy. + ## Discover Advertised Tools Pass `--tools` with one server name to request the server's advertised tool names: @@ -119,7 +124,9 @@ unset GITHUB_MCP_TOKEN `restart` requires a successful provider update, waits until the sandbox receives a new opaque provider revision, reapplies policy, and refreshes the adapter. An ambiguous or failed update is not treated as successful merely because another writer advanced the provider revision. -Restart resolves the endpoint hostname again before updating its pinned network policy. +For a trusted-private entry, restart replays the exact address pins recorded by `mcp add`. +It does not resolve that endpoint again or widen its policy from ambient DNS. +For a public entry, restart resolves the hostname again and refreshes the generated policy with the current validated public addresses. The raw value passes only through the OpenShell provider command's process environment and is not added to argv, NemoClaw state, or sandbox config. Revoke the old credential upstream after restart succeeds. @@ -132,6 +139,26 @@ If the provider is missing, export the recorded variable before retrying. Running restart without a server name refreshes every managed server. Export only the variables whose credentials you intend to replace. +## Change Endpoint Pins + +For a trusted-private server, review every destination change before NemoClaw records new pins. +Neither status, restart, rebuild, nor restore changes the recorded address set. + +If the endpoint moves to another address, remove and re-add the server: + +```bash +export LOCAL_MCP_TOKEN='replace-with-secret-manager-value' +$$nemoclaw my-sandbox mcp remove local-tools +$$nemoclaw my-sandbox mcp add local-tools \ + --url https://mcp-host.corp.example/mcp \ + --env LOCAL_MCP_TOKEN \ + --trusted-private-host mcp-host.corp.example +unset LOCAL_MCP_TOKEN +``` + +Removing the server deletes the registry-owned provider, policy, and adapter state after the existing ownership checks pass. +The re-add performs a new DNS preflight and records the reviewed exact address set. + ## Remove a Server ```bash @@ -154,6 +181,8 @@ It never detaches the provider from other sandboxes. `rebuild` preserves providers that match the recorded ID, type, and credential-key metadata. It removes adapter entries and detaches providers before replacing the sandbox, then reattaches providers, waits for credential readiness, reapplies policy, and restores adapters. +For a trusted-private entry, the restored policy uses the recorded exact address pins and does not widen them from current DNS answers. +Public entries continue to resolve and validate their endpoint addresses during restoration. NemoClaw revalidates the prepared Deep Agents replacement after MCP preparation and before stopping inference or deleting the old sandbox. diff --git a/docs/network-policy/create-custom-policy-presets.mdx b/docs/network-policy/create-custom-policy-presets.mdx index 930871e302..034d28cb58 100644 --- a/docs/network-policy/create-custom-policy-presets.mdx +++ b/docs/network-policy/create-custom-policy-presets.mdx @@ -69,6 +69,43 @@ NemoClaw rejects that field in files passed through `--from-file` or `--from-dir Use hostnames, ports, protocols, methods, paths, and binary restrictions instead. The only exception is the `host.openshell.internal` bridge endpoint for explicit sandbox-to-host service access. +## Admit an Exact Private Host + +Use explicit private-host trust when a custom preset targets an operator-controlled endpoint on RFC1918, carrier-grade network address translation (CGNAT), or IPv6 unique local address space. +This flow applies to REST, WebSocket, JSON-RPC, and MCP endpoint protocols. + + +The `--trusted-private-host` option and `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` grant the custom preset access to each matching exact private host. +Review the preset, resolved addresses, requesting binaries, methods, and paths before you apply it. + + +Pass the exact endpoint host with `--from-file` or `--from-dir`: + +```bash +$$nemoclaw my-assistant policy add \ + --from-file ./presets/my-internal-api.yaml \ + --trusted-private-host api.corp.example \ + --dry-run +``` + +The option is invalid for a built-in preset because maintained presets own their reviewed destinations. +NemoClaw rejects unused, unrelated, wildcard, suffix, CIDR, URL-shaped, duplicate, or malformed `--trusted-private-host` declarations. +It also rejects loopback, link-local, metadata, unspecified, multicast, documentation, translation, benchmarking, and other reserved ranges. + +As an alternative, set `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` to a comma-separated list of exact hosts for the current command. +NemoClaw combines the environment list with any `--trusted-private-host` options. +It normalizes and deduplicates environment entries and ignores entries unrelated to the custom preset batch. + +After schema validation, NemoClaw resolves each declared endpoint and inserts every validated address as an exact `allowed_ips` value in memory. +The dry-run output shows the generated pins for review. +NemoClaw applies and records the transformed preset instead of the unpinned source file. +Rebuild replays recorded pins from the sandbox registry without depending on the ambient environment. +A snapshot does not grant private-host authority to a clean target by itself; reapply the source preset with explicit trust after a cross-sandbox restore. + +To change a recorded address set, apply the source preset again with explicit trust. +NemoClaw performs a new preflight and shows the changed pins before it applies them. +Do not add `allowed_ips` to the source YAML. + ## Apply a Single File Preview the file before you apply it: diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2c527bca5d..3cf27db232 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1894,10 +1894,19 @@ Fix the gateway or policy read problem, then rerun the command. For custom presets, the command also reports when the preset reached the gateway but NemoClaw could not record it in the local sandbox registry, because unrecorded custom presets will not appear in `policy list` or `status`. Recover or re-onboard the sandbox, then re-apply the custom preset. +With `--from-file` or `--from-dir`, pass a repeatable `--trusted-private-host ` option to admit matching RFC1918, carrier-grade network address translation (CGNAT), or IPv6 unique local endpoints. +The option is invalid for built-in presets. +You can supply exact hosts through `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` instead, and NemoClaw combines the variable with command options. +NemoClaw resolves each matching exact host and adds generated `allowed_ips` pins to an in-memory copy of the preset. +User-authored `allowed_ips` remains rejected. +Dry-run output shows the generated pins, and rebuild replays the transformed preset recorded in the sandbox registry without widening it from ambient DNS. +A snapshot alone does not grant private-host authority to a clean target; after a cross-sandbox restore, reapply the source preset with explicit trust. + | Flag | Description | |------|-------------| | `--from-file ` | Apply a custom preset YAML file instead of a built-in preset | | `--from-dir ` | Apply every custom preset YAML file in a directory in lexicographic order | +| `--trusted-private-host ` | Admit one exact private endpoint host from a custom preset and generate exact address pins; repeat for additional hosts | | `--yes`, `--force` | Skip the confirmation prompt (requires a preset name, `--from-file`, or `--from-dir`) | | `--dry-run` | Preview the endpoints a preset would open without applying changes | @@ -1913,6 +1922,15 @@ Apply a custom preset file when you need to grant access to an endpoint that is $$nemoclaw my-assistant policy add --from-file ./presets/my-internal-api.yaml ``` +For a trusted private endpoint, preview the generated pins before applying them: + +```bash +$$nemoclaw my-assistant policy add \ + --from-file ./presets/my-internal-api.yaml \ + --trusted-private-host api.corp.example \ + --dry-run +``` + For batch workflows, apply all preset files from a directory: ```bash @@ -2274,6 +2292,11 @@ $$nemoclaw my-assistant mcp list [--json] Add an MCP Streamable HTTP server to a sandbox. Pass `--url` for the MCP endpoint and the required single `--env KEY` bearer credential for the sandbox-side MCP client. +Pass a repeatable `--trusted-private-host ` option to admit an exact RFC1918, CGNAT, or IPv6 unique local destination for the current command. +The declaration must equal the normalized host from `--url`. +For managed MCP, use a DNS hostname for an IPv6 unique local address because OpenShell `v0.0.85` cannot represent a direct IPv6 literal proxy target. +You can supply exact hosts through `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` instead, and NemoClaw combines the variable with command options. +NemoClaw records the resulting exact trust intent and address pins, so restart, rebuild, and restore do not depend on the ambient environment. NemoClaw registers that credential in an OpenShell provider, installs a generated OpenShell `protocol: mcp` policy for the target endpoint, attaches the provider to the running sandbox, and writes only an `openshell:resolve:env:KEY` placeholder into the agent config. Inline `--env KEY=VALUE` is rejected because it would expose the value in NemoClaw process arguments. Load the variable from a secret manager or masked prompt, export it without recording the value in shell history, and pass only `--env KEY`. @@ -2312,10 +2335,24 @@ $$nemoclaw my-assistant mcp add github --url https://api.githubcopilot.com/mcp/ unset GITHUB_MCP_TOKEN ``` +For a private endpoint, use its exact URL host: + +```bash +export LOCAL_MCP_TOKEN='replace-with-secret-manager-value' +$$nemoclaw my-assistant mcp add local-tools \ + --url https://mcp-host.corp.example/mcp \ + --env LOCAL_MCP_TOKEN \ + --trusted-private-host mcp-host.corp.example +unset LOCAL_MCP_TOKEN +``` + ### `$$nemoclaw mcp status` Inspect MCP server state for one server or for all configured servers. Status includes OpenShell provider presence and credential-key shape, provider attachment, generated policy content match, adapter registration, current host-variable availability, and the selected agent's MCP support mode. +For a trusted private endpoint, status also compares current DNS answers with recorded pins without changing the policy. +Text output reports `private address pins: match`, `drift`, or `unresolved`. +JSON output reports the same value in `trustedPrivateTarget.state` and includes the recorded pins. While a managed provider is attached, text and JSON status warn that its credential is sandbox-scoped until OpenShell supports endpoint-exclusive binding plus Host, scheme, and query enforcement. When a single server is named, status requests a differential wire-level credential-resolution probe. It sends no probe traffic unless the exact generated policy matches the effective gateway policy, the expected provider attachment is confirmed, and the live provider has the recorded ID, generic type, a valid resource version, and exactly one credential key matching the recorded key; a readiness failure reports `unknown` with a `probe skipped` detail. @@ -2360,6 +2397,8 @@ $$nemoclaw my-assistant mcp status [server] [--json] [--probe|--no-probe] [--too Refresh one MCP server registration, or every server on the sandbox when no server is supplied. Restart reapplies the generated policy, reattaches the OpenShell provider when needed, and refreshes the sandbox agent adapter registration. +For a trusted-private entry, restart replays recorded address pins without resolving the endpoint again or widening the policy. +For a public entry, restart resolves the hostname again and refreshes the policy with the current validated public addresses. If the recorded host variable is exported, restart replaces the provider credential and waits for its new opaque revision. Otherwise, restart reuses an existing provider whose current metadata match the registry. A missing provider requires the variable to be exported before retrying. @@ -3777,7 +3816,8 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_TOOL_DISCLOSURE` | `progressive` or `direct` | Selects progressive tool discovery or the prior direct-exposure behavior. Defaults to `progressive`; `--tool-disclosure` takes precedence when both are set. | | `NEMOCLAW_ENDPOINT_URL` | URL | Custom endpoint URL. Used together with `NEMOCLAW_PROVIDER=custom` for OpenAI-compatible endpoints or `NEMOCLAW_PROVIDER=anthropicCompatible` for Anthropic-compatible endpoints. | | `NEMOCLAW_COMPATIBLE_AUTH_MODE` | `none` or unset | Explicitly selects no authentication for an HTTP OpenAI-compatible endpoint using `localhost`, `127.0.0.1`, or `[::1]` and port `8000`, `11434`, or `11435` during non-interactive onboarding. | -| `NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` | comma-separated exact hostnames or IP literals | Allows operator-owned RFC1918, CGNAT, or IPv6 ULA inference endpoints during custom endpoint onboarding. Link-local metadata and other reserved ranges remain blocked; DNS resolution and connection pinning remain active; wildcards are not supported. | +| `NEMOCLAW_TRUSTED_PRIVATE_HOSTS` | comma-separated exact hostnames or IP literals | Allows operator-owned RFC1918, CGNAT, or IPv6 unique local destinations through supported inference, managed MCP, and custom-policy registration paths. Link-local metadata and other reserved ranges remain blocked; DNS resolution and exact address pinning remain active; wildcards are not supported. | +| `NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS` | comma-separated exact hostnames or IP literals | Inference-only compatibility alias. Inference onboarding combines entries from this variable and `NEMOCLAW_TRUSTED_PRIVATE_HOSTS`. | | `NEMOCLAW_PREFERRED_API` | `completions` (currently the only honored value) | Forces the validation probe to use the `/v1/chat/completions` API path instead of the newer `/v1/responses` API. | | `NEMOCLAW_INFERENCE_INPUTS` | comma-separated list of `text` and/or `image` | Declares model input modalities for vision-capable models. Validated strictly; unknown tokens are ignored. | | `NEMOCLAW_OLLAMA_REQUIRE_TOOLS` | `0` to disable, anything else to keep the default | When set to `0`, skips the Ollama tool-calling capability check during local-inference onboarding. | diff --git a/docs/reference/troubleshoot-mcp-servers.mdx b/docs/reference/troubleshoot-mcp-servers.mdx index 05b838027b..3da7c9c124 100644 --- a/docs/reference/troubleshoot-mcp-servers.mdx +++ b/docs/reference/troubleshoot-mcp-servers.mdx @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 title: "Troubleshoot MCP Servers" sidebar-title: "Troubleshoot MCP Servers" -description: "Diagnose managed MCP credential resolution, incomplete transactions, capability gaps, policy drift, and lifecycle-lock failures." -description-agent: "Provides symptom-based remediation for NemoClaw-managed MCP servers. Use when mcp add, status, restart, remove, rebuild, or destroy does not converge." -keywords: ["troubleshoot nemoclaw mcp", "mcp credential resolution", "mcp policy drift", "mcp transaction"] +description: "Diagnose managed MCP credential resolution, DNS pin drift, incomplete transactions, capability gaps, policy drift, and lifecycle-lock failures." +description-agent: "Provides symptom-based remediation for NemoClaw-managed MCP servers, including trusted private endpoints. Use when mcp add, status, restart, remove, rebuild, or destroy does not converge." +keywords: ["troubleshoot nemoclaw mcp", "mcp credential resolution", "mcp dns pin drift", "mcp policy drift", "mcp transaction"] content: type: "troubleshooting" --- @@ -178,17 +178,56 @@ Before a credential or provider side effect, the MCP command loads the exact gen A runtime that rejects `protocol: mcp` therefore fails closed. -## Local or Plain-HTTP MCP URL Is Rejected +## Private or Plain-HTTP MCP URL Is Rejected -If `$$nemoclaw mcp add` rejects a `http://` or local URL and requires HTTPS, this is the managed registration boundary, not a Hermes-specific limitation. -Managed `mcp add` enforces HTTPS, a publicly reachable endpoint, and pinned egress policy the same way for OpenClaw, Hermes, and Deep Agents Code so OpenShell forwards the bearer credential over TLS. +If `$$nemoclaw mcp add` rejects an `http://` URL, use HTTPS before you retry. +This requirement applies to OpenClaw, Hermes, and Deep Agents Code so OpenShell forwards the bearer credential over TLS. + +If the endpoint resolves to RFC1918, carrier-grade network address translation (CGNAT), or IPv6 unique local addresses, add its exact host with `--trusted-private-host`. +The declared host must equal the normalized hostname from `--url`. +NemoClaw rejects a private endpoint without that explicit declaration before provider, policy, registry, or adapter mutation. An agent-native registration path, such as OpenClaw `mcporter` run inside the sandbox, may accept a plain-HTTP or local URL, but it bypasses NemoClaw credential replacement and generated egress policy. A URL that the agent-native path accepts is therefore not registrable through managed `mcp add`. -For local development, expose the MCP server over HTTPS on a publicly reachable endpoint — a public hostname or IP address with public address records — then register it with `$$nemoclaw mcp add`. +For a host-local server, keep the MCP process on loopback when practical. +Place an operator-managed HTTPS reverse proxy on a stable private address that the OpenShell gateway can route to. +The certificate must match the original endpoint hostname and chain to a trust root available to the managed runtime. +Restrict the proxy listener to the OpenShell gateway or required deployment subnet with host firewall rules. + +Direct `127.0.0.1`, `::1`, and hostnames that resolve to loopback remain rejected. +Sandbox loopback is not the host service, and `--trusted-private-host` does not create a route to it. +Use a DNS hostname for an IPv6 unique local address. +OpenShell `v0.0.85` cannot represent a direct IPv6 literal proxy target. See [Add an MCP Server](../manage-sandboxes/mcp-servers/add-an-mcp-server) for the full endpoint requirements. +## DNS Pins Drift + +If `mcp status ` reports `private address pins: drift`, review the endpoint address change before updating access. +Status, restart, rebuild, and restore do not add the new addresses. + +Remove and re-add the server with the same exact trusted host to perform a new preflight and record new pins: + +```bash +export LOCAL_MCP_TOKEN='replace-with-secret-manager-value' +$$nemoclaw mcp remove +$$nemoclaw mcp add \ + --url https://mcp-host.corp.example/mcp \ + --env LOCAL_MCP_TOKEN \ + --trusted-private-host mcp-host.corp.example +unset LOCAL_MCP_TOKEN +``` + +A `CONNECT 403` after an address change can mean that OpenShell rejected a current DNS answer outside `allowed_ips`. +Do not add a provider's complete address range or broaden the policy to bypass the denial. + +Cloudflare Quick Tunnels rotate public edge addresses and are not durable for exact address pinning. +A named tunnel can keep one hostname while its resolved addresses still change. +Use stable private DNS and a routed private HTTPS endpoint for a host-local MCP server. + +If status reports matching pins but the request still receives `CONNECT 403`, inspect the OpenShell policy and audit logs. +Do not treat matching DNS pins as evidence that the path, method, adapter identity, or provider attachment also matches. + ## Lifecycle Lock Times Out Confirm that no `mcp add`, `mcp restart`, `mcp remove`, `rebuild`, or `destroy` command for the sandbox is still running, then retry the original command. diff --git a/src/commands/sandbox/policy/add.ts b/src/commands/sandbox/policy/add.ts index 7b961dee75..02a12506ae 100644 --- a/src/commands/sandbox/policy/add.ts +++ b/src/commands/sandbox/policy/add.ts @@ -17,12 +17,13 @@ export default class PolicyAddCommand extends NemoClawCommand { static summary = "Add a network or filesystem policy preset"; static description = "Add a built-in or custom policy preset to a sandbox."; static usage = [ - " [preset] [--yes|-y] [--dry-run] [--from-file ] [--from-dir ]", + " [preset] [--yes|-y] [--dry-run] [--from-file ] [--from-dir ] [--trusted-private-host ]", ]; static examples = [ "<%= config.bin %> sandbox policy add alpha slack --yes", "<%= config.bin %> sandbox policy add alpha --from-file ./policy.yaml --dry-run", "<%= config.bin %> sandbox policy add alpha --from-dir ./policies --yes", + "<%= config.bin %> sandbox policy add alpha --from-file ./policy.yaml --trusted-private-host api.corp.example --dry-run", ]; static args = policyMutationArgs; static flags = { @@ -35,6 +36,10 @@ export default class PolicyAddCommand extends NemoClawCommand { description: "Load all custom preset YAML files in a directory", exclusive: ["from-file"], }), + "trusted-private-host": Flags.string({ + description: "Trust one exact private endpoint host in custom preset input. Repeatable.", + multiple: true, + }), }; public async run(): Promise { @@ -44,6 +49,7 @@ export default class PolicyAddCommand extends NemoClawCommand { ...commonPolicyOptions(flags), fromFile: flags["from-file"], fromDir: flags["from-dir"], + trustedPrivateHosts: flags["trusted-private-host"], }); } } diff --git a/src/commands/sandbox/policy/mutate.test.ts b/src/commands/sandbox/policy/mutate.test.ts index bf33da0856..add2a76426 100644 --- a/src/commands/sandbox/policy/mutate.test.ts +++ b/src/commands/sandbox/policy/mutate.test.ts @@ -37,6 +37,32 @@ describe("policy mutation oclif commands", () => { dryRun: true, fromFile: "/tmp/preset.yaml", fromDir: undefined, + trustedPrivateHosts: undefined, + }); + }); + + it("maps repeatable trusted private hosts for custom policy input (#8176)", async () => { + await PolicyAddCommand.run( + [ + "alpha", + "--from-file", + "/tmp/preset.yaml", + "--trusted-private-host", + "api.corp.example", + "--trusted-private-host", + "10.20.30.40", + ], + rootDir, + ); + + expect(mocks.addSandboxPolicy).toHaveBeenCalledWith("alpha", { + preset: undefined, + yes: false, + force: false, + dryRun: false, + fromFile: "/tmp/preset.yaml", + fromDir: undefined, + trustedPrivateHosts: ["api.corp.example", "10.20.30.40"], }); }); diff --git a/src/lib/actions/sandbox/mcp-bridge-add-restart.ts b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts index 6f86b04448..77b4a0e5cd 100644 --- a/src/lib/actions/sandbox/mcp-bridge-add-restart.ts +++ b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts @@ -5,6 +5,11 @@ import crypto from "node:crypto"; import type { AgentMcpAdapter } from "../../agent/defs"; import * as policies from "../../policy"; +import { + normalizeTrustedPrivateHost, + parseTrustedPrivateHosts, + replayTrustedPrivateEndpoint, +} from "../../security/trusted-private-endpoint"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; @@ -51,15 +56,16 @@ import { nowIso, writeBridgeEntry, } from "./mcp-bridge-state"; +import type { McpBridgeTargetValidation } from "./mcp-bridge-url-validation"; import { assertAuthenticatedCredentialReference, assertMcpCredentialBoundaryRuntimeVersion, buildMcpBridgeProviderName, normalizeMcpServerUrl, + preflightMcpServerUrlResolvedTarget, resolveCredentialEnv, uniqueEnvNames, validateMcpServerName, - validateMcpServerUrlResolvedTarget, validateSandboxName, } from "./mcp-bridge-validation"; @@ -71,6 +77,11 @@ function sameMcpAddIntent(existing: McpBridgeEntry, requested: McpBridgeEntry): existing.url === requested.url && existing.providerName === requested.providerName && existing.policyName === requested.policyName && + existing.trustedPrivateHost === requested.trustedPrivateHost && + (existing.allowedIps?.length ?? 0) === (requested.allowedIps?.length ?? 0) && + (existing.allowedIps ?? []).every( + (address, index) => address === requested.allowedIps?.[index], + ) && existing.env.length === requested.env.length && existing.env.every((name, index) => name === requested.env[index]) ); @@ -80,7 +91,7 @@ function assertPreparedMcpAddResourcesAbsent( sandboxName: string, adapter: AgentMcpAdapter, entry: McpBridgeEntry, - resolvedAddresses?: readonly string[], + target: McpBridgeTargetValidation, ): void { const adapterInspection = inspectAgentAdapterRegistration(sandboxName, adapter, entry); if (adapterInspection.state !== "absent") { @@ -116,7 +127,7 @@ function assertPreparedMcpAddResourcesAbsent( entry.server, entry.url, adapter, - resolvedAddresses, + target.addresses, ); const policyState = policies.getPresetContentGatewayState(sandboxName, policyContent); if (policyState !== "absent") { @@ -140,8 +151,39 @@ async function addMcpBridgeUnlocked( validateSandboxName(sandboxName); validateMcpServerName(options.server); assertAuthenticatedCredentialReference(options.env); - const normalizedUrl = normalizeMcpServerUrl(options.url); - const resolvedAddresses = await validateMcpServerUrlResolvedTarget(new URL(normalizedUrl)); + let explicitTrustedPrivateHosts: string[]; + let configuredTrustedPrivateHosts: string[]; + try { + explicitTrustedPrivateHosts = (options.trustedPrivateHosts ?? []).map((host) => + normalizeTrustedPrivateHost(host), + ); + configuredTrustedPrivateHosts = parseTrustedPrivateHosts( + process.env.NEMOCLAW_TRUSTED_PRIVATE_HOSTS, + ); + } catch (error) { + throw new McpBridgeError(error instanceof Error ? error.message : String(error), 2); + } + if (new Set(explicitTrustedPrivateHosts).size !== explicitTrustedPrivateHosts.length) { + throw new McpBridgeError( + "Duplicate --trusted-private-host declarations are not accepted after normalization.", + 2, + ); + } + const allTrustedPrivateHosts = [ + ...new Set([...explicitTrustedPrivateHosts, ...configuredTrustedPrivateHosts]), + ]; + const normalizedUrl = normalizeMcpServerUrl(options.url, { + trustedPrivateHosts: allTrustedPrivateHosts, + }); + const urlHost = new URL(normalizedUrl).hostname.toLowerCase(); + const unrelatedExplicitHost = explicitTrustedPrivateHosts.find((host) => host !== urlHost); + if (unrelatedExplicitHost) { + throw new McpBridgeError( + `--trusted-private-host ${unrelatedExplicitHost} does not match MCP server URL host '${urlHost}'.`, + 2, + ); + } + const matchingTrustedPrivateHosts = allTrustedPrivateHosts.filter((host) => host === urlHost); const sandbox = getSandboxOrThrow(sandboxName); assertMcpDestroyNotPending(sandbox); const agent = getSandboxAgent(sandbox); @@ -152,6 +194,39 @@ async function addMcpBridgeUnlocked( `MCP server '${options.server}' already exists on sandbox '${sandboxName}'.`, ); } + let target: McpBridgeTargetValidation; + if (existingEntry?.trustedPrivateHost) { + if ( + existingEntry.trustedPrivateHost !== urlHost || + !matchingTrustedPrivateHosts.includes(existingEntry.trustedPrivateHost) + ) { + throw new McpBridgeError( + `MCP server '${options.server}' has an incomplete add transaction with different trusted-private host intent. Re-run the original add command or remove it with --force before changing the definition.`, + 2, + ); + } + try { + const replay = replayTrustedPrivateEndpoint( + existingEntry.trustedPrivateHost, + existingEntry.allowedIps ?? [], + ); + target = { + addresses: [...replay.addresses], + trustedPrivateCapability: replay.trustedPrivateCapability, + trustedPrivateHost: replay.host, + }; + } catch (error) { + throw new McpBridgeError( + `MCP server '${options.server}' has invalid durable trusted-private intent: ${error instanceof Error ? error.message : String(error)}. Remove it with --force and add it again.`, + 2, + ); + } + } else { + target = await preflightMcpServerUrlResolvedTarget(new URL(normalizedUrl), { + trustedPrivateHosts: matchingTrustedPrivateHosts, + requireTrustedPrivateEndpoint: explicitTrustedPrivateHosts.length > 0, + }); + } const envNames = uniqueEnvNames(options.env); const envCollision = Object.values(bridgeState(sandbox)).find( @@ -189,6 +264,12 @@ async function addMcpBridgeUnlocked( adapter, url: normalizedUrl, env: envNames, + ...(target.trustedPrivateHost + ? { + trustedPrivateHost: target.trustedPrivateHost, + allowedIps: [...target.addresses], + } + : {}), ...(providerName ? { providerName } : {}), policyName, addedAt: existingEntry?.addedAt ?? nowIso(), @@ -203,7 +284,11 @@ async function addMcpBridgeUnlocked( } let entry: McpBridgeEntry = existingEntry - ? { ...existingEntry, env: [...existingEntry.env] } + ? { + ...existingEntry, + env: [...existingEntry.env], + ...(existingEntry.allowedIps ? { allowedIps: [...existingEntry.allowedIps] } : {}), + } : requestedEntry; const resumingPreflightedAdd = existingEntry?.addState === "preflighted"; if (existingEntry?.addState === "prepared" && !Object.hasOwn(adapterEnvValues, entry.env[0])) { @@ -268,7 +353,7 @@ async function addMcpBridgeUnlocked( } if (entry.addState === "prepared") { - assertPreparedMcpAddResourcesAbsent(sandboxName, adapter, entry, resolvedAddresses); + assertPreparedMcpAddResourcesAbsent(sandboxName, adapter, entry, target); entry = { ...entry, addState: "preflighted" }; // This second durable boundary proves the derived resource names and the // adapter slot were absent before any side effect. After a crash, retries @@ -295,7 +380,7 @@ async function addMcpBridgeUnlocked( // Loading the real protocol:mcp policy with --wait is the authoritative // running-supervisor capability check. Do it before any host credential is // created or updated so unsupported runtimes fail without that side effect. - applyGeneratedPolicy(sandboxName, entry, resolvedAddresses); + applyGeneratedPolicy(sandboxName, entry, target); policyApplied = true; const providerResult = upsertMcpProvider(providerName ?? "", options.env, { // A first mutation must still observe the absence proven above. Only a diff --git a/src/lib/actions/sandbox/mcp-bridge-contracts.ts b/src/lib/actions/sandbox/mcp-bridge-contracts.ts index a19e5480de..474f1834bb 100644 --- a/src/lib/actions/sandbox/mcp-bridge-contracts.ts +++ b/src/lib/actions/sandbox/mcp-bridge-contracts.ts @@ -23,6 +23,7 @@ export interface ParsedMcpAddArgs { server: string; url: string; env: ParsedEnvReference[]; + trustedPrivateHosts?: string[]; } export interface McpBridgeAddOptions extends ParsedMcpAddArgs {} @@ -38,6 +39,13 @@ export interface McpBridgeStatus { reason?: string; }; url?: string; + trustedPrivateTarget?: { + host: string; + recordedPins: string[]; + currentPins?: string[]; + state: "match" | "drift" | "unresolved"; + detail?: string; + }; env: { names: string[]; missing: string[]; diff --git a/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts b/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts index 7dde510f8c..b9aaa914a4 100644 --- a/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts +++ b/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts @@ -33,7 +33,11 @@ export interface McpDestroyPreparation { } export function cloneMcpBridgeEntry(entry: McpBridgeEntry): McpBridgeEntry { - return { ...entry, env: [...entry.env] }; + return { + ...entry, + env: [...entry.env], + ...(entry.allowedIps ? { allowedIps: [...entry.allowedIps] } : {}), + }; } function mcpBridgeEntriesEqual(left: McpBridgeEntry, right: McpBridgeEntry): boolean { @@ -42,6 +46,9 @@ function mcpBridgeEntriesEqual(left: McpBridgeEntry, right: McpBridgeEntry): boo left.agent === right.agent && left.adapter === right.adapter && left.url === right.url && + left.trustedPrivateHost === right.trustedPrivateHost && + (left.allowedIps?.length ?? 0) === (right.allowedIps?.length ?? 0) && + (left.allowedIps ?? []).every((address, index) => address === right.allowedIps?.[index]) && left.providerName === right.providerName && left.providerId === right.providerId && left.policyName === right.policyName && diff --git a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts index 3e0d6941fa..c2d36b5116 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts @@ -6,6 +6,10 @@ import dns from "node:dns/promises"; import { describe, expect, it, vi } from "vitest"; import { addMcpBridge, normalizeMcpServerUrl } from "./mcp-bridge"; +import { + inspectMcpRecordedTargetPins, + preflightMcpServerUrlResolvedTarget, +} from "./mcp-bridge-url-validation"; import { validateMcpServerUrlResolvedTarget } from "./mcp-bridge-validation"; describe("MCP URL target validation", () => { @@ -41,6 +45,89 @@ describe("MCP URL target validation", () => { } }); + it("issues exact private pins only for the matching operator trust (#8176)", async () => { + const lookup = vi.spyOn(dns, "lookup").mockResolvedValue([ + { address: "10.20.30.41", family: 4 }, + { address: "10.20.30.40", family: 4 }, + { address: "10.20.30.40", family: 4 }, + ] as never); + try { + await expect( + preflightMcpServerUrlResolvedTarget(new URL("https://mcp.corp.example/mcp"), { + trustedPrivateHosts: ["mcp.corp.example"], + requireTrustedPrivateEndpoint: true, + }), + ).resolves.toMatchObject({ + addresses: ["10.20.30.40", "10.20.30.41"], + trustedPrivateHost: "mcp.corp.example", + }); + } finally { + lookup.mockRestore(); + } + }); + + it("rejects mixed answers and an unused trusted-private option (#8267)", async () => { + const lookup = vi.spyOn(dns, "lookup"); + try { + lookup.mockResolvedValueOnce([ + { address: "10.20.30.40", family: 4 }, + { address: "8.8.8.8", family: 4 }, + ] as never); + await expect( + preflightMcpServerUrlResolvedTarget(new URL("https://mcp.corp.example/mcp"), { + trustedPrivateHosts: ["mcp.corp.example"], + requireTrustedPrivateEndpoint: true, + }), + ).rejects.toThrow(/mixed public and private addresses/); + + lookup.mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }] as never); + await expect( + preflightMcpServerUrlResolvedTarget(new URL("https://mcp.corp.example/mcp"), { + trustedPrivateHosts: ["mcp.corp.example"], + requireTrustedPrivateEndpoint: true, + }), + ).rejects.toThrow(/is unused/); + } finally { + lookup.mockRestore(); + } + }); + + it("reports recorded private pins as match, drift, or unresolved without mutation (#8267)", async () => { + const lookup = vi.spyOn(dns, "lookup"); + try { + lookup.mockResolvedValueOnce([{ address: "10.20.30.40", family: 4 }] as never); + await expect( + inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ + "10.20.30.40", + ]), + ).resolves.toMatchObject({ state: "match", currentAddresses: ["10.20.30.40"] }); + + lookup.mockResolvedValueOnce([{ address: "10.20.30.41", family: 4 }] as never); + await expect( + inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ + "10.20.30.40", + ]), + ).resolves.toMatchObject({ state: "drift", currentAddresses: ["10.20.30.41"] }); + + lookup.mockRejectedValueOnce(new Error("resolver unavailable")); + await expect( + inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ + "10.20.30.40", + ]), + ).resolves.toMatchObject({ state: "unresolved" }); + } finally { + lookup.mockRestore(); + } + }); + + it("requires a routed private endpoint for an explicitly trusted loopback URL (#8267)", () => { + expect(() => + normalizeMcpServerUrl("https://127.0.0.1/mcp", { + trustedPrivateHosts: ["127.0.0.1"], + }), + ).toThrow(/Sandbox loopback is not the host MCP service.*stable routed private address/); + }); + it("rejects hostile OpenShell alias registrations before sandbox or network side effects", async () => { const lookup = vi.spyOn(dns, "lookup"); try { diff --git a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts index 93b2e02bcf..b0c9b4f4d2 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { SUBPROCESS_ENV_ALLOWED_NAMES, @@ -33,6 +33,64 @@ describe("MCP CLI input validation", () => { }); }); + it("normalizes one exact trusted-private host from a repeated add option (#8267)", () => { + expect( + parseMcpAddArgs([ + "local", + "--url", + "https://10.20.30.40/mcp", + "--env", + "LOCAL_MCP_TOKEN", + "--trusted-private-host", + "10.20.30.40", + ]), + ).toEqual({ + server: "local", + url: "https://10.20.30.40/mcp", + env: [{ name: "LOCAL_MCP_TOKEN" }], + trustedPrivateHosts: ["10.20.30.40"], + }); + + expect(() => + parseMcpAddArgs([ + "local", + "--url", + "https://mcp.corp.example/mcp", + "--env", + "LOCAL_MCP_TOKEN", + "--trusted-private-host", + "MCP.CORP.EXAMPLE.", + "--trusted-private-host=mcp.corp.example", + ]), + ).toThrow(/Duplicate --trusted-private-host/); + }); + + it("uses generic trusted-private hosts without persisting unrelated entries (#8176)", () => { + vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_HOSTS", "unrelated.corp.example,10.20.30.40"); + + expect( + parseMcpAddArgs(["local", "--url", "https://10.20.30.40/mcp", "--env", "LOCAL_MCP_TOKEN"]), + ).toEqual({ + server: "local", + url: "https://10.20.30.40/mcp", + env: [{ name: "LOCAL_MCP_TOKEN" }], + }); + }); + + it("rejects a trusted-private add option for a different URL host (#8267)", () => { + expect(() => + parseMcpAddArgs([ + "local", + "--url", + "https://mcp.corp.example/mcp", + "--env", + "LOCAL_MCP_TOKEN", + "--trusted-private-host", + "other.corp.example", + ]), + ).toThrow(/does not match MCP server URL host/); + }); + it("rejects inline env values that would leak through process arguments", () => { expect(() => parseMcpAddArgs(["srv", "--url=https://mcp.example.test/rpc", "--env=TOKEN=a=b=c"]), diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 88a5e02465..807ab0d238 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -3,6 +3,7 @@ import type { AgentMcpAdapter } from "../../agent/defs"; import * as policies from "../../policy"; +import { isTrustedPrivateEndpointCapability } from "../../security/trusted-private-endpoint"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { @@ -15,6 +16,7 @@ import { buildMcpBridgePolicyName, buildMcpBridgePolicyYaml, } from "./mcp-bridge-policy-render"; +import type { McpBridgeTargetValidation } from "./mcp-bridge-url-validation"; export { buildMcpBridgePolicyKey, @@ -96,8 +98,9 @@ function reconcileGeneratedPolicyRegistration( export function applyGeneratedPolicy( sandboxName: string, entry: McpBridgeEntry, - resolvedAddresses: readonly string[], + target: McpBridgeTargetValidation | readonly string[], ): void { + const resolvedAddresses = assertMcpBridgePolicyTarget(entry, asMcpBridgePolicyTarget(target)); if (resolvedAddresses.length === 0) { throw new McpBridgeError( `Refusing to apply generated MCP policy '${entry.policyName}' without exact public address pins.`, @@ -199,6 +202,54 @@ export function applyGeneratedPolicy( ); } +function asMcpBridgePolicyTarget( + value: McpBridgeTargetValidation | readonly string[], +): McpBridgeTargetValidation { + return Array.isArray(value) ? { addresses: [...value] } : (value as McpBridgeTargetValidation); +} + +export function assertMcpBridgePolicyTarget( + entry: McpBridgeEntry, + target: McpBridgeTargetValidation, +): readonly string[] { + if (target.addresses.length === 0) { + throw new McpBridgeError( + `Refusing to apply generated MCP policy '${entry.policyName}' without exact ${entry.trustedPrivateHost ? "trusted-private" : "public"} address pins.`, + ); + } + if (!entry.trustedPrivateHost) { + if (target.trustedPrivateCapability || target.trustedPrivateHost) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no durable trusted-private intent. Refusing private policy mutation.`, + ); + } + return target.addresses; + } + if ( + target.trustedPrivateHost !== entry.trustedPrivateHost || + !isTrustedPrivateEndpointCapability(target.trustedPrivateCapability) + ) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no provenance-checked capability for trusted private host '${entry.trustedPrivateHost}'.`, + ); + } + const recordedPins = entry.allowedIps ?? []; + const capabilityPins = [...target.trustedPrivateCapability.addresses].sort(); + if ( + recordedPins.length === 0 || + target.addresses.length !== recordedPins.length || + target.addresses.some((address, index) => address !== recordedPins[index]) || + capabilityPins.length !== recordedPins.length || + capabilityPins.some((address, index) => address !== recordedPins[index]) + ) { + throw new McpBridgeError( + `MCP server '${entry.server}' no longer resolves to its recorded trusted-private address pins. Remove and re-add the server to approve changed pins.`, + 2, + ); + } + return recordedPins; +} + function generatedPolicyContent(entry: McpBridgeEntry): string { const adapter = isAgentMcpAdapter(entry.adapter) ? entry.adapter : "mcporter"; return buildMcpBridgePolicyYaml(entry.server, entry.url, adapter); @@ -252,8 +303,9 @@ export function assertGeneratedPolicyExactReadOnly( sandboxName: string, entry: McpBridgeEntry, adapter: AgentMcpAdapter, - resolvedAddresses: readonly string[], + target: McpBridgeTargetValidation | readonly string[], ): registry.CustomPolicyEntry { + const resolvedAddresses = assertMcpBridgePolicyTarget(entry, asMcpBridgePolicyTarget(target)); const canonicalOwnershipError = (): McpBridgeError => new McpBridgeError( "Generated MCP policy ownership is not canonical for its recorded bridge definition. Refusing host-side rebuild recovery.", diff --git a/src/lib/actions/sandbox/mcp-bridge-private-lifecycle.test.ts b/src/lib/actions/sandbox/mcp-bridge-private-lifecycle.test.ts new file mode 100644 index 0000000000..f977f206e1 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-private-lifecycle.test.ts @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import dns from "node:dns/promises"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import type { AgentMcpAdapter } from "../../agent/defs"; +import { isTrustedPrivateEndpointCapability } from "../../security/trusted-private-endpoint"; +import type { McpBridgeEntry } from "../../state/registry"; +import { assertMcpBridgePolicyTarget } from "./mcp-bridge-policy"; +import { preflightMcpEntryTargets } from "./mcp-bridge-provider"; + +const adapters: Array<{ adapter: AgentMcpAdapter; agent: string }> = [ + { adapter: "mcporter", agent: "openclaw" }, + { adapter: "hermes-config", agent: "hermes" }, + { adapter: "deepagents-config", agent: "deepagents" }, +]; + +function privateEntry(adapter: AgentMcpAdapter, agent: string): McpBridgeEntry { + return { + server: "local", + agent, + adapter, + url: "https://mcp.corp.example/mcp", + env: ["LOCAL_MCP_TOKEN"], + trustedPrivateHost: "mcp.corp.example", + allowedIps: ["10.20.30.40", "fd00::40"], + providerName: "alpha-mcp-local", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-local", + addedAt: "2026-08-04T00:00:00.000Z", + }; +} + +describe("trusted-private MCP lifecycle replay", () => { + it.each(adapters)("replays recorded pins without ambient DNS for $agent (#8267)", async ({ + adapter, + agent, + }) => { + const lookup = vi.spyOn(dns, "lookup").mockRejectedValue(new Error("ambient DNS used")); + const entry = privateEntry(adapter, agent); + + const targets = await preflightMcpEntryTargets([entry]); + const target = targets.get(entry.server); + + expect(lookup).not.toHaveBeenCalled(); + expect(target?.addresses).toEqual(entry.allowedIps); + expect(target?.trustedPrivateHost).toBe(entry.trustedPrivateHost); + expect(isTrustedPrivateEndpointCapability(target?.trustedPrivateCapability)).toBe(true); + expect(target && assertMcpBridgePolicyTarget(entry, target)).toEqual(entry.allowedIps); + }); + + it("rejects invalid durable private pins without consulting DNS (#8267)", async () => { + const lookup = vi.spyOn(dns, "lookup").mockRejectedValue(new Error("ambient DNS used")); + const entry = privateEntry("mcporter", "openclaw"); + entry.allowedIps = ["10.20.30.40", "8.8.8.8"]; + + await expect(preflightMcpEntryTargets([entry])).rejects.toThrow( + /invalid durable trusted-private intent/, + ); + expect(lookup).not.toHaveBeenCalled(); + }); + + it("rejects durable private intent for a different stored URL host (#8267)", async () => { + const lookup = vi.spyOn(dns, "lookup").mockRejectedValue(new Error("ambient DNS used")); + const entry = privateEntry("mcporter", "openclaw"); + entry.url = "https://other.corp.example/mcp"; + + await expect(preflightMcpEntryTargets([entry])).rejects.toThrow( + /trusted-private intent for a host that does not match its stored URL/, + ); + expect(lookup).not.toHaveBeenCalled(); + }); + + it("resumes an incomplete private add from recorded pins without ambient DNS (#8267)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-mcp-add-replay-")); + const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +delete process.env.LOCAL_MCP_TOKEN; +const dns = require("node:dns/promises"); +let dnsCalls = 0; +dns.lookup = async () => { dnsCalls += 1; throw new Error("ambient DNS used"); }; +const registry = require("./src/lib/state/registry.js"); +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { local: { + server: "local", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.corp.example/mcp", + env: ["LOCAL_MCP_TOKEN"], + trustedPrivateHost: "mcp.corp.example", + allowedIps: ["10.20.30.40"], + providerName: "alpha-mcp-local", + policyName: "mcp-bridge-local", + addedAt: "2026-08-04T00:00:00.000Z", + addState: "prepared", + } } }, +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.addMcpBridge("alpha", { + server: "local", + url: "https://mcp.corp.example/mcp", + env: [{ name: "LOCAL_MCP_TOKEN" }], + trustedPrivateHosts: ["mcp.corp.example"], +}).then( + () => process.exit(9), + (error) => process.stdout.write(JSON.stringify({ message: error.message, dnsCalls })), +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--require=${sourceRequireHook}`] + .filter(Boolean) + .join(" "), + }, + timeout: 15_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + message: + "Host environment variable 'LOCAL_MCP_TOKEN' is required to create MCP provider 'alpha-mcp-local'.", + dnsCalls: 0, + }); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts b/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts index e0e31d2f02..b946560292 100644 --- a/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts +++ b/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts @@ -3,13 +3,15 @@ import { stripAnsi } from "../../adapters/openshell/client"; import { runOpenshellProviderCommand } from "../../adapters/openshell/provider-command"; +import { replayTrustedPrivateEndpoint } from "../../security/trusted-private-endpoint"; import type { McpBridgeEntry } from "../../state/registry"; import { McpBridgeError } from "./mcp-bridge-contracts"; import { commandOutput, type OpenShellCommandResult } from "./mcp-bridge-output"; +import type { McpBridgeTargetValidation } from "./mcp-bridge-url-validation"; import { assertAuthenticatedBridgeEntry, normalizeMcpServerUrl, - validateMcpServerUrlResolvedTarget, + preflightMcpServerUrlResolvedTarget, } from "./mcp-bridge-validation"; export type McpProviderInspection = { @@ -261,18 +263,54 @@ export function assertMcpProviderRecoverable(entry: McpBridgeEntry): McpProvider export async function preflightMcpEntryTargets( entries: readonly McpBridgeEntry[], -): Promise> { +): Promise> { for (const entry of entries) assertAuthenticatedBridgeEntry(entry); const results = await Promise.all( entries.map(async (entry) => { - const normalized = normalizeMcpServerUrl(entry.url); + const trustedPrivateHosts = entry.trustedPrivateHost ? [entry.trustedPrivateHost] : undefined; + const normalized = normalizeMcpServerUrl(entry.url, { trustedPrivateHosts }); if (normalized !== entry.url) { throw new McpBridgeError( `MCP server '${entry.server}' has a non-canonical stored URL. Remove it with --force and add it again before lifecycle operations.`, ); } - const addresses = await validateMcpServerUrlResolvedTarget(new URL(normalized)); - return [entry.server, addresses] as const; + if (entry.trustedPrivateHost) { + if (new URL(normalized).hostname.toLowerCase() !== entry.trustedPrivateHost) { + throw new McpBridgeError( + `MCP server '${entry.server}' has trusted-private intent for a host that does not match its stored URL. Remove it with --force and add it again.`, + 2, + ); + } + const recordedPins = entry.allowedIps ?? []; + let replay; + try { + replay = replayTrustedPrivateEndpoint(entry.trustedPrivateHost, recordedPins); + } catch (error) { + throw new McpBridgeError( + `MCP server '${entry.server}' has invalid durable trusted-private intent: ${error instanceof Error ? error.message : String(error)}. Remove it with --force and add it again.`, + 2, + ); + } + if ( + replay.host !== entry.trustedPrivateHost || + recordedPins.length === 0 || + replay.addresses.length !== recordedPins.length || + replay.addresses.some((address, index) => address !== recordedPins[index]) + ) { + throw new McpBridgeError( + `MCP server '${entry.server}' has non-canonical trusted-private address pins. Remove it with --force and add it again.`, + 2, + ); + } + const target: McpBridgeTargetValidation = { + addresses: [...replay.addresses], + trustedPrivateCapability: replay.trustedPrivateCapability, + trustedPrivateHost: replay.host, + }; + return [entry.server, target] as const; + } + const target = await preflightMcpServerUrlResolvedTarget(new URL(normalized)); + return [entry.server, target] as const; }), ); return new Map(results); diff --git a/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts b/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts index bf2c4b8841..e02501968d 100644 --- a/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts +++ b/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts @@ -20,6 +20,7 @@ import { getSandboxAgent, getSandboxOrThrow, } from "./mcp-bridge-state"; +import type { McpBridgeTargetValidation } from "./mcp-bridge-url-validation"; import { assertAuthenticatedBridgeEntry, validateSandboxName } from "./mcp-bridge-validation"; type ReadOnlyValidationSnapshot = { @@ -123,13 +124,13 @@ function providerFingerprint(provider: ReturnType(); const targetsByServer = new Map(); for (const entry of entries) { - const resolvedAddresses = resolvedTargets.get(entry.server); - const policy = assertGeneratedPolicyExactReadOnly( - sandboxName, - entry, - adapter, - resolvedAddresses ?? [], - ); + const target = resolvedTargets.get(entry.server); + const policy = assertGeneratedPolicyExactReadOnly(sandboxName, entry, adapter, target ?? []); policyByServer.set(entry.server, policyFingerprint(policy)); const provider = inspectExactMcpDestroyProvider(entry, { allowMissing: false }); providerByServer.set(entry.server, providerFingerprint(provider)); - targetsByServer.set(entry.server, targetFingerprint(resolvedAddresses)); + targetsByServer.set(entry.server, targetFingerprint(target)); } return { policyByServer, providerByServer, targetsByServer }; } diff --git a/src/lib/actions/sandbox/mcp-bridge-rebuild.ts b/src/lib/actions/sandbox/mcp-bridge-rebuild.ts index 215a38d027..962dc2de26 100644 --- a/src/lib/actions/sandbox/mcp-bridge-rebuild.ts +++ b/src/lib/actions/sandbox/mcp-bridge-rebuild.ts @@ -224,7 +224,7 @@ export async function restoreMcpBridgesAfterRebuild( if (entries.length === 0) return; for (const entry of entries) assertAuthenticatedBridgeEntry(entry); const bridges = Object.fromEntries( - entries.map((entry) => [entry.server, { ...entry, env: [...entry.env] }]), + entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ); // Persist the recovery contract before touching the gateway. If refresh // fails, `mcp restart` remains retryable after the operator fixes the cause. diff --git a/src/lib/actions/sandbox/mcp-bridge-render.ts b/src/lib/actions/sandbox/mcp-bridge-render.ts index 1589d9e506..6f4bf49ae0 100644 --- a/src/lib/actions/sandbox/mcp-bridge-render.ts +++ b/src/lib/actions/sandbox/mcp-bridge-render.ts @@ -58,6 +58,10 @@ export function renderMcpBridgeStatus( console.log(` support: ${status.support.mode}`); if (status.support.reason) console.log(` reason: ${status.support.reason}`); if (status.url) console.log(` endpoint: ${status.url}`); + if (status.trustedPrivateTarget) { + console.log(` trusted private host: ${status.trustedPrivateTarget.host}`); + console.log(` private address pins: ${status.trustedPrivateTarget.state}`); + } if (status.addState) console.log(` add transaction: incomplete (${status.addState})`); console.log( ` provider: ${status.provider.registryPresent ? status.provider.name : "(none)"}`, diff --git a/src/lib/actions/sandbox/mcp-bridge-restart.ts b/src/lib/actions/sandbox/mcp-bridge-restart.ts index 680760c60f..18cb323875 100644 --- a/src/lib/actions/sandbox/mcp-bridge-restart.ts +++ b/src/lib/actions/sandbox/mcp-bridge-restart.ts @@ -36,6 +36,7 @@ import { nowIso, writeBridgeEntry, } from "./mcp-bridge-state"; +import type { McpBridgeTargetValidation } from "./mcp-bridge-url-validation"; import { assertAuthenticatedBridgeEntry, assertMcpCredentialBoundaryRuntimeVersion, @@ -44,16 +45,16 @@ import { } from "./mcp-bridge-validation"; function resolvedTargetPins( - resolvedByServer: ReadonlyMap, + resolvedByServer: ReadonlyMap, entry: McpBridgeEntry, -): string[] { - const addresses = resolvedByServer.get(entry.server); - if (!addresses || addresses.length === 0) { +): McpBridgeTargetValidation { + const target = resolvedByServer.get(entry.server); + if (!target || target.addresses.length === 0) { throw new McpBridgeError( - `MCP server '${entry.server}' has no validated public address pins. Refusing policy mutation.`, + `MCP server '${entry.server}' has no validated address pins. Refusing policy mutation.`, ); } - return addresses; + return target; } export async function restartMcpBridge(sandboxName: string, server?: string): Promise { @@ -122,12 +123,12 @@ async function restartMcpBridgeUnlocked(sandboxName: string, server?: string): P let entry = storedEntry; const envRefs = entry.env.map((envName) => ({ name: envName })); const adapterEnvValues = resolveCredentialEnv(envRefs); - const resolvedAddresses = resolvedTargetPins(resolvedByServer, entry); + const target = resolvedTargetPins(resolvedByServer, entry); let previousCredentialRevision: McpCredentialRevisionObservation | undefined; assertNoAttachedProviderCredentialCollision(sandboxName, entry); // Revalidate the actual running supervisor before rotating, recreating, // attaching, or re-registering an authenticated provider. - applyGeneratedPolicy(sandboxName, entry, resolvedAddresses); + applyGeneratedPolicy(sandboxName, entry, target); const providerResult = upsertMcpProvider(entry.providerName ?? "", envRefs, { allowExisting: true, expectedProviderId: entry.providerId, diff --git a/src/lib/actions/sandbox/mcp-bridge-status.ts b/src/lib/actions/sandbox/mcp-bridge-status.ts index eb95241368..a23df96769 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status.ts @@ -32,6 +32,10 @@ import { getSandboxOrThrow, } from "./mcp-bridge-state"; import { discoverMcpTools } from "./mcp-bridge-tool-discovery"; +import { + inspectMcpRecordedTargetPins, + type McpBridgeRecordedPinStatus, +} from "./mcp-bridge-url-validation"; import { assertAuthenticatedBridgeEntry, normalizeMcpServerUrl, @@ -68,7 +72,9 @@ const UNSUPPORTED_STORED_CREDENTIAL_WARNING = function storedUrlWarning(entry: McpBridgeEntry): string | undefined { try { - return normalizeMcpServerUrl(entry.url) === entry.url + return normalizeMcpServerUrl(entry.url, { + trustedPrivateHosts: entry.trustedPrivateHost ? [entry.trustedPrivateHost] : undefined, + }) === entry.url ? undefined : UNSUPPORTED_STORED_URL_WARNING; } catch { @@ -201,6 +207,21 @@ export async function statusMcpBridge( ); } + const privatePinStatusByServer = new Map(); + await Promise.all( + entries.map(async ([name, entry]) => { + if (!entry?.trustedPrivateHost || !entry.allowedIps) return; + privatePinStatusByServer.set( + name, + await inspectMcpRecordedTargetPins( + new URL(entry.url), + entry.trustedPrivateHost, + entry.allowedIps, + ), + ); + }), + ); + return entries.map(([name, entry]) => { const support = entry ? getPersistedBridgeSupport(entry) : getSupportSummary(agent); const registeredPolicy = getRegisteredGeneratedPolicy(sandboxName, entry); @@ -238,6 +259,16 @@ export async function statusMcpBridge( credentialWarning = storedCredentialWarning(entry); if (credentialWarning) warnings.push(credentialWarning); } + const privatePinStatus = privatePinStatusByServer.get(name); + if (privatePinStatus?.state === "drift") { + warnings.push( + "Trusted-private DNS answers differ from the recorded pins. Remove and re-add this server to approve changed pins.", + ); + } else if (privatePinStatus?.state === "unresolved") { + warnings.push( + "Trusted-private DNS resolution is unavailable. The recorded policy pins were not changed.", + ); + } const unsafeCredentialMayBeAttached = !!credentialWarning && !!entry?.providerName && attached !== false; const readiness = { @@ -278,6 +309,19 @@ export async function statusMcpBridge( warnings, support, ...(entry ? { url: entry.url } : {}), + ...(entry?.trustedPrivateHost && entry.allowedIps && privatePinStatus + ? { + trustedPrivateTarget: { + host: entry.trustedPrivateHost, + recordedPins: [...entry.allowedIps], + ...(privatePinStatus.currentAddresses + ? { currentPins: privatePinStatus.currentAddresses } + : {}), + state: privatePinStatus.state, + ...(privatePinStatus.detail ? { detail: privatePinStatus.detail } : {}), + }, + } + : {}), ...(entry?.addState ? { addState: entry.addState } : {}), env: { names: entry?.env ?? [], diff --git a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts index 9206b0b003..e59e1b646f 100644 --- a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts @@ -2,12 +2,19 @@ // SPDX-License-Identifier: Apache-2.0 import { resolveHostAddresses } from "../../adapters/dns/resolve"; +import { isLoopbackHostname } from "../../private-networks"; import { isBlockedMcpUrlTargetHost, isOpenShellMcpHostAlias, MCP_SERVER_URL_MAX_LENGTH, } from "../../security/mcp-url-target"; import { TOKEN_PREFIX_PATTERNS } from "../../security/secret-patterns"; +import { + assertEndpointResolvesPublic, + isTrustedPrivateEndpointCapability, + normalizeTrustedPrivateHost, + type TrustedPrivateEndpointCapability, +} from "../../security/trusted-private-endpoint"; import { McpBridgeError } from "./mcp-bridge-contracts"; export { MCP_SERVER_URL_MAX_LENGTH } from "../../security/mcp-url-target"; @@ -52,16 +59,47 @@ function rejectUnsupportedOpenShellMcpHostAlias(hostname: string): void { ); } -function validateMcpServerUrlTarget(parsed: URL): void { - if (isBlockedMcpUrlTargetHost(parsed.hostname)) { +function validateMcpServerUrlTarget( + parsed: URL, + trustedPrivateHosts: readonly string[] = [], +): void { + const normalizedHostname = parsed.hostname.toLowerCase(); + if ( + isBlockedMcpUrlTargetHost(parsed.hostname) && + !trustedPrivateHosts.includes(normalizedHostname) + ) { throw new McpBridgeError( - `MCP server URL host '${parsed.hostname}' is a private, local, or special-use IP address. Use a normal HTTPS DNS endpoint with public address records.`, + `MCP server URL host '${parsed.hostname}' is a private, local, or special-use IP address. Use a routed private HTTPS endpoint and pass --trusted-private-host ${parsed.hostname}, or use a normal public HTTPS DNS endpoint.`, 2, ); } } -export function normalizeMcpServerUrl(rawUrl: string): string { +export interface NormalizeMcpServerUrlOptions { + trustedPrivateHosts?: readonly string[]; +} + +export interface McpBridgeTargetValidation { + addresses: string[]; + trustedPrivateCapability?: TrustedPrivateEndpointCapability; + trustedPrivateHost?: string; +} + +export interface McpBridgeTargetPreflightOptions { + trustedPrivateHosts?: readonly string[]; + requireTrustedPrivateEndpoint?: boolean; +} + +export interface McpBridgeRecordedPinStatus { + state: "match" | "drift" | "unresolved"; + currentAddresses?: string[]; + detail?: string; +} + +export function normalizeMcpServerUrl( + rawUrl: string, + options: NormalizeMcpServerUrlOptions = {}, +): string { if (rawUrl.length > MCP_SERVER_URL_MAX_LENGTH) { throw new McpBridgeError( `MCP server URL must be at most ${MCP_SERVER_URL_MAX_LENGTH} characters.`, @@ -142,7 +180,19 @@ export function normalizeMcpServerUrl(rawUrl: string): string { ); } rejectUnsupportedOpenShellMcpHostAlias(parsed.hostname); - validateMcpServerUrlTarget(parsed); + const trustedPrivateHosts = (options.trustedPrivateHosts ?? []).map((host) => + normalizeTrustedPrivateHost(host), + ); + if ( + trustedPrivateHosts.includes(parsed.hostname.toLowerCase()) && + isLoopbackHostname(parsed.hostname) + ) { + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' is loopback. Sandbox loopback is not the host MCP service. Expose the exact MCP route through an HTTPS reverse proxy on a stable routed private address, then trust that routed host.`, + 2, + ); + } + validateMcpServerUrlTarget(parsed, trustedPrivateHosts); if (parsed.hostname.endsWith(".")) { throw new McpBridgeError( "MCP server URL hostnames must use canonical spelling without a trailing dot.", @@ -161,7 +211,10 @@ export function normalizeMcpServerUrl(rawUrl: string): string { return normalized; } -export async function validateMcpServerUrlResolvedTarget(parsed: URL): Promise { +export async function preflightMcpServerUrlResolvedTarget( + parsed: URL, + options: McpBridgeTargetPreflightOptions = {}, +): Promise { // invalidState: a hostname is public at add time but later rebinds to an // unpinned address. sourceBoundary: NemoClaw pins the add-time public answers; // OpenShell v0.0.85 resolves, validates every answer against allowed_ips, and @@ -174,34 +227,114 @@ export async function validateMcpServerUrlResolvedTarget(parsed: URL): Promise + normalizeTrustedPrivateHost(host), + ); if (isBlockedMcpUrlTargetHost(parsed.hostname)) { - validateMcpServerUrlTarget(parsed); + validateMcpServerUrlTarget(parsed, normalizedTrustedHosts); } - let addresses: Array<{ address: string }>; - try { - addresses = await resolveHostAddresses(parsed.hostname); - } catch (error) { - const detail = error instanceof Error && error.message ? ` ${error.message}` : ""; + const result = await assertEndpointResolvesPublic( + parsed.toString(), + async (hostname) => resolveHostAddresses(hostname), + { trustedPrivateHosts: normalizedTrustedHosts }, + ); + if (!result.ok) { + const privateAnswer = result.reason?.match( + /endpoint host "[^"]+" resolves to private\/internal address "([^"]+)"/, + )?.[1]; + if (privateAnswer) { + const guidance = isLoopbackHostname(privateAnswer) + ? " Sandbox loopback is not the host MCP service. Use an HTTPS reverse proxy on a stable routed private address." + : ` Use a routed private HTTPS endpoint and pass --trusted-private-host ${parsed.hostname}.`; + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' resolves to private, local, or special-use address '${privateAnswer}'.${guidance}`, + 2, + ); + } throw new McpBridgeError( - `MCP server URL host '${parsed.hostname}' could not be resolved before policy registration.${detail}`, + `MCP server URL target validation failed: ${result.reason ?? "the endpoint was rejected"}.`, 2, ); } + const literalAddress = /^\d{1,3}(?:\.\d{1,3}){3}$/.test(parsed.hostname) + ? parsed.hostname + : undefined; + const addresses = [...new Set(result.addresses?.length ? result.addresses : [literalAddress])] + .filter((address): address is string => !!address) + .map((address) => address.toLowerCase()) + .sort(); if (addresses.length === 0) { throw new McpBridgeError( `MCP server URL host '${parsed.hostname}' resolved without any addresses before policy registration.`, 2, ); } - for (const { address } of addresses) { - if (isBlockedMcpUrlTargetHost(address)) { + const normalizedHostname = normalizeTrustedPrivateHost(parsed.hostname); + const explicitTrust = normalizedTrustedHosts.includes(normalizedHostname); + if (result.trustedPrivateEndpoint) { + if (!isTrustedPrivateEndpointCapability(result.trustedPrivateCapability)) { throw new McpBridgeError( - `MCP server URL host '${parsed.hostname}' resolves to private, local, or special-use address '${address}'. Use a normal HTTPS DNS endpoint with public address records.`, + `MCP server URL host '${normalizedHostname}' did not return a provenance-checked trusted-private capability.`, 2, ); } + const capabilityAddresses = [...result.trustedPrivateCapability.addresses] + .map((address) => address.toLowerCase()) + .sort(); + if ( + capabilityAddresses.length !== addresses.length || + capabilityAddresses.some((address, index) => address !== addresses[index]) + ) { + throw new McpBridgeError( + `MCP server URL host '${normalizedHostname}' returned mixed public and private addresses. Trusted-private MCP endpoints must resolve only to supported routed private addresses.`, + 2, + ); + } + return { + addresses, + trustedPrivateCapability: result.trustedPrivateCapability, + trustedPrivateHost: normalizedHostname, + }; + } + if (explicitTrust && options.requireTrustedPrivateEndpoint) { + throw new McpBridgeError( + `--trusted-private-host ${normalizedHostname} is unused because the MCP endpoint did not resolve to supported routed private addresses.`, + 2, + ); + } + return { addresses }; +} + +export async function validateMcpServerUrlResolvedTarget(parsed: URL): Promise { + return (await preflightMcpServerUrlResolvedTarget(parsed)).addresses; +} + +export async function inspectMcpRecordedTargetPins( + parsed: URL, + trustedPrivateHost: string, + recordedPins: readonly string[], +): Promise { + try { + const target = await preflightMcpServerUrlResolvedTarget(parsed, { + trustedPrivateHosts: [trustedPrivateHost], + requireTrustedPrivateEndpoint: true, + }); + const matches = + recordedPins.length === target.addresses.length && + recordedPins.every((address, index) => address === target.addresses[index]); + return { + state: matches ? "match" : "drift", + currentAddresses: target.addresses, + ...(!matches + ? { detail: "Current DNS answers differ from the recorded trusted-private pins." } + : {}), + }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const unresolved = + /cannot resolve|could not be resolved|did not resolve|without any addresses/i.test(detail); + return { state: unresolved ? "unresolved" : "drift", detail }; } - return [...new Set(addresses.map(({ address }) => address.toLowerCase()))].sort(); } export function parseMcpUrl(rawUrl: string): URL { diff --git a/src/lib/actions/sandbox/mcp-bridge-validation.ts b/src/lib/actions/sandbox/mcp-bridge-validation.ts index 8369acf05f..e7106d59fd 100644 --- a/src/lib/actions/sandbox/mcp-bridge-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-validation.ts @@ -6,6 +6,10 @@ import crypto from "node:crypto"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { diagnosticPreview } from "../../name-validation"; +import { + normalizeTrustedPrivateHost, + parseTrustedPrivateHosts, +} from "../../security/trusted-private-endpoint"; import type { McpBridgeEntry } from "../../state/registry"; import { buildSubprocessEnv, isSubprocessEnvNameAllowed } from "../../subprocess-env"; import { @@ -24,6 +28,7 @@ export { MCP_SERVER_URL_MAX_LENGTH, normalizeMcpServerUrl, parseMcpUrl, + preflightMcpServerUrlResolvedTarget, validateMcpServerUrlResolvedTarget, } from "./mcp-bridge-url-validation"; @@ -227,8 +232,9 @@ export function validatePersistedMcpCredentialEnvName(name: string): void { export function parseMcpAddArgs(argv: string[]): ParsedMcpAddArgs { const env: ParsedEnvReference[] = []; + const trustedPrivateHosts: string[] = []; let server = ""; - let url = ""; + let rawUrl = ""; for (let i = 0; i < argv.length; i++) { const token = argv[i]; @@ -267,11 +273,29 @@ export function parseMcpAddArgs(argv: string[]): ParsedMcpAddArgs { continue; } if (token === "--url") { - url = normalizeMcpServerUrl(argv[++i] ?? ""); + rawUrl = argv[++i] ?? ""; continue; } if (token?.startsWith("--url=")) { - url = normalizeMcpServerUrl(token.slice("--url=".length)); + rawUrl = token.slice("--url=".length); + continue; + } + if (token === "--trusted-private-host") { + try { + trustedPrivateHosts.push(normalizeTrustedPrivateHost(argv[++i] ?? "")); + } catch (error) { + throw new McpBridgeError(error instanceof Error ? error.message : String(error), 2); + } + continue; + } + if (token?.startsWith("--trusted-private-host=")) { + try { + trustedPrivateHosts.push( + normalizeTrustedPrivateHost(token.slice("--trusted-private-host=".length)), + ); + } catch (error) { + throw new McpBridgeError(error instanceof Error ? error.message : String(error), 2); + } continue; } if (token?.startsWith("-")) { @@ -283,20 +307,45 @@ export function parseMcpAddArgs(argv: string[]): ParsedMcpAddArgs { continue; } throw new McpBridgeError( - "Usage: nemoclaw mcp add --url --env KEY", + "Usage: nemoclaw mcp add --url --env KEY [--trusted-private-host HOST]", 2, ); } if (!server) { throw new McpBridgeError( - "Usage: nemoclaw mcp add --url --env KEY", + "Usage: nemoclaw mcp add --url --env KEY [--trusted-private-host HOST]", 2, ); } - if (!url) { + if (!rawUrl) { throw new McpBridgeError("MCP server URL is required. Pass --url .", 2); } + if (new Set(trustedPrivateHosts).size !== trustedPrivateHosts.length) { + throw new McpBridgeError( + "Duplicate --trusted-private-host declarations are not accepted after normalization.", + 2, + ); + } + let configuredTrustedPrivateHosts: string[]; + try { + configuredTrustedPrivateHosts = parseTrustedPrivateHosts( + process.env.NEMOCLAW_TRUSTED_PRIVATE_HOSTS, + ); + } catch (error) { + throw new McpBridgeError(error instanceof Error ? error.message : String(error), 2); + } + const url = normalizeMcpServerUrl(rawUrl, { + trustedPrivateHosts: [...new Set([...trustedPrivateHosts, ...configuredTrustedPrivateHosts])], + }); + const urlHost = new URL(url).hostname.toLowerCase(); + const unrelatedTrustedHost = trustedPrivateHosts.find((host) => host !== urlHost); + if (unrelatedTrustedHost) { + throw new McpBridgeError( + `--trusted-private-host ${unrelatedTrustedHost} does not match MCP server URL host '${urlHost}'.`, + 2, + ); + } if (env.length !== 1) { throw new McpBridgeError( "Authenticated MCP requires exactly one --env KEY bearer credential reference.", @@ -304,7 +353,12 @@ export function parseMcpAddArgs(argv: string[]): ParsedMcpAddArgs { ); } - return { server, url, env }; + return { + server, + url, + env, + ...(trustedPrivateHosts.length > 0 ? { trustedPrivateHosts } : {}), + }; } export function uniqueEnvNames(env: readonly ParsedEnvReference[] | readonly string[]): string[] { diff --git a/src/lib/actions/sandbox/mcp-bridge.ts b/src/lib/actions/sandbox/mcp-bridge.ts index ef7f2d77ce..ff2b5a783e 100644 --- a/src/lib/actions/sandbox/mcp-bridge.ts +++ b/src/lib/actions/sandbox/mcp-bridge.ts @@ -249,11 +249,13 @@ function renderMcpHelp(subcommand: string): void { switch (subcommand) { case "add": console.log(`USAGE - nemoclaw mcp add --url --env KEY + nemoclaw mcp add --url --env KEY [--trusted-private-host HOST] FLAGS --url URL MCP Streamable HTTP endpoint --env KEY Required host credential reference registered with OpenShell + --trusted-private-host HOST + Trust the exact URL host when it resolves only to routed private addresses --no-probe Skip the post-add wire-level credential-resolution probe SECURITY @@ -314,7 +316,7 @@ export async function dispatchMcpBridgeCommand( const { probe, rest: addRest } = parseProbeFlags(rest); if (probe === true) throw new McpBridgeError( - "Usage: nemoclaw mcp add --url --env KEY [--no-probe]", + "Usage: nemoclaw mcp add --url --env KEY [--trusted-private-host HOST] [--no-probe]", 2, ); const options = parseMcpAddArgs(addRest); diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index ebad45040b..8f9f149f2f 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -181,7 +181,14 @@ async function addSandboxPolicyUnlocked( sandboxName: string, options: PolicyAddOptions, ): Promise { - const { dryRun, skipConfirm, source, presetArg } = parsePolicyAddOptions(options); + const { + dryRun, + skipConfirm, + source, + presetArg, + trustedPrivateHosts, + commandTrustedPrivateHosts, + } = parsePolicyAddOptions(options); if (source.kind === "error") { console.error(` ${source.message}`); @@ -189,7 +196,19 @@ async function addSandboxPolicyUnlocked( } if (source.kind === "file") { - const ok = await applyExternalPreset(sandboxName, source.path, { dryRun, yes: skipConfirm }); + const prepared = await prepareExternalPolicyPresets( + [source.path], + trustedPrivateHosts, + commandTrustedPrivateHosts, + ); + if (!prepared) { + process.exit(1); + return; + } + const ok = await applyExternalPreset(sandboxName, prepared[0], { + dryRun, + yes: skipConfirm, + }); if (!ok) process.exit(1); return; } @@ -213,10 +232,21 @@ async function addSandboxPolicyUnlocked( console.error(` No .yaml/.yml preset files in ${dirPath}`); process.exit(1); } - for (const f of files) { - const ok = await applyExternalPreset(sandboxName, f, { dryRun, yes: skipConfirm }); + const prepared = await prepareExternalPolicyPresets( + files, + trustedPrivateHosts, + commandTrustedPrivateHosts, + ); + if (!prepared) { + process.exit(1); + return; + } + for (const preset of prepared) { + const ok = await applyExternalPreset(sandboxName, preset, { dryRun, yes: skipConfirm }); if (!ok) { - console.error(` Aborting --from-dir: ${f} failed. Remaining presets not applied.`); + console.error( + ` Aborting --from-dir: ${preset.filePath} failed. Remaining presets not applied.`, + ); process.exit(1); } } @@ -346,21 +376,40 @@ async function addSandboxPolicyUnlocked( * `policies.applyPresetContent`. Returns `true` on success, `false` on any * load/apply failure so the caller can decide whether to abort. */ -async function applyExternalPreset( - sandboxName: string, - filePath: string, - { dryRun, yes }: { dryRun: boolean; yes: boolean }, -): Promise { - let loaded; +async function prepareExternalPolicyPresets( + filePaths: readonly string[], + trustedPrivateHosts: readonly string[], + commandTrustedPrivateHosts: readonly string[], +): Promise { + const loaded: policies.ExternalPolicyPreset[] = []; + for (const filePath of filePaths) { + let preset; + try { + preset = policies.loadPresetFromFile(filePath); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.error(` Failed to load preset ${filePath}: ${message}`); + return null; + } + if (!preset) return null; + loaded.push({ filePath, ...preset }); + } try { - loaded = policies.loadPresetFromFile(filePath); + return await policies.prepareTrustedPrivatePolicyPresets(loaded, trustedPrivateHosts, { + requiredDeclarations: commandTrustedPrivateHosts, + }); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); - console.error(` Failed to load preset ${filePath}: ${message}`); - return false; + console.error(` Failed to prepare trusted private policy endpoints: ${message}`); + return null; } - if (!loaded) return false; +} +async function applyExternalPreset( + sandboxName: string, + loaded: policies.ExternalPolicyPreset, + { dryRun, yes }: { dryRun: boolean; yes: boolean }, +): Promise { const scopeLines = policies.renderPresetScope(loaded.content); if (scopeLines.length > 0) { console.log(` [${loaded.presetName}]`); @@ -377,14 +426,19 @@ async function applyExternalPreset( if (!yes) { const confirm = await askPrompt( - ` Apply '${loaded.presetName}' from ${filePath} to sandbox '${sandboxName}'? [Y/n]: `, + ` Apply '${loaded.presetName}' from ${loaded.filePath} to sandbox '${sandboxName}'? [Y/n]: `, ); if (confirm.trim().toLowerCase().startsWith("n")) return true; // user-cancel counts as success (no abort) } try { const result = policies.applyPresetContent(sandboxName, loaded.presetName, loaded.content, { - custom: { sourcePath: path.resolve(filePath) }, + custom: { + sourcePath: path.resolve(loaded.filePath), + ...(loaded.trustedPrivatePinCapability + ? { trustedPrivatePinCapability: loaded.trustedPrivatePinCapability } + : {}), + }, suppressDisclosure: true, }); if (result !== false) { diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 0768062bb0..6b97c75427 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -377,6 +377,22 @@ async function rebuildSandboxUnlocked( }, durableConfig.webSearchConfig, ); + const capturedCustomPolicies = + backup.backupManifest?.customPolicies?.map((entry) => ({ ...entry })) ?? + preservedCustomPolicies; + const customPoliciesWithRegistryPinAuthority = capturedCustomPolicies.map((entry) => { + const { trustedPrivatePins: _capturedPinAuthority, ...captured } = entry; + const registryAuthority = preservedCustomPolicies.find( + (candidate) => + candidate.name === entry.name && + candidate.content === entry.content && + candidate.trustedPrivatePins?.contentDigest === entry.trustedPrivatePins?.contentDigest, + )?.trustedPrivatePins; + return { + ...captured, + ...(registryAuthority ? { trustedPrivatePins: registryAuthority } : {}), + }; + }); const restored = runRebuildRestorePhase({ sandboxName, @@ -384,9 +400,7 @@ async function rebuildSandboxUnlocked( targetImageIsCustom: Boolean(fromDockerfile), backupManifest: backup.backupManifest, policyPresets: targetPolicyPresets, - customPolicies: - backup.backupManifest?.customPolicies?.map((entry) => ({ ...entry })) ?? - preservedCustomPolicies, + customPolicies: customPoliciesWithRegistryPinAuthority, reconcileManagedDcodeObservability: rebuildAgent === DCODE_AGENT_NAME, log, }); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts index 831101486d..cab3cf5263 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; + import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as policies from "../../policy"; @@ -138,11 +140,17 @@ describe("rebuild policy restore fidelity", () => { failedFiles: [], }); const applyPresetContent = vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + const privateContent = + "network_policies:\n custom-egress:\n endpoints:\n - host: api.corp.example\n allowed_ips: [10.20.30.40]\n"; const customPolicies = [ { name: "custom-egress", - content: "network_policies:\n custom-egress: {}\n", + content: privateContent, sourcePath: "/tmp/custom-egress.yaml", + trustedPrivatePins: { + version: 1 as const, + contentDigest: createHash("sha256").update(privateContent).digest("hex"), + }, }, ]; const result = runStandardRebuildRestorePhase({ @@ -158,7 +166,14 @@ describe("rebuild policy restore fidelity", () => { "alpha", "custom-egress", customPolicies[0]!.content, - { custom: { sourcePath: "/tmp/custom-egress.yaml" } }, + { + custom: { + sourcePath: "/tmp/custom-egress.yaml", + trustedPrivatePinCapability: expect.objectContaining({ + receipt: customPolicies[0]!.trustedPrivatePins, + }), + }, + }, ); expect(result.restoredPresets).toEqual(["custom-egress"]); expect(result.finalPresets).toEqual(["custom-egress"]); diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts index 057d7bd9ba..e2ccf7e2e1 100644 --- a/src/lib/actions/sandbox/rebuild-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -8,6 +8,7 @@ import { OBSERVABILITY_POLICY_BINDING, } from "../../onboard/observability-policy-presets"; import * as policies from "../../policy"; +import { replayTrustedPrivatePolicyPinCapability } from "../../policy/trusted-private-endpoints"; import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; @@ -260,8 +261,14 @@ export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): Rebuild for (const entry of replayableCustomPolicies) { try { log(`Applying custom preset: ${entry.name}`); + const trustedPrivatePinCapability = entry.trustedPrivatePins + ? replayTrustedPrivatePolicyPinCapability(entry.content, entry.trustedPrivatePins) + : undefined; const applied = policies.applyPresetContent(sandboxName, entry.name, entry.content, { - custom: { sourcePath: entry.sourcePath }, + custom: { + sourcePath: entry.sourcePath, + ...(trustedPrivatePinCapability ? { trustedPrivatePinCapability } : {}), + }, }); if (applied) { restoredCustomPresets.push(entry.name); diff --git a/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts b/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts index be0e810023..d03fb7f5d1 100644 --- a/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts @@ -8,6 +8,44 @@ import * as f from "./snapshot-restore-test-fixture"; beforeEach(f.resetSnapshotRestoreMocks); afterEach(f.cleanupSnapshotRestoreMocks); describe("runSandboxSnapshot restore: observability policy reconciliation", () => { + it("does not promote a forged snapshot digest into trusted-private pin authority", async () => { + const content = + "network_policies:\n private-api:\n endpoints:\n - host: api.corp.example\n allowed_ips:\n - 10.20.30.40\n"; + const customPolicy = { + name: "private-api", + content, + sourcePath: "/policies/private-api.yaml", + trustedPrivatePins: { + version: 1 as const, + contentDigest: "a".repeat(64), + }, + }; + f.getSandboxMock.mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + policyTier: "balanced", + } as never); + f.getLatestBackupMock.mockReturnValue({ + ...f.latestBackupFixture, + policyPresets: [customPolicy.name], + customPolicies: [customPolicy], + }); + f.getCustomPoliciesMock.mockReturnValue([]); + f.applyPresetContentMock.mockReturnValue(false); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(f.applyPresetContentMock).toHaveBeenCalledWith( + "alpha", + customPolicy.name, + customPolicy.content, + { custom: { sourcePath: customPolicy.sourcePath } }, + ); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain("private-api (apply failed)"); + }); + it("does not resurrect an earlier removed preset while restoring unverified OTLP attribution", async () => { let registryEntry = { name: "alpha", diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 77595a0ab1..1445aed138 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -924,7 +924,12 @@ function reconcileSnapshotCustomPolicies( const toRemove = currentCustom.filter((c) => !snapshotByName.has(c.name)); const toAdd = snapshotCustom.filter((sp) => { const current = currentByName.get(sp.name); - return !current || current.content !== sp.content || current.sourcePath !== sp.sourcePath; + return ( + !current || + current.content !== sp.content || + current.sourcePath !== sp.sourcePath || + current.trustedPrivatePins?.contentDigest !== sp.trustedPrivatePins?.contentDigest + ); }); if (toRemove.length === 0 && toAdd.length === 0) return; @@ -946,9 +951,20 @@ function reconcileSnapshotCustomPolicies( } for (const entry of toAdd) { try { + const currentAuthority = currentByName.get(entry.name); + const trustedPrivatePinCapability = + currentAuthority?.content === entry.content && currentAuthority.trustedPrivatePins + ? policies.replayTrustedPrivatePolicyPinCapability( + currentAuthority.content, + currentAuthority.trustedPrivatePins, + ) + : undefined; if ( !policies.applyPresetContent(targetSandbox, entry.name, entry.content, { - custom: { sourcePath: entry.sourcePath }, + custom: { + sourcePath: entry.sourcePath, + ...(trustedPrivatePinCapability ? { trustedPrivatePinCapability } : {}), + }, }) ) { failed.push(`${entry.name} (apply failed)`); diff --git a/src/lib/cli/public-display-defaults.ts b/src/lib/cli/public-display-defaults.ts index 161d980ae2..fcf368d48e 100644 --- a/src/lib/cli/public-display-defaults.ts +++ b/src/lib/cli/public-display-defaults.ts @@ -360,7 +360,8 @@ const PUBLIC_DISPLAY_LAYOUT: Record = { { group: "Policy Presets", order: 17, - flags: "(--yes, -y, --dry-run, --from-file , --from-dir )", + flags: + "(--yes, -y, --dry-run, --from-file , --from-dir , --trusted-private-host )", }, ], "sandbox:policy:explain": [ diff --git a/src/lib/domain/policy-channel.test.ts b/src/lib/domain/policy-channel.test.ts index faf0d6fd07..a85f39da56 100644 --- a/src/lib/domain/policy-channel.test.ts +++ b/src/lib/domain/policy-channel.test.ts @@ -17,6 +17,8 @@ describe("policy channel helpers", () => { skipConfirm: true, source: { kind: "file", path: "preset.yaml" }, presetArg: "github", + trustedPrivateHosts: [], + commandTrustedPrivateHosts: [], }); }); @@ -26,12 +28,48 @@ describe("policy channel helpers", () => { skipConfirm: false, source: { kind: "error", message: "--from-file and --from-dir are mutually exclusive." }, presetArg: null, + trustedPrivateHosts: [], + commandTrustedPrivateHosts: [], }); expect(parsePolicyAddOptions({ fromFile: "" }, {})).toEqual({ dryRun: false, skipConfirm: false, source: { kind: "error", message: "--from-file requires a path argument." }, presetArg: null, + trustedPrivateHosts: [], + commandTrustedPrivateHosts: [], + }); + }); + + it("limits trusted private hosts to custom policy input (#8176)", () => { + expect(parsePolicyAddOptions({ trustedPrivateHosts: ["api.corp.example"] }, {})).toEqual({ + dryRun: false, + skipConfirm: false, + source: { + kind: "error", + message: "--trusted-private-host requires --from-file or --from-dir.", + }, + presetArg: null, + trustedPrivateHosts: ["api.corp.example"], + commandTrustedPrivateHosts: ["api.corp.example"], + }); + + expect( + parsePolicyAddOptions( + { fromFile: "preset.yaml", trustedPrivateHosts: ["api.corp.example"] }, + {}, + ).trustedPrivateHosts, + ).toEqual(["api.corp.example"]); + + expect( + parsePolicyAddOptions( + { fromFile: "preset.yaml" }, + { NEMOCLAW_TRUSTED_PRIVATE_HOSTS: "api.corp.example,other.corp.example" }, + ), + ).toMatchObject({ + source: { kind: "file", path: "preset.yaml" }, + trustedPrivateHosts: ["api.corp.example", "other.corp.example"], + commandTrustedPrivateHosts: [], }); }); diff --git a/src/lib/domain/policy-channel.ts b/src/lib/domain/policy-channel.ts index 5480368a1b..6cf7fcc835 100644 --- a/src/lib/domain/policy-channel.ts +++ b/src/lib/domain/policy-channel.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { parseTrustedPrivateHosts } from "../security/trusted-private-endpoint"; + export type CustomPolicySource = | { kind: "none" } | { kind: "file"; path: string } @@ -12,6 +14,8 @@ export type ParsedPolicyAddOptions = { skipConfirm: boolean; source: CustomPolicySource; presetArg: string | null; + trustedPrivateHosts: readonly string[]; + commandTrustedPrivateHosts: readonly string[]; }; export type PolicyAddOptions = { @@ -21,6 +25,7 @@ export type PolicyAddOptions = { force?: boolean; fromFile?: string; fromDir?: string; + trustedPrivateHosts?: readonly string[]; }; export type PolicyRemoveOptions = { @@ -63,10 +68,25 @@ export function parsePolicyAddOptions( options: PolicyAddOptions = {}, env: Record = process.env, ): ParsedPolicyAddOptions { + const source = customPolicySourceFromOptions(options); + const commandTrustedPrivateHosts = options.trustedPrivateHosts ?? []; + const trustedPrivateHosts = [ + ...parseTrustedPrivateHosts(env.NEMOCLAW_TRUSTED_PRIVATE_HOSTS), + ...commandTrustedPrivateHosts, + ]; + const effectiveSource = + source.kind === "none" && commandTrustedPrivateHosts.length > 0 + ? { + kind: "error" as const, + message: "--trusted-private-host requires --from-file or --from-dir.", + } + : source; return { dryRun: Boolean(options.dryRun), skipConfirm: Boolean(options.yes || options.force || env.NEMOCLAW_NON_INTERACTIVE === "1"), - source: customPolicySourceFromOptions(options), + source: effectiveSource, presetArg: options.preset ?? null, + trustedPrivateHosts, + commandTrustedPrivateHosts, }; } diff --git a/src/lib/inference/compatible-endpoint-context.test.ts b/src/lib/inference/compatible-endpoint-context.test.ts index 70b5948a6b..d67b488257 100644 --- a/src/lib/inference/compatible-endpoint-context.test.ts +++ b/src/lib/inference/compatible-endpoint-context.test.ts @@ -205,7 +205,7 @@ describe("compatible-endpoint context window", () => { it("probes an exactly allowlisted private endpoint with its address capability (#6861)", async () => { const fetchModels = vi.fn(() => ({ data: [{ id: "model-a", max_model_len: 65_536 }] })); const env: NodeJS.ProcessEnv = { - NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS: "llm.corp.example", + NEMOCLAW_TRUSTED_PRIVATE_HOSTS: "llm.corp.example", }; await applyCompatibleEndpointContextWindow("https://llm.corp.example/v1", "model-a", { env, diff --git a/src/lib/inference/compatible-endpoint-context.ts b/src/lib/inference/compatible-endpoint-context.ts index e694b78721..7760a6a503 100644 --- a/src/lib/inference/compatible-endpoint-context.ts +++ b/src/lib/inference/compatible-endpoint-context.ts @@ -8,7 +8,7 @@ import { assertEndpointResolvesPublic, buildResolvePinArgs, type EndpointDnsLookupFn, - parseTrustedPrivateInferenceHosts, + parseTrustedPrivateInferenceHostsFromEnv, type TrustedPrivateEndpointCapability, } from "./endpoint-ssrf-preflight"; import { @@ -246,9 +246,7 @@ export async function applyCompatibleEndpointContextWindow( // options.resolveHost. No env-gated bypass: an ambient VITEST flag must never // disable SSRF enforcement (cv review, PR #6293). const preflight = await assertEndpointResolvesPublic(endpointUrl, options.resolveHost, { - trustedPrivateHosts: parseTrustedPrivateInferenceHosts( - env.NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS, - ), + trustedPrivateHosts: parseTrustedPrivateInferenceHostsFromEnv(env), }); if (!preflight.ok) { logger.warn( diff --git a/src/lib/inference/endpoint-ssrf-preflight.test.ts b/src/lib/inference/endpoint-ssrf-preflight.test.ts index 744bbe4a9a..bdcbebfa29 100644 --- a/src/lib/inference/endpoint-ssrf-preflight.test.ts +++ b/src/lib/inference/endpoint-ssrf-preflight.test.ts @@ -11,6 +11,7 @@ import { isOpenShellManagedHost, isTrustedPrivateEndpointCapability, parseTrustedPrivateInferenceHosts, + parseTrustedPrivateInferenceHostsFromEnv, } from "./endpoint-ssrf-preflight"; const resolverTo = (address: string): EndpointDnsLookupFn => @@ -167,6 +168,15 @@ describe("assertEndpointResolvesPublic (#6293)", () => { ).toEqual(["llm.corp.example", "10.0.0.8"]); }); + it("unions generic and legacy inference trust sources (#8176)", () => { + expect( + parseTrustedPrivateInferenceHostsFromEnv({ + NEMOCLAW_TRUSTED_PRIVATE_HOSTS: "mcp.corp.example,llm.corp.example", + NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS: "LLM.CORP.EXAMPLE.,10.0.0.8", + }), + ).toEqual(["mcp.corp.example", "llm.corp.example", "10.0.0.8"]); + }); + it.each([ "http://127.0.0.1:8000/v1", "http://localhost:8000/v1", diff --git a/src/lib/inference/endpoint-ssrf-preflight.ts b/src/lib/inference/endpoint-ssrf-preflight.ts index 5015f6f9b7..7e95c411d5 100644 --- a/src/lib/inference/endpoint-ssrf-preflight.ts +++ b/src/lib/inference/endpoint-ssrf-preflight.ts @@ -1,285 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { lookup as dnsLookup } from "node:dns/promises"; -import { BlockList, isIP } from "node:net"; +import { parseTrustedPrivateHosts } from "../security/trusted-private-endpoint"; -/** Injectable DNS resolver, shaped like `dns/promises` `lookup(host, {all:true})`. */ -export type EndpointDnsLookupFn = ( - hostname: string, - options: { all: true }, -) => Promise>; +export * from "../security/trusted-private-endpoint"; +export { parseTrustedPrivateHosts as parseTrustedPrivateInferenceHosts } from "../security/trusted-private-endpoint"; -/** - * NemoClaw's own OpenShell-managed infrastructure hostnames. These resolve to - * the host loopback or the OpenShell L7 proxy *by design* (see - * `subprocess-env` `withLocalNoProxy` and `verify-deployment` for - * `inference.local`), so — unlike an arbitrary user-supplied public name — they - * are trusted aliases, not attacker-controlled names subject to DNS rebinding. - * They are exempt from the public-resolution requirement (like explicit - * loopback) and connect normally without `--resolve` pinning. This mirrors the - * MCP URL-target allowlist (`isOpenShellMcpHostAlias`), additionally covering - * `inference.local` — the managed sandbox inference route a compatible endpoint - * legitimately targets (#6293). - */ -const OPENSHELL_MANAGED_HOSTS = new Set([ - "inference.local", - "host.openshell.internal", - "host.docker.internal", - "host.containers.internal", -]); - -// An explicit operator allowlist may admit routable enterprise/private address -// space, but never link-local metadata, multicast, documentation, translation, -// or other reserved ranges covered by the broader SSRF denylist. -const OPERATOR_TRUSTABLE_PRIVATE_NETWORKS = new BlockList(); -OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("10.0.0.0", 8, "ipv4"); -OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("100.64.0.0", 10, "ipv4"); -OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("172.16.0.0", 12, "ipv4"); -OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("192.168.0.0", 16, "ipv4"); -OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("fc00::", 7, "ipv6"); - -declare const trustedPrivateEndpointCapabilityBrand: unique symbol; - -/** - * Ephemeral proof that the shared SSRF preflight admitted an exact set of - * operator-trusted private addresses. Callers can carry this value, but only - * this module can issue one and the curl boundary validates its provenance. - */ -export interface TrustedPrivateEndpointCapability { - readonly addresses: readonly string[]; - readonly [trustedPrivateEndpointCapabilityBrand]: true; -} - -const TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES = new WeakSet(); - -function issueTrustedPrivateEndpointCapability( - addresses: readonly string[], -): TrustedPrivateEndpointCapability { - const capability = Object.freeze({ - addresses: Object.freeze([...new Set(addresses)]), - }) as unknown as TrustedPrivateEndpointCapability; - TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES.add(capability); - return capability; -} - -/** True only for a capability issued by this module in the current process. */ -export function isTrustedPrivateEndpointCapability( - value: unknown, -): value is TrustedPrivateEndpointCapability { - return ( - typeof value === "object" && value !== null && TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES.has(value) - ); -} -export function isOperatorTrustablePrivateIp(address: string): boolean { - const family = isIP(address); - return ( - family !== 0 && - OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.check(address, family === 6 ? "ipv6" : "ipv4") - ); -} - -/** True when `hostname` is a NemoClaw OpenShell-managed infrastructure alias. */ -export function isOpenShellManagedHost(hostname: string): boolean { - const normalised = ( - hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname - ) - .replace(/\.$/, "") - .toLowerCase(); - return OPENSHELL_MANAGED_HOSTS.has(normalised); -} - -export interface EndpointSsrfPreflightResult { - ok: boolean; - /** Human-readable reason, present only when `ok === false`. */ - reason?: string; - /** - * Validated public addresses the endpoint host resolved to, for connection - * pinning (curl `--resolve`) so a subsequent probe cannot re-resolve the name - * to a rebound private/internal address (TOCTOU). Present only when - * `ok === true` and pinning applies — resolved public names and public IP - * literals. An empty array is the explicit trusted-no-pin capability for - * loopback, OpenShell-managed aliases, and public IP literals. Callers must - * preserve it so credentialed probes bypass ambient proxies even when no - * curl `--resolve` argument is needed. - */ - addresses?: string[]; - /** Non-forgeable proof of the exact private addresses admitted by the operator allowlist. */ - trustedPrivateCapability?: TrustedPrivateEndpointCapability; - /** True only when an exact operator allowlist entry admitted a private address. */ - trustedPrivateEndpoint?: true; -} - -export interface EndpointSsrfPreflightOptions { - /** Exact hostnames or IP literals the operator explicitly trusts on a private network. */ - trustedPrivateHosts?: readonly string[]; -} - -function normalizeEndpointHost(hostname: string): string { - return (hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname) - .replace(/\.$/, "") - .toLowerCase(); -} - -/** Parse the explicit private-inference hostname allowlist. Wildcards are not supported. */ -export function parseTrustedPrivateInferenceHosts(value: string | undefined): string[] { +/** Read the generic trust source and the legacy inference-only source. */ +export function parseTrustedPrivateInferenceHostsFromEnv(env: NodeJS.ProcessEnv): string[] { return [ - ...new Set( - String(value ?? "") - .split(",") - .map((entry) => normalizeEndpointHost(entry.trim())) - .filter(Boolean), - ), + ...new Set([ + ...parseTrustedPrivateHosts(env.NEMOCLAW_TRUSTED_PRIVATE_HOSTS), + ...parseTrustedPrivateHosts(env.NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS), + ]), ]; } - -/** - * DNS-backed SSRF preflight for a user-supplied inference endpoint, run before - * any privileged host-side curl during onboarding. - * - * The string-level `isPrivateHostname` guards elsewhere block literal private - * IPs and reserved names, but a public-looking name (`https://vllm.example/v1`) - * can still resolve to `127.0.0.1`, `169.254.169.254`, or RFC1918 space and make - * the onboarding host contact internal services before the sandbox and its - * OpenShell network policy exist. This resolves the hostname first and refuses - * when it — or any resolved address — is private/reserved. It complements the - * authoritative config-write DNS-pinning boundary (`validateUrlValueWithDnsResult`) - * which runs later, before the URL is persisted. - * - * Loopback (127.0.0.0/8, ::1, localhost) is exempt ONLY when the endpoint - * hostname is itself loopback — a locally-run vLLM/Ollama server the user - * explicitly configured. A public name that *resolves* to loopback is treated - * as a rebinding attempt and refused. The resolver is injectable for tests and - * the check fails closed on resolver error or an empty result. - * - * See PR #6293 PRA-4 (GPT-5.5 advisor). - */ -export async function assertEndpointResolvesPublic( - endpointUrl: string, - lookup: EndpointDnsLookupFn = dnsLookup as unknown as EndpointDnsLookupFn, - options: EndpointSsrfPreflightOptions = {}, -): Promise { - let hostname: string; - try { - hostname = new URL(String(endpointUrl)).hostname; - } catch { - return { ok: false, reason: `"${String(endpointUrl)}" is not a valid URL` }; - } - - // Keep the capability and range helpers import-light for generic curl - // validation. The YAML-backed private-network classifier is needed only - // when a caller actually runs the endpoint preflight. - const { isLoopbackHostname, isPrivateHostname, isPrivateIp } = - require("../private-networks") as typeof import("../private-networks"); - - const normalizedHostname = normalizeEndpointHost(hostname); - const trustedPrivateHost = (options.trustedPrivateHosts ?? []).some( - (candidate) => normalizeEndpointHost(candidate.trim()) === normalizedHostname, - ); - - // An explicit loopback host is a legitimate local inference server. - if (isLoopbackHostname(hostname)) return { ok: true, addresses: [] }; - - // NemoClaw's own OpenShell-managed aliases (inference.local, host.*.internal) - // resolve to the managed proxy/loopback by design and are trusted, not - // rebinding surfaces. Exempt like loopback — connect normally (no pinning) — - // and exempt BEFORE isPrivateHostname, which would otherwise reject their - // reserved .local/.internal suffixes (#6293). - if (isOpenShellManagedHost(hostname)) return { ok: true, addresses: [] }; - - // A literal private IP or reserved private name is refused without resolving. - if (isPrivateHostname(hostname) && !trustedPrivateHost) { - return { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; - } - - // A public IP literal needs neither DNS resolution nor connection pinning: - // the URL already contains the address curl will connect to. - const bare = normalizedHostname; - if (isIP(bare)) { - if (!isPrivateIp(bare)) return { ok: true, addresses: [] }; - return trustedPrivateHost && isOperatorTrustablePrivateIp(bare) - ? { - ok: true, - addresses: [], - trustedPrivateCapability: issueTrustedPrivateEndpointCapability([bare]), - trustedPrivateEndpoint: true, - } - : { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; - } - - let addresses: Array<{ address: string; family?: number }>; - try { - addresses = await lookup(bare, { all: true }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return { ok: false, reason: `cannot resolve endpoint host "${hostname}": ${message}` }; - } - if (!Array.isArray(addresses) || addresses.length === 0) { - return { ok: false, reason: `endpoint host "${hostname}" did not resolve to any address` }; - } - for (const { address } of addresses) { - // A resolved private address — including loopback reached via a public name - // (DNS rebinding) — is refused; the explicit-loopback case returned above. - if (isPrivateIp(address) && (!trustedPrivateHost || !isOperatorTrustablePrivateIp(address))) { - return { - ok: false, - reason: `endpoint host "${hostname}" resolves to private/internal address "${address}"`, - }; - } - } - const resolvedAddresses = addresses.map(({ address }) => address); - const trustedPrivateAddresses = resolvedAddresses.filter((address) => isPrivateIp(address)); - return trustedPrivateHost && trustedPrivateAddresses.length > 0 - ? { - ok: true, - addresses: resolvedAddresses, - trustedPrivateCapability: issueTrustedPrivateEndpointCapability(trustedPrivateAddresses), - trustedPrivateEndpoint: true, - } - : { ok: true, addresses: resolvedAddresses }; -} - -/** - * Build curl `--resolve ::` arguments that pin a probe's - * connection to the address(es) `assertEndpointResolvesPublic` already - * validated, while leaving the request URL (and therefore its Host header / TLS - * SNI) untouched. This closes the DNS-rebinding / TOCTOU window between the SSRF - * preflight and the privileged host-side probe curl: without pinning, curl would - * re-resolve the hostname and a second lookup could return a rebound - * private/internal address after the public preflight passed (cv review, #6293). - * - * `host` is the URL hostname (IPv6 brackets stripped, as curl `--resolve` - * expects a bare address); `port` is the explicit URL port or the scheme default - * (443 for https, 80 for http). Returns `[]` when there are no pinned addresses - * (explicit-loopback endpoints, or callers that never ran the preflight) so the - * probe connects normally, and `[]` on an unparseable URL. - */ -export function buildResolvePinArgs( - targetUrl: string, - pinnedAddresses?: readonly string[] | null, -): string[] { - if (!pinnedAddresses || pinnedAddresses.length === 0) return []; - let host: string; - let port: string; - try { - const url = new URL(String(targetUrl)); - host = - url.hostname.startsWith("[") && url.hostname.endsWith("]") - ? url.hostname.slice(1, -1) - : url.hostname; - port = url.port || (url.protocol === "https:" ? "443" : "80"); - } catch { - return []; - } - if (!host) return []; - const addresses = [...new Set(pinnedAddresses.filter(Boolean))]; - if (addresses.length === 0) return []; - // One --resolve entry preserves every accepted address. Repeating the same - // host:port entry makes curl retain only the last mapping, silently dropping - // dual-stack/failover addresses. Bracket IPv6 addresses in the comma list so - // curl can distinguish their colons from the host:port separators. - const encodedAddresses = addresses.map((address) => - address.includes(":") ? `[${address}]` : address, - ); - return ["--resolve", `${host}:${port}:${encodedAddresses.join(",")}`]; -} diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index 0307bc11bf..80f728093c 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -200,7 +200,7 @@ describe("inference selection validation", () => { }); it("probes an exactly allowlisted private endpoint with DNS pinning (#6861)", async () => { - vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "llm.corp.example"); + vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_HOSTS", "llm.corp.example"); const probeOpenAiLikeEndpoint = vi.fn(() => ({ ok: true, api: "openai-completions" })); const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); const capabilityCache = new OnboardInferenceCapabilityCache(); diff --git a/src/lib/onboard/inference-selection-validation.ts b/src/lib/onboard/inference-selection-validation.ts index 1f2a7d665c..bad8dc2d81 100644 --- a/src/lib/onboard/inference-selection-validation.ts +++ b/src/lib/onboard/inference-selection-validation.ts @@ -36,7 +36,7 @@ type OpenAiLikeProbe = ( import { assertEndpointResolvesPublic, type EndpointDnsLookupFn, - parseTrustedPrivateInferenceHosts, + parseTrustedPrivateInferenceHostsFromEnv, } from "../inference/endpoint-ssrf-preflight"; import { shouldForceCompletionsApi } from "../validation"; import { getProbeRecovery } from "../validation-recovery"; @@ -142,8 +142,7 @@ export function createInferenceSelectionValidationHelpers( const runAnthropicProbe = deps.probeAnthropicEndpoint ?? probeAnthropicEndpoint; const runOpenAiLikeProbe = deps.probeOpenAiLikeEndpoint ?? probeOpenAiLikeEndpointOptimized; const trustedPrivateEndpointHosts = - deps.trustedPrivateEndpointHosts ?? - parseTrustedPrivateInferenceHosts(process.env.NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS); + deps.trustedPrivateEndpointHosts ?? parseTrustedPrivateInferenceHostsFromEnv(process.env); function exitNonInteractiveValidationFailure(): never { process.exitCode = 1; @@ -193,7 +192,7 @@ export function createInferenceSelectionValidationHelpers( if (preflight.trustedPrivateEndpoint) { console.warn( " ⚠ Using an operator-trusted private inference endpoint; keep " + - "NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS restricted to infrastructure you control.", + "trusted-private host configuration restricted to infrastructure you control.", ); } return { diff --git a/src/lib/onboard/setup-inference-route-containment.test.ts b/src/lib/onboard/setup-inference-route-containment.test.ts index 15044ddc88..93583de2f2 100644 --- a/src/lib/onboard/setup-inference-route-containment.test.ts +++ b/src/lib/onboard/setup-inference-route-containment.test.ts @@ -38,7 +38,7 @@ describe("onboard shared gateway route containment", () => { expectedTrustedPrivateAddresses: [], }, ])("handles a resumed $scenario endpoint at the shared preflight", async (scenario) => { - vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", scenario.trustedHosts); + vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_HOSTS", scenario.trustedHosts); let lookupCount = 0; const resolveEndpointHost = vi.fn(async () => { lookupCount += 1; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 28abd05289..423b1ba055 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -6,7 +6,7 @@ import { isBedrockRuntimeEndpoint } from "../inference/bedrock-runtime"; import { assertEndpointResolvesPublic, type EndpointDnsLookupFn, - parseTrustedPrivateInferenceHosts, + parseTrustedPrivateInferenceHostsFromEnv, } from "../inference/endpoint-ssrf-preflight"; import { type CurrentGatewayRouteCompatibilityCheck, @@ -304,9 +304,7 @@ export function createSetupInference( { trustedPrivateHosts: deps.trustedPrivateEndpointHosts ?? - parseTrustedPrivateInferenceHosts( - process.env.NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS, - ), + parseTrustedPrivateInferenceHostsFromEnv(process.env), }, ); if (!preflight.ok) { diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index dfcfc7f3f2..c50f26fbfa 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -61,6 +61,13 @@ import { import { escapeTerminalText, logPresetScope, renderPresetScope } from "./preset-scope-render"; import { parseAndValidateSandboxPolicy } from "./sandbox-policy-validation"; import { splitSemanticFindings, validatePolicySemantics } from "./semantic-validation"; +import { + type ExternalPolicyPreset, + isTrustedPrivatePolicyPinCapability, + prepareTrustedPrivatePolicyPresets, + replayTrustedPrivatePolicyPinCapability, + type TrustedPrivatePolicyPinCapability, +} from "./trusted-private-endpoints"; const PRESETS_DIR = path.join(ROOT, "nemoclaw-blueprint", "policies", "presets"); @@ -1607,7 +1614,10 @@ function applyPresetContent( presetName: string, presetContent: string, options: { - custom?: { sourcePath?: string }; + custom?: { + sourcePath?: string; + trustedPrivatePinCapability?: TrustedPrivatePolicyPinCapability; + }; expectedExistingNetworkPolicyContent?: string | null; nonFatal?: boolean; skipRegistryUpdate?: boolean; @@ -1627,7 +1637,18 @@ function applyPresetContent( if (options.custom) { const np = parseNetworkPolicies(presetContent); - if (np && networkPoliciesHasAllowedIps(np)) { + const hasGeneratedPins = np !== null && networkPoliciesHasAllowedIps(np); + const trustedPrivatePinsValid = isTrustedPrivatePolicyPinCapability( + presetContent, + options.custom.trustedPrivatePinCapability, + ); + if (options.custom.trustedPrivatePinCapability && !trustedPrivatePinsValid) { + console.error( + ` Preset '${presetName}' has an invalid trusted-private pin receipt for its content.`, + ); + return false; + } + if (hasGeneratedPins && !trustedPrivatePinsValid) { console.error( ` Preset '${presetName}' contains 'allowed_ips', which is not permitted in user-supplied presets.`, ); @@ -1766,6 +1787,9 @@ function applyPresetContent( name: presetName, content: presetContent, sourcePath: options.custom.sourcePath, + ...(options.custom.trustedPrivatePinCapability + ? { trustedPrivatePins: options.custom.trustedPrivatePinCapability.receipt } + : {}), }); } else { const pols = sandbox.policies || []; @@ -2299,6 +2323,7 @@ function applyPermissivePolicy(sandboxName: string): void { console.log(" Applied permissive policy."); } +export type { ExternalPolicyPreset }; export { applyPermissivePolicy, applyPreset, @@ -2339,11 +2364,13 @@ export { PRESETS_DIR, parseCurrentPolicyOrEmpty as parseCurrentPolicy, parsePresetPolicyKeys, + prepareTrustedPrivatePolicyPresets, presetContentMatchesGateway, removeBuiltinPresetAttribution, removePreset, removePresetFromPolicy, renderPresetScope, + replayTrustedPrivatePolicyPinCapability, resolveAgentBaselinePolicy, resolvePermissivePolicyPath, resolveSandboxBaselinePolicy, diff --git a/src/lib/policy/preset-allowed-ips.test.ts b/src/lib/policy/preset-allowed-ips.test.ts index 3522152255..b131e52e45 100644 --- a/src/lib/policy/preset-allowed-ips.test.ts +++ b/src/lib/policy/preset-allowed-ips.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; @@ -227,4 +228,31 @@ network_policies: }), ).toBe(false); }); + + it("rejects a forged content-digest receipt before any side effects (#8176)", () => { + const content = `preset: + name: forged-private +network_policies: + private: + endpoints: + - host: api.corp.example + port: 443 + allowed_ips: [10.20.30.40] +`; + const forged = { + receipt: { + version: 1, + contentDigest: createHash("sha256").update(content).digest("hex"), + }, + }; + + expect( + applyPresetContent("test-sandbox", "forged-private", content, { + custom: { + sourcePath: "forged-private.yaml", + trustedPrivatePinCapability: forged as never, + }, + }), + ).toBe(false); + }); }); diff --git a/src/lib/policy/preset-scope-render.test.ts b/src/lib/policy/preset-scope-render.test.ts index 92be27e08b..ca0612bea1 100644 --- a/src/lib/policy/preset-scope-render.test.ts +++ b/src/lib/policy/preset-scope-render.test.ts @@ -37,6 +37,21 @@ network_policies: `; describe("renderPresetScope (#7179)", () => { + it("discloses generated destination pins in previews (#8176)", () => { + const content = `network_policies: + private-api: + endpoints: + - host: api.corp.example + port: 443 + protocol: rest + allowed_ips: [10.20.30.40, 2001:db8::20] +`; + + expect(renderPresetScope(content).join("\n")).toContain( + "allowed IPs: 10.20.30.40, 2001:db8::20", + ); + }); + it("returns an empty list for content with no network_policies", () => { expect(renderPresetScope("preset:\n name: x\n description: 'y'\n")).toEqual([]); expect(renderPresetScope("")).toEqual([]); diff --git a/src/lib/policy/preset-scope-render.ts b/src/lib/policy/preset-scope-render.ts index 4328af5ee6..4f1785c41f 100644 --- a/src/lib/policy/preset-scope-render.ts +++ b/src/lib/policy/preset-scope-render.ts @@ -14,6 +14,7 @@ type RuleScope = { type EndpointScope = { host: string; + allowedIps: string[]; port?: number | string; protocol?: string; access?: string; @@ -128,6 +129,7 @@ function extractPresetScope(content: string): PresetScope | null { if (!host) continue; endpoints.push({ host, + allowedIps: stringArray(rawEndpoint.allowed_ips), port: toPortOrUndefined(rawEndpoint.port), protocol: toStringOrUndefined(rawEndpoint.protocol as PolicyValue | undefined), access: toStringOrUndefined(rawEndpoint.access as PolicyValue | undefined), @@ -153,13 +155,17 @@ function formatEndpoint(endpoint: EndpointScope): string[] { } const modeSuffix = modeBits.length > 0 ? ` (${modeBits.join(", ")})` : ""; const header = ` - ${renderTerminalText(endpoint.host)}:${port}${modeSuffix}`; - if (endpoint.rules.length === 0) return [header]; + const pinLines = + endpoint.allowedIps.length > 0 + ? [` allowed IPs: ${endpoint.allowedIps.map(renderTerminalText).join(", ")}`] + : []; + if (endpoint.rules.length === 0) return [header, ...pinLines]; const ruleLines = endpoint.rules.map((rule) => { const methods = rule.methods.map(renderTerminalText).join(", "); const paths = rule.paths.map(renderTerminalText).join(", "); return ` ${rule.action}: ${methods} ${paths}`; }); - return [header, ...ruleLines]; + return [header, ...pinLines, ...ruleLines]; } export function renderPresetScope( diff --git a/src/lib/policy/trusted-private-endpoints.test.ts b/src/lib/policy/trusted-private-endpoints.test.ts new file mode 100644 index 0000000000..ad97cfc787 --- /dev/null +++ b/src/lib/policy/trusted-private-endpoints.test.ts @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import type { EndpointDnsLookupFn } from "../security/trusted-private-endpoint"; +import { + hasTrustedPrivatePolicyPinReceipt, + isTrustedPrivatePolicyPinCapability, + normalizeTrustedPrivatePolicyPinReceipt, + prepareTrustedPrivatePolicyPresets, + replayTrustedPrivatePolicyPinCapability, +} from "./trusted-private-endpoints"; + +function preset(content: string) { + return { filePath: "/tmp/private.yaml", presetName: "private", content }; +} + +function lookup(records: Record): EndpointDnsLookupFn { + return async (hostname) => (records[hostname] ?? []).map((address) => ({ address })); +} + +describe("trusted private custom policy preparation", () => { + it("pins trusted endpoints across every policy protocol (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + services: + name: services + endpoints: + - { host: api.corp.example, port: 443, protocol: rest } + - { host: api.corp.example, port: 443, protocol: websocket } + - { host: api.corp.example, port: 443, protocol: jsonrpc } + - { host: api.corp.example, port: 443, protocol: mcp } + binaries: + - { path: /usr/local/bin/node } +`); + + const [prepared] = await prepareTrustedPrivatePolicyPresets([input], ["API.CORP.EXAMPLE."], { + lookup: lookup({ "api.corp.example": ["10.20.30.40", "2001:4860:4860::8888"] }), + }); + const document = YAML.parse(prepared.content) as { + network_policies: { services: { endpoints: Array<{ allowed_ips?: string[] }> } }; + }; + expect(document.network_policies.services.endpoints).toHaveLength(4); + for (const endpoint of document.network_policies.services.endpoints) { + expect(endpoint.allowed_ips).toEqual(["10.20.30.40", "2001:4860:4860::8888"]); + } + expect(prepared.trustedPrivatePins).toMatchObject({ + version: 1, + contentDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(hasTrustedPrivatePolicyPinReceipt(prepared.content, prepared.trustedPrivatePins)).toBe( + true, + ); + expect( + isTrustedPrivatePolicyPinCapability(prepared.content, prepared.trustedPrivatePinCapability), + ).toBe(true); + expect( + isTrustedPrivatePolicyPinCapability(prepared.content, { + receipt: prepared.trustedPrivatePins, + }), + ).toBe(false); + expect( + isTrustedPrivatePolicyPinCapability( + prepared.content, + replayTrustedPrivatePolicyPinCapability(prepared.content, prepared.trustedPrivatePins), + ), + ).toBe(true); + expect( + hasTrustedPrivatePolicyPinReceipt( + `${prepared.content}\n# changed`, + prepared.trustedPrivatePins, + ), + ).toBe(false); + expect(input.content).not.toContain("allowed_ips"); + }); + + it("rejects a pin receipt that is stale, malformed, or unversioned (#8176)", () => { + const content = "network_policies: {}\n"; + for (const receipt of [ + { version: 1, contentDigest: "a".repeat(64) }, + { version: 2, contentDigest: "a".repeat(64) }, + { contentDigest: "a".repeat(64) }, + { version: 1, contentDigest: "short" }, + ]) { + expect(() => normalizeTrustedPrivatePolicyPinReceipt(content, receipt)).toThrow( + /does not match its exact content/, + ); + } + }); + + it("rejects durable replay receipts that pin reserved destinations (#8176)", () => { + const content = `network_policies: + private: + endpoints: + - host: metadata.local + allowed_ips: [169.254.169.254] +`; + const receipt = { + version: 1, + contentDigest: createHash("sha256").update(content).digest("hex"), + }; + + expect(() => replayTrustedPrivatePolicyPinCapability(content, receipt)).toThrow( + /disallowed address pin/, + ); + }); + + it("preserves the reviewed host-gateway exception beside generated private pins (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + services: + endpoints: + - host: host.openshell.internal + allowed_ips: [10.0.0.0/8] + - host: api.corp.example + allowed_ips: [] +`); + // Source files may contain bridge pins, but not a placeholder on the + // private endpoint. Remove it before preparation to model valid input. + input.content = input.content.replace(" allowed_ips: []\n", ""); + + const [prepared] = await prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"], { + lookup: lookup({ "api.corp.example": ["10.20.30.40"] }), + }); + const document = YAML.parse(prepared.content) as { + network_policies: { services: { endpoints: Array<{ allowed_ips?: string[] }> } }; + }; + expect(document.network_policies.services.endpoints[0]?.allowed_ips).toEqual(["10.0.0.0/8"]); + expect(document.network_policies.services.endpoints[1]?.allowed_ips).toEqual(["10.20.30.40"]); + expect( + isTrustedPrivatePolicyPinCapability(prepared.content, prepared.trustedPrivatePinCapability), + ).toBe(true); + }); + + it("requires every declaration to match an endpoint (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + service: + endpoints: + - { host: api.corp.example, port: 443, protocol: rest } +`); + await expect( + prepareTrustedPrivatePolicyPresets([input], ["api.corp.example", "unused.corp.example"], { + lookup: lookup({ "api.corp.example": ["10.20.30.40"] }), + }), + ).rejects.toThrow(/unused\.corp\.example.*does not match/i); + }); + + it("treats an ambient declaration for a public endpoint as a no-op (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + service: + endpoints: + - { host: api.corp.example, port: 443, protocol: rest } +`); + const [prepared] = await prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"], { + lookup: lookup({ "api.corp.example": ["93.184.216.34"] }), + requiredDeclarations: [], + }); + + expect(prepared.content).not.toContain("allowed_ips"); + expect(prepared.trustedPrivatePins).toBeUndefined(); + expect(prepared.trustedPrivatePinCapability).toBeUndefined(); + }); + + it("rejects duplicate, public-only, and disallowed private declarations (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + service: + endpoints: + - { host: api.corp.example, port: 443, protocol: rest } +`); + await expect( + prepareTrustedPrivatePolicyPresets([input], ["api.corp.example", "API.CORP.EXAMPLE."]), + ).rejects.toThrow(/declared more than once/i); + await expect( + prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"], { + lookup: lookup({ "api.corp.example": ["8.8.8.8"] }), + }), + ).rejects.toThrow(/did not resolve to an operator-trustable private address/i); + await expect( + prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"], { + lookup: lookup({ "api.corp.example": ["127.0.0.1"] }), + }), + ).rejects.toThrow(/failed destination preflight/i); + }); + + it("defensively rejects user-authored pins before generated injection (#8176)", async () => { + const input = preset(`preset: + name: private +network_policies: + service: + endpoints: + - host: api.corp.example + port: 443 + protocol: rest + allowed_ips: [10.20.30.40] +`); + await expect(prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"])).rejects.toThrow( + /user-authored allowed_ips/i, + ); + }); +}); diff --git a/src/lib/policy/trusted-private-endpoints.ts b/src/lib/policy/trusted-private-endpoints.ts new file mode 100644 index 0000000000..db34952f29 --- /dev/null +++ b/src/lib/policy/trusted-private-endpoints.ts @@ -0,0 +1,351 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { isIP } from "node:net"; + +import YAML from "yaml"; + +import { isPrivateIp, OPENSHELL_SANDBOX_HOST_BRIDGE } from "../private-networks"; +import { + assertEndpointResolvesPublic, + type EndpointDnsLookupFn, + isOperatorTrustablePrivateIp, + isTrustedPrivateEndpointCapability, + normalizeTrustedPrivateHost, +} from "../security/trusted-private-endpoint"; + +export interface ExternalPolicyPreset { + filePath: string; + presetName: string; + content: string; + trustedPrivatePins?: TrustedPrivatePolicyPinReceipt; + trustedPrivatePinCapability?: TrustedPrivatePolicyPinCapability; +} + +export interface TrustedPrivatePolicyPinReceipt { + version: 1; + contentDigest: string; +} + +declare const trustedPrivatePolicyPinCapabilityBrand: unique symbol; + +export interface TrustedPrivatePolicyPinCapability { + readonly receipt: TrustedPrivatePolicyPinReceipt; + readonly [trustedPrivatePolicyPinCapabilityBrand]: true; +} + +const TRUSTED_PRIVATE_POLICY_PIN_CAPABILITIES = new WeakSet(); + +export interface TrustedPrivatePolicyPreparationDependencies { + lookup?: EndpointDnsLookupFn; + /** Command-scoped declarations that must match this exact input batch. */ + requiredDeclarations?: readonly string[]; +} + +type EndpointReference = { + endpoint: Record; + host: string; +}; + +const SHA256_DIGEST_PATTERN = /^[a-f0-9]{64}$/; + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function policyContentDigest(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +function createTrustedPrivatePolicyPinReceipt(content: string): TrustedPrivatePolicyPinReceipt { + validateTrustedPrivatePinnedContent(content); + return { version: 1, contentDigest: policyContentDigest(content) }; +} + +function isHostGatewayBridge(host: string): boolean { + return host === OPENSHELL_SANDBOX_HOST_BRIDGE; +} + +/** + * Re-parse durable generated content before it can regain process-local pin + * authority. The digest binds bytes; this check binds semantics and prevents a + * crafted registry receipt from admitting loopback, metadata, link-local, or + * other reserved destinations. The existing host-gateway bridge exception is + * separate reviewed policy authority and is ignored here. + */ +function validateTrustedPrivatePinnedContent(content: string): void { + let document: unknown; + try { + document = YAML.parse(content) as unknown; + } catch { + throw new Error("trusted private policy pin receipt content is not valid YAML"); + } + if (!isObjectRecord(document) || !isObjectRecord(document.network_policies)) { + throw new Error("trusted private policy pin receipt content has no network policies"); + } + + let validatedPinnedEndpoint = false; + for (const policy of Object.values(document.network_policies)) { + if (!isObjectRecord(policy)) continue; + if (Object.prototype.hasOwnProperty.call(policy, "allowed_ips")) { + throw new Error("trusted private policy pins must be attached to exact endpoints"); + } + if (!Array.isArray(policy.endpoints)) continue; + for (const endpoint of policy.endpoints) { + if ( + !isObjectRecord(endpoint) || + !Object.prototype.hasOwnProperty.call(endpoint, "allowed_ips") + ) { + continue; + } + if (typeof endpoint.host !== "string") { + throw new Error("trusted private policy pin endpoint has no exact host"); + } + const host = normalizeTrustedPrivateHost(endpoint.host); + if (isHostGatewayBridge(host)) continue; + if (!Array.isArray(endpoint.allowed_ips) || endpoint.allowed_ips.length === 0) { + throw new Error(`trusted private policy endpoint '${host}' has no exact address pins`); + } + const addresses = endpoint.allowed_ips.map((address) => { + if ( + typeof address !== "string" || + isIP(address) === 0 || + address !== address.toLowerCase() + ) { + throw new Error(`trusted private policy endpoint '${host}' has a malformed address pin`); + } + if (isPrivateIp(address) && !isOperatorTrustablePrivateIp(address)) { + throw new Error(`trusted private policy endpoint '${host}' has a disallowed address pin`); + } + return address; + }); + const canonical = [...new Set(addresses)].sort(); + if ( + canonical.length !== addresses.length || + canonical.some((address, index) => address !== addresses[index]) || + !addresses.some((address) => isOperatorTrustablePrivateIp(address)) + ) { + throw new Error(`trusted private policy endpoint '${host}' has non-canonical private pins`); + } + validatedPinnedEndpoint = true; + } + } + if (!validatedPinnedEndpoint) { + throw new Error("trusted private policy pin receipt content has no generated private pins"); + } +} + +function issueTrustedPrivatePolicyPinCapability( + receipt: TrustedPrivatePolicyPinReceipt, +): TrustedPrivatePolicyPinCapability { + const capability = Object.freeze({ + receipt: Object.freeze({ ...receipt }), + }) as unknown as TrustedPrivatePolicyPinCapability; + TRUSTED_PRIVATE_POLICY_PIN_CAPABILITIES.add(capability); + return capability; +} + +/** Validate and clone the durable receipt bound to generated policy content. */ +export function normalizeTrustedPrivatePolicyPinReceipt( + content: string, + value: unknown, +): TrustedPrivatePolicyPinReceipt | undefined { + if (value === undefined) return undefined; + if ( + !isObjectRecord(value) || + value.version !== 1 || + typeof value.contentDigest !== "string" || + !SHA256_DIGEST_PATTERN.test(value.contentDigest) || + Object.keys(value).some((key) => key !== "version" && key !== "contentDigest") || + value.contentDigest !== policyContentDigest(content) + ) { + throw new Error("trusted private policy pin receipt does not match its exact content"); + } + validateTrustedPrivatePinnedContent(content); + return { version: 1, contentDigest: value.contentDigest }; +} + +/** True when a durable receipt is valid for this exact generated content. */ +export function hasTrustedPrivatePolicyPinReceipt(content: string, value: unknown): boolean { + try { + return normalizeTrustedPrivatePolicyPinReceipt(content, value) !== undefined; + } catch { + return false; + } +} + +/** True only for process-local authority issued for this exact policy content. */ +export function isTrustedPrivatePolicyPinCapability( + content: string, + value: unknown, +): value is TrustedPrivatePolicyPinCapability { + return ( + typeof value === "object" && + value !== null && + TRUSTED_PRIVATE_POLICY_PIN_CAPABILITIES.has(value) && + hasTrustedPrivatePolicyPinReceipt(content, (value as TrustedPrivatePolicyPinCapability).receipt) + ); +} + +/** + * Reissue process-local authority from a validated durable registry receipt. + * The host registry is the operator-approved authority boundary for rebuild; + * snapshot content alone is deliberately insufficient to reach this function. + */ +export function replayTrustedPrivatePolicyPinCapability( + content: string, + receipt: unknown, +): TrustedPrivatePolicyPinCapability { + const normalized = normalizeTrustedPrivatePolicyPinReceipt(content, receipt); + if (!normalized) throw new Error("trusted private policy pin receipt is missing"); + return issueTrustedPrivatePolicyPinCapability(normalized); +} + +function endpointUrl(host: string): string { + return `https://${isIP(host) === 6 ? `[${host}]` : host}/`; +} + +function collectEndpointReferences(document: unknown): EndpointReference[] { + if (!isObjectRecord(document) || !isObjectRecord(document.network_policies)) return []; + const references: EndpointReference[] = []; + for (const policy of Object.values(document.network_policies)) { + if (!isObjectRecord(policy) || !Array.isArray(policy.endpoints)) continue; + for (const endpoint of policy.endpoints) { + if (!isObjectRecord(endpoint) || typeof endpoint.host !== "string") continue; + let host: string; + try { + host = normalizeTrustedPrivateHost(endpoint.host); + } catch { + continue; + } + references.push({ endpoint, host }); + } + } + return references; +} + +function normalizeDeclarations(values: readonly string[], rejectDuplicates: boolean): string[] { + const normalized: string[] = []; + const seen = new Set(); + for (const value of values) { + const host = normalizeTrustedPrivateHost(value); + if (seen.has(host) && rejectDuplicates) { + throw new Error( + `Trusted private host '${host}' was declared more than once after normalization.`, + ); + } + if (!seen.has(host)) { + seen.add(host); + normalized.push(host); + } + } + return normalized; +} + +/** + * Inject exact, validated private-address pins into already-validated custom + * policy presets. User-authored content remains untrusted: only a + * provenance-checked capability issued by the shared preflight can produce + * `allowed_ips` entries. + */ +export async function prepareTrustedPrivatePolicyPresets( + presets: readonly ExternalPolicyPreset[], + declarations: readonly string[], + dependencies: TrustedPrivatePolicyPreparationDependencies = {}, +): Promise { + if (declarations.length === 0) return presets.map((preset) => ({ ...preset })); + + const trustedHosts = normalizeDeclarations(declarations, false); + const requiredHosts = normalizeDeclarations( + dependencies.requiredDeclarations ?? declarations, + true, + ); + const requiredHostSet = new Set(requiredHosts); + const parsedPresets = presets.map((preset) => { + const document = YAML.parse(preset.content) as unknown; + if (!isObjectRecord(document)) { + throw new Error(`Preset '${preset.presetName}' is not a YAML mapping.`); + } + const references = collectEndpointReferences(document); + for (const { endpoint, host } of references) { + if ( + Object.prototype.hasOwnProperty.call(endpoint, "allowed_ips") && + !isHostGatewayBridge(host) + ) { + throw new Error( + `Preset '${preset.presetName}' contains user-authored allowed_ips, which is not permitted.`, + ); + } + } + return { preset, document, references, injectedEndpoints: new Set>() }; + }); + + const referencesByHost = new Map(); + for (const host of trustedHosts) referencesByHost.set(host, []); + for (const { references } of parsedPresets) { + for (const reference of references) { + referencesByHost.get(reference.host)?.push(reference); + } + } + + for (const host of requiredHosts) { + if ((referencesByHost.get(host)?.length ?? 0) === 0) { + throw new Error( + `Trusted private host '${host}' does not match an endpoint in the custom preset input.`, + ); + } + } + for (const [host, references] of referencesByHost) { + if (references.length === 0) continue; + const result = await assertEndpointResolvesPublic(endpointUrl(host), dependencies.lookup, { + trustedPrivateHosts: [host], + }); + if (!result.ok) { + throw new Error( + `Trusted private host '${host}' failed destination preflight: ${result.reason ?? "validation failed"}.`, + ); + } + if (!isTrustedPrivateEndpointCapability(result.trustedPrivateCapability)) { + if (requiredHostSet.has(host)) { + throw new Error( + `Trusted private host '${host}' did not resolve to an operator-trustable private address.`, + ); + } + continue; + } + const pins = [ + ...new Set( + (result.addresses?.length + ? result.addresses + : result.trustedPrivateCapability.addresses + ).map((address) => address.toLowerCase()), + ), + ].sort(); + if (pins.length === 0) { + throw new Error(`Trusted private host '${host}' produced no validated address pins.`); + } + for (const { endpoint } of references) { + endpoint.allowed_ips = [...pins]; + for (const parsedPreset of parsedPresets) { + if (parsedPreset.references.some((reference) => reference.endpoint === endpoint)) { + parsedPreset.injectedEndpoints.add(endpoint); + break; + } + } + } + } + + return parsedPresets.map(({ preset, document, injectedEndpoints }) => { + const content = YAML.stringify(document); + const injectedPins = injectedEndpoints.size > 0; + if (!injectedPins) return { ...preset, content }; + const trustedPrivatePins = createTrustedPrivatePolicyPinReceipt(content); + return { + ...preset, + content, + trustedPrivatePins, + trustedPrivatePinCapability: issueTrustedPrivatePolicyPinCapability(trustedPrivatePins), + }; + }); +} diff --git a/src/lib/security/trusted-private-endpoint.test.ts b/src/lib/security/trusted-private-endpoint.test.ts new file mode 100644 index 0000000000..92c9bfce12 --- /dev/null +++ b/src/lib/security/trusted-private-endpoint.test.ts @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + assertEndpointResolvesPublic, + type EndpointDnsLookupFn, + isOperatorTrustablePrivateIp, + isTrustedPrivateEndpointCapability, + normalizeTrustedPrivateHost, + parseTrustedPrivateHosts, + replayTrustedPrivateEndpoint, +} from "./trusted-private-endpoint"; + +const resolverTo = (address: string): EndpointDnsLookupFn => + vi.fn(async () => [{ address, family: address.includes(":") ? 6 : 4 }]); + +describe("trusted private endpoint hosts", () => { + it.each([ + [" MCP.CORP.EXAMPLE. ", "mcp.corp.example"], + ["10.20.30.40", "10.20.30.40"], + ["[fd00::10]", "fd00::10"], + ["FD00::10", "fd00::10"], + ["fd00:0:0:0:0:0:0:10", "fd00::10"], + ])("normalizes the exact host %s (#8176)", (raw, expected) => { + expect(normalizeTrustedPrivateHost(raw)).toBe(expected); + }); + + it.each([ + "", + "https://mcp.corp.example", + "mcp.corp.example:443", + "mcp.corp.example/path", + "mcp.corp.example?query", + "mcp.corp.example#fragment", + "*.corp.example", + ".corp.example", + "10.0.0.0/8", + "user@mcp.corp.example", + "mcp..corp.example", + "-mcp.corp.example", + "mcp-.corp.example", + "999.1.1.1", + "[mcp.corp.example]", + ])("rejects the non-exact host input %s (#8176)", (raw) => { + expect(() => normalizeTrustedPrivateHost(raw)).toThrow(/trusted private host/); + }); + + it("parses and deduplicates exact hosts from the generic source (#8176)", () => { + expect(parseTrustedPrivateHosts(" MCP.CORP.EXAMPLE.,10.0.0.8,mcp.corp.example ")).toEqual([ + "mcp.corp.example", + "10.0.0.8", + ]); + expect(parseTrustedPrivateHosts(undefined)).toEqual([]); + }); + + it("rejects an empty entry in a configured host list (#8176)", () => { + expect(() => parseTrustedPrivateHosts("mcp.corp.example,,10.0.0.8")).toThrow( + /must not be empty/, + ); + }); +}); + +describe("trusted private endpoint preflight", () => { + it.each([ + "10.0.0.1", + "100.64.0.1", + "172.16.0.1", + "192.168.0.1", + "fd00::1", + ])("classifies the operator-trustable address %s (#8176)", (address) => { + expect(isOperatorTrustablePrivateIp(address)).toBe(true); + }); + + it.each([ + "127.0.0.1", + "169.254.169.254", + "198.18.0.1", + "fe80::1", + "ff00::1", + ])("keeps the reserved address %s outside operator trust (#8176)", (address) => { + expect(isOperatorTrustablePrivateIp(address)).toBe(false); + }); + + it("issues a provenance-checked capability for an exact trusted host (#8176)", async () => { + const result = await assertEndpointResolvesPublic( + "https://mcp.corp.example/mcp", + resolverTo("10.0.0.8"), + { trustedPrivateHosts: ["mcp.corp.example"] }, + ); + + expect(result).toMatchObject({ + ok: true, + addresses: ["10.0.0.8"], + trustedPrivateEndpoint: true, + }); + expect(result.trustedPrivateCapability?.addresses).toEqual(["10.0.0.8"]); + expect(isTrustedPrivateEndpointCapability(result.trustedPrivateCapability)).toBe(true); + expect(isTrustedPrivateEndpointCapability({ addresses: ["10.0.0.8"] })).toBe(false); + }); + + it("rejects a private result for a different exact host (#8176)", async () => { + const result = await assertEndpointResolvesPublic( + "https://attacker.mcp.corp.example/mcp", + resolverTo("10.0.0.8"), + { trustedPrivateHosts: ["mcp.corp.example"] }, + ); + + expect(result.ok).toBe(false); + expect(result.trustedPrivateCapability).toBeUndefined(); + }); + + it("rejects link-local metadata for an exact trusted host (#8176)", async () => { + const result = await assertEndpointResolvesPublic( + "https://mcp.corp.example/mcp", + resolverTo("169.254.169.254"), + { trustedPrivateHosts: ["mcp.corp.example"] }, + ); + + expect(result.ok).toBe(false); + expect(result.reason).toContain("169.254.169.254"); + }); + + it("preserves public endpoint preflight behavior (#8176)", async () => { + const result = await assertEndpointResolvesPublic( + "https://mcp.example/mcp", + resolverTo("93.184.216.34"), + ); + + expect(result).toEqual({ ok: true, addresses: ["93.184.216.34"] }); + }); +}); + +describe("trusted private endpoint replay", () => { + it("reissues capability authority from exact durable pins without DNS (#8267)", () => { + const replay = replayTrustedPrivateEndpoint("MCP.CORP.EXAMPLE.", [ + "fd00:0:0:0:0:0:0:10", + "10.0.0.8", + ]); + + expect(replay.host).toBe("mcp.corp.example"); + expect(replay.addresses).toEqual(["10.0.0.8", "fd00::10"]); + expect(replay.trustedPrivateCapability.addresses).toEqual(replay.addresses); + expect(isTrustedPrivateEndpointCapability(replay.trustedPrivateCapability)).toBe(true); + }); + + it.each([ + ["no pins", []], + ["public pin", ["93.184.216.34"]], + ["loopback pin", ["127.0.0.1"]], + ["link-local pin", ["169.254.169.254"]], + ["duplicate pin", ["10.0.0.8", "10.0.0.8"]], + ])("rejects replay with %s (#8267)", (_label, addresses) => { + expect(() => replayTrustedPrivateEndpoint("mcp.corp.example", addresses)).toThrow( + /recorded address pin/, + ); + }); +}); diff --git a/src/lib/security/trusted-private-endpoint.ts b/src/lib/security/trusted-private-endpoint.ts new file mode 100644 index 0000000000..7033cd2043 --- /dev/null +++ b/src/lib/security/trusted-private-endpoint.ts @@ -0,0 +1,370 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { lookup as dnsLookup } from "node:dns/promises"; +import { BlockList, isIP } from "node:net"; + +/** Injectable DNS resolver, shaped like `dns/promises` `lookup(host, {all:true})`. */ +export type EndpointDnsLookupFn = ( + hostname: string, + options: { all: true }, +) => Promise>; + +/** + * NemoClaw's OpenShell-managed infrastructure hostnames. These resolve to the + * host loopback or the OpenShell L7 proxy by design (see + * `subprocess-env` `withLocalNoProxy` and `verify-deployment` for + * `inference.local`), so — unlike an arbitrary user-supplied public name — they + * are trusted aliases, not attacker-controlled names subject to DNS rebinding. + * They are exempt from the public-resolution requirement, like explicit + * loopback, and connect normally without `--resolve` pinning. This mirrors the + * MCP URL-target allowlist (`isOpenShellMcpHostAlias`) and also covers + * `inference.local` — the managed sandbox inference route a compatible endpoint + * legitimately targets (#6293). + */ +const OPENSHELL_MANAGED_HOSTS = new Set([ + "inference.local", + "host.openshell.internal", + "host.docker.internal", + "host.containers.internal", +]); + +// An explicit operator allowlist may admit routable enterprise/private address +// space, but never link-local metadata, multicast, documentation, translation, +// or other reserved ranges covered by the broader SSRF denylist. +const OPERATOR_TRUSTABLE_PRIVATE_NETWORKS = new BlockList(); +OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("10.0.0.0", 8, "ipv4"); +OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("100.64.0.0", 10, "ipv4"); +OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("172.16.0.0", 12, "ipv4"); +OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("192.168.0.0", 16, "ipv4"); +OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.addSubnet("fc00::", 7, "ipv6"); + +declare const trustedPrivateEndpointCapabilityBrand: unique symbol; + +/** + * Ephemeral proof that the shared SSRF preflight admitted an exact set of + * operator-trusted private addresses. Callers can carry this value, but only + * this module can issue one and the curl boundary validates its provenance. + */ +export interface TrustedPrivateEndpointCapability { + readonly addresses: readonly string[]; + readonly [trustedPrivateEndpointCapabilityBrand]: true; +} + +const TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES = new WeakSet(); + +function issueTrustedPrivateEndpointCapability( + addresses: readonly string[], +): TrustedPrivateEndpointCapability { + const capability = Object.freeze({ + addresses: Object.freeze([...new Set(addresses)]), + }) as unknown as TrustedPrivateEndpointCapability; + TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES.add(capability); + return capability; +} + +/** True only for a capability issued by this module in the current process. */ +export function isTrustedPrivateEndpointCapability( + value: unknown, +): value is TrustedPrivateEndpointCapability { + return ( + typeof value === "object" && value !== null && TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES.has(value) + ); +} +export function isOperatorTrustablePrivateIp(address: string): boolean { + const family = isIP(address); + return ( + family !== 0 && + OPERATOR_TRUSTABLE_PRIVATE_NETWORKS.check(address, family === 6 ? "ipv6" : "ipv4") + ); +} + +/** True when `hostname` is a NemoClaw OpenShell-managed infrastructure alias. */ +export function isOpenShellManagedHost(hostname: string): boolean { + const normalised = ( + hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname + ) + .replace(/\.$/, "") + .toLowerCase(); + return OPENSHELL_MANAGED_HOSTS.has(normalised); +} + +export interface EndpointSsrfPreflightResult { + ok: boolean; + /** Human-readable reason, present only when `ok === false`. */ + reason?: string; + /** + * Validated public addresses the endpoint host resolved to, for connection + * pinning (curl `--resolve`) so a subsequent probe cannot re-resolve the name + * to a rebound private/internal address (TOCTOU). Present only when + * `ok === true` and pinning applies — resolved public names and public IP + * literals. An empty array is the explicit trusted-no-pin capability for + * loopback, OpenShell-managed aliases, and public IP literals. Callers must + * preserve it so credentialed probes bypass ambient proxies even when no + * curl `--resolve` argument is needed. + */ + addresses?: string[]; + /** Non-forgeable proof of the exact private addresses admitted by the operator allowlist. */ + trustedPrivateCapability?: TrustedPrivateEndpointCapability; + /** True only when an exact operator allowlist entry admitted a private address. */ + trustedPrivateEndpoint?: true; +} + +export interface EndpointSsrfPreflightOptions { + /** Exact hostnames or IP literals the operator explicitly trusts on a private network. */ + trustedPrivateHosts?: readonly string[]; +} + +function normalizeIpLiteral(address: string): string { + if (isIP(address) !== 6) return address.toLowerCase(); + const hostname = new URL(`http://[${address}]/`).hostname; + return hostname.slice(1, -1).toLowerCase(); +} + +/** Normalize one exact hostname or IP literal for an operator trust decision. */ +export function normalizeTrustedPrivateHost(raw: string): string { + const value = String(raw).trim(); + if (!value) throw new Error("trusted private host must not be empty"); + + if (value.startsWith("[") || value.endsWith("]")) { + if (!(value.startsWith("[") && value.endsWith("]"))) { + throw new Error(`trusted private host "${value}" is malformed`); + } + const address = value.slice(1, -1).toLowerCase(); + if (isIP(address) !== 6) { + throw new Error(`trusted private host "${value}" is not an IPv6 literal`); + } + return normalizeIpLiteral(address); + } + + if ( + value.includes("://") || + /[\\/?#@*]/.test(value) || + value.startsWith(".") || + value.includes("%") + ) { + throw new Error(`trusted private host "${value}" must be an exact hostname or IP literal`); + } + + const normalized = value.replace(/\.$/, "").toLowerCase(); + if (!normalized || normalized.endsWith(".")) { + throw new Error(`trusted private host "${value}" is malformed`); + } + if (isIP(normalized) !== 0) return normalizeIpLiteral(normalized); + if (normalized.includes(":")) { + throw new Error(`trusted private host "${value}" must not include a port`); + } + if (/^\d+(?:\.\d+){3}$/.test(normalized)) { + throw new Error(`trusted private host "${value}" is not a valid IP literal`); + } + if (normalized.length > 253) { + throw new Error(`trusted private host "${value}" is too long`); + } + + const labels = normalized.split("."); + if ( + labels.some( + (label) => + label.length === 0 || label.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label), + ) + ) { + throw new Error(`trusted private host "${value}" is malformed`); + } + return normalized; +} + +/** Parse a comma-separated allowlist of exact private endpoint hosts. */ +export function parseTrustedPrivateHosts(value: string | undefined): string[] { + const input = String(value ?? "").trim(); + if (!input) return []; + return [...new Set(input.split(",").map((entry) => normalizeTrustedPrivateHost(entry)))]; +} + +export interface TrustedPrivateEndpointReplay { + readonly host: string; + readonly addresses: readonly string[]; + readonly trustedPrivateCapability: TrustedPrivateEndpointCapability; +} + +/** Reissue in-process capability authority from exact durable private pins. */ +export function replayTrustedPrivateEndpoint( + host: string, + addresses: readonly string[], +): TrustedPrivateEndpointReplay { + const normalizedHost = normalizeTrustedPrivateHost(host); + if (addresses.length === 0) { + throw new Error(`trusted private host "${normalizedHost}" has no recorded address pins`); + } + const normalizedAddresses = addresses.map((address) => { + if (typeof address !== "string" || !isOperatorTrustablePrivateIp(address)) { + throw new Error( + `trusted private host "${normalizedHost}" has a disallowed recorded address pin`, + ); + } + return normalizeIpLiteral(address); + }); + if (new Set(normalizedAddresses).size !== normalizedAddresses.length) { + throw new Error(`trusted private host "${normalizedHost}" has duplicate recorded address pins`); + } + const pinnedAddresses = Object.freeze([...normalizedAddresses].sort()); + return Object.freeze({ + host: normalizedHost, + addresses: pinnedAddresses, + trustedPrivateCapability: issueTrustedPrivateEndpointCapability(pinnedAddresses), + }); +} + +/** + * DNS-backed SSRF preflight for a user-supplied endpoint. Callers run this + * before a privileged host-side request. + * + * The string-level `isPrivateHostname` guards elsewhere block literal private + * IP addresses and reserved names. A public-looking endpoint name can still + * resolve to `127.0.0.1`, `169.254.169.254`, or RFC1918 space. This resolves the + * hostname first and rejects a private or reserved address before a host + * request. It complements the + * authoritative config-write DNS-pinning boundary (`validateUrlValueWithDnsResult`) + * which runs later, before the URL is persisted. + * + * Loopback (`127.0.0.0/8`, `::1`, and `localhost`) remains exempt only when the + * endpoint hostname is itself loopback. This preserves local inference + * behavior. A public name that resolves to loopback is treated as a rebinding + * attempt and rejected. The check fails closed on a resolver error or empty + * result. + * + * See PR #6293 PRA-4 (GPT-5.5 advisor). + */ +export async function assertEndpointResolvesPublic( + endpointUrl: string, + lookup: EndpointDnsLookupFn = dnsLookup as unknown as EndpointDnsLookupFn, + options: EndpointSsrfPreflightOptions = {}, +): Promise { + let hostname: string; + try { + hostname = new URL(String(endpointUrl)).hostname; + } catch { + return { ok: false, reason: `"${String(endpointUrl)}" is not a valid URL` }; + } + + // Keep the capability and range helpers import-light for generic curl + // validation. The YAML-backed private-network classifier is needed only + // when a caller actually runs the endpoint preflight. + const { isLoopbackHostname, isPrivateHostname, isPrivateIp } = + require("../private-networks") as typeof import("../private-networks"); + + const normalizedHostname = normalizeTrustedPrivateHost(hostname); + let trustedPrivateHosts: string[]; + try { + trustedPrivateHosts = (options.trustedPrivateHosts ?? []).map(normalizeTrustedPrivateHost); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, reason: message }; + } + const trustedPrivateHost = trustedPrivateHosts.includes(normalizedHostname); + + // An explicit loopback host is a legitimate local inference server. + if (isLoopbackHostname(hostname)) return { ok: true, addresses: [] }; + + // NemoClaw's own OpenShell-managed aliases (inference.local, host.*.internal) + // resolve to the managed proxy/loopback by design and are trusted, not + // rebinding surfaces. Exempt like loopback — connect normally (no pinning) — + // and exempt BEFORE isPrivateHostname, which would otherwise reject their + // reserved .local/.internal suffixes (#6293). + if (isOpenShellManagedHost(hostname)) return { ok: true, addresses: [] }; + + // A literal private IP or reserved private name is refused without resolving. + if (isPrivateHostname(hostname) && !trustedPrivateHost) { + return { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; + } + + // A public IP literal needs neither DNS resolution nor connection pinning: + // the URL already contains the address curl will connect to. + const bare = normalizedHostname; + if (isIP(bare)) { + if (!isPrivateIp(bare)) return { ok: true, addresses: [] }; + return trustedPrivateHost && isOperatorTrustablePrivateIp(bare) + ? { + ok: true, + addresses: [], + trustedPrivateCapability: issueTrustedPrivateEndpointCapability([bare]), + trustedPrivateEndpoint: true, + } + : { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; + } + + let addresses: Array<{ address: string; family?: number }>; + try { + addresses = await lookup(bare, { all: true }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { ok: false, reason: `cannot resolve endpoint host "${hostname}": ${message}` }; + } + if (!Array.isArray(addresses) || addresses.length === 0) { + return { ok: false, reason: `endpoint host "${hostname}" did not resolve to any address` }; + } + for (const { address } of addresses) { + // A resolved private address — including loopback reached via a public name + // (DNS rebinding) — is refused; the explicit-loopback case returned above. + if (isPrivateIp(address) && (!trustedPrivateHost || !isOperatorTrustablePrivateIp(address))) { + return { + ok: false, + reason: `endpoint host "${hostname}" resolves to private/internal address "${address}"`, + }; + } + } + const resolvedAddresses = addresses.map(({ address }) => address); + const trustedPrivateAddresses = resolvedAddresses.filter((address) => isPrivateIp(address)); + return trustedPrivateHost && trustedPrivateAddresses.length > 0 + ? { + ok: true, + addresses: resolvedAddresses, + trustedPrivateCapability: issueTrustedPrivateEndpointCapability(trustedPrivateAddresses), + trustedPrivateEndpoint: true, + } + : { ok: true, addresses: resolvedAddresses }; +} + +/** + * Build curl `--resolve ::` arguments that pin a probe's + * connection to the address(es) `assertEndpointResolvesPublic` already + * validated, while leaving the request URL (and therefore its Host header / TLS + * SNI) untouched. This closes the DNS-rebinding / TOCTOU window between the SSRF + * preflight and the privileged host-side probe curl: without pinning, curl would + * re-resolve the hostname and a second lookup could return a rebound + * private/internal address after the public preflight passed (cv review, #6293). + * + * `host` is the URL hostname (IPv6 brackets stripped, as curl `--resolve` + * expects a bare address); `port` is the explicit URL port or the scheme default + * (443 for https, 80 for http). Returns `[]` when there are no pinned addresses + * (explicit-loopback endpoints, or callers that never ran the preflight) so the + * probe connects normally, and `[]` on an unparseable URL. + */ +export function buildResolvePinArgs( + targetUrl: string, + pinnedAddresses?: readonly string[] | null, +): string[] { + if (!pinnedAddresses || pinnedAddresses.length === 0) return []; + let host: string; + let port: string; + try { + const url = new URL(String(targetUrl)); + host = + url.hostname.startsWith("[") && url.hostname.endsWith("]") + ? url.hostname.slice(1, -1) + : url.hostname; + port = url.port || (url.protocol === "https:" ? "443" : "80"); + } catch { + return []; + } + if (!host) return []; + const addresses = [...new Set(pinnedAddresses.filter(Boolean))]; + if (addresses.length === 0) return []; + // One --resolve entry preserves every accepted address. Repeating the same + // host:port entry makes curl retain only the last mapping, silently dropping + // dual-stack/failover addresses. Bracket IPv6 addresses in the comma list so + // curl can distinguish their colons from the host:port separators. + const encodedAddresses = addresses.map((address) => + address.includes(":") ? `[${address}]` : address, + ); + return ["--resolve", `${host}:${port}:${encodedAddresses.join(",")}`]; +} diff --git a/src/lib/state/registry-mcp.ts b/src/lib/state/registry-mcp.ts index e3c07b3b0d..335eb69590 100644 --- a/src/lib/state/registry-mcp.ts +++ b/src/lib/state/registry-mcp.ts @@ -3,6 +3,10 @@ import { isObjectRecord } from "../core/json-types"; import { isBlockedMcpUrlTargetHost, MCP_SERVER_URL_MAX_LENGTH } from "../security/mcp-url-target"; +import { + isOperatorTrustablePrivateIp, + normalizeTrustedPrivateHost, +} from "../security/trusted-private-endpoint"; export interface McpBridgeEntry { server: string; @@ -10,6 +14,15 @@ export interface McpBridgeEntry { adapter?: string; url: string; env: string[]; + /** Exact URL host explicitly admitted for routed private access. */ + trustedPrivateHost?: string; + /** + * Immutable validated private address pins recorded when the bridge was + * added. After strict registry normalization, this durable host state is the + * operator-approved replay authority; lifecycle commands never widen it + * from ambient DNS. + */ + allowedIps?: string[]; providerName?: string; /** Immutable OpenShell ObjectMeta.id captured after provider creation. */ providerId?: string; @@ -101,7 +114,7 @@ export function normalizeSandboxMcpState(value: unknown): SandboxMcpState | unde }; } -function normalizeMcpUrl(value: string): string | null { +function normalizeMcpUrl(value: string, trustedPrivateHost?: string): string | null { if (value.length > MCP_SERVER_URL_MAX_LENGTH) return null; let parsed: URL; try { @@ -111,7 +124,12 @@ function normalizeMcpUrl(value: string): string | null { } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; if (!parsed.hostname || parsed.username || parsed.password) return null; - if (isBlockedMcpUrlTargetHost(parsed.hostname)) return null; + if ( + isBlockedMcpUrlTargetHost(parsed.hostname) && + parsed.hostname.toLowerCase() !== trustedPrivateHost + ) { + return null; + } if (parsed.hash) parsed.hash = ""; if (!parsed.pathname) parsed.pathname = "/"; const normalized = parsed.toString(); @@ -122,9 +140,46 @@ function normalizeMcpBridgeEntry(server: string, value: unknown): McpBridgeEntry if (!isObjectRecord(value)) return null; const serverName = typeof value.server === "string" && value.server ? value.server : server; if (!MCP_SERVER_RE.test(serverName)) return null; - const url = typeof value.url === "string" ? normalizeMcpUrl(value.url) : null; + let trustedPrivateHost: string | undefined; + if (value.trustedPrivateHost !== undefined) { + if (typeof value.trustedPrivateHost !== "string") return null; + try { + trustedPrivateHost = normalizeTrustedPrivateHost(value.trustedPrivateHost); + } catch { + return null; + } + if (trustedPrivateHost !== value.trustedPrivateHost) return null; + } + const url = typeof value.url === "string" ? normalizeMcpUrl(value.url, trustedPrivateHost) : null; const policyName = typeof value.policyName === "string" ? value.policyName : ""; if (!url || !MCP_SAFE_NAME_RE.test(policyName)) return null; + if (trustedPrivateHost && new URL(url).hostname.toLowerCase() !== trustedPrivateHost) return null; + let allowedIps: string[] | undefined; + const rawAllowedIps = value.allowedIps; + if (trustedPrivateHost) { + if ( + !Array.isArray(rawAllowedIps) || + rawAllowedIps.length === 0 || + !rawAllowedIps.every( + (address): address is string => + typeof address === "string" && + address === address.toLowerCase() && + isOperatorTrustablePrivateIp(address), + ) + ) { + return null; + } + const validatedAllowedIps = rawAllowedIps as string[]; + allowedIps = [...new Set(validatedAllowedIps)].sort(); + if ( + allowedIps.length !== validatedAllowedIps.length || + allowedIps.some((address, index) => address !== validatedAllowedIps[index]) + ) { + return null; + } + } else if (rawAllowedIps !== undefined) { + return null; + } const rawEnv = value.env; const env = Array.isArray(rawEnv) && @@ -156,6 +211,7 @@ function normalizeMcpBridgeEntry(server: string, value: unknown): McpBridgeEntry ...(adapter ? { adapter } : {}), url, env, + ...(trustedPrivateHost ? { trustedPrivateHost, allowedIps } : {}), ...(providerName ? { providerName } : {}), ...(providerId ? { providerId } : {}), policyName, diff --git a/src/lib/state/registry-normalization.test.ts b/src/lib/state/registry-normalization.test.ts index 2ef36ae0f8..50f1554fc9 100644 --- a/src/lib/state/registry-normalization.test.ts +++ b/src/lib/state/registry-normalization.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +11,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { normalizeBaselineExclusions, normalizeBaselineExclusionTransition, + normalizeCustomPolicyEntries, } from "./registry-normalization"; const originalHome = process.env.HOME; @@ -140,6 +142,60 @@ describe("sandbox registry normalization", () => { }); }); +describe("custom policy pin receipt normalization (#8176)", () => { + const content = `network_policies: + private-api: + endpoints: + - host: api.corp.example + allowed_ips: [10.20.30.40] +`; + const receipt = { + version: 1 as const, + contentDigest: createHash("sha256").update(content).digest("hex"), + }; + + it("keeps exact generated-pin authority bound to custom policy content", () => { + expect( + normalizeCustomPolicyEntries([ + { + name: "private-api", + content, + sourcePath: "/tmp/private-api.yaml", + trustedPrivatePins: receipt, + }, + ]), + ).toEqual([ + { + name: "private-api", + content, + sourcePath: "/tmp/private-api.yaml", + trustedPrivatePins: receipt, + }, + ]); + }); + + it("fails closed when persisted pin authority does not match exact content", () => { + expect(() => + normalizeCustomPolicyEntries([ + { + name: "private-api", + content: `${content}\n# changed`, + trustedPrivatePins: receipt, + }, + ]), + ).toThrow(/invalid trusted-private pin authority.*before rebuilding/i); + expect(() => + normalizeCustomPolicyEntries([ + { + name: "private-api", + content, + trustedPrivatePins: { contentDigest: receipt.contentDigest }, + }, + ]), + ).toThrow(/invalid trusted-private pin authority.*before rebuilding/i); + }); +}); + describe("baseline exclusion normalization (#7178)", () => { const digest = "a".repeat(64); const entry = { diff --git a/src/lib/state/registry-normalization.ts b/src/lib/state/registry-normalization.ts index 69e3e17666..7381b9c838 100644 --- a/src/lib/state/registry-normalization.ts +++ b/src/lib/state/registry-normalization.ts @@ -2,13 +2,65 @@ // SPDX-License-Identifier: Apache-2.0 import { isObjectRecord } from "../core/json-types"; -import type { BaselineExclusionEntry, BaselineExclusionTransition, SandboxEntry } from "./registry"; +import { normalizeTrustedPrivatePolicyPinReceipt } from "../policy/trusted-private-endpoints"; +import type { + BaselineExclusionEntry, + BaselineExclusionTransition, + CustomPolicyEntry, + SandboxEntry, +} from "./registry"; const BASELINE_TRANSITION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const BASELINE_TRANSITION_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; const SHA256_DIGEST_PATTERN = /^[a-f0-9]{64}$/; +/** Normalize persisted custom policy content and its generated-pin authority. */ +export function normalizeCustomPolicyEntries(value: unknown): CustomPolicyEntry[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new Error( + "Sandbox registry customPolicies must be an array; repair the registry before rebuilding", + ); + } + const entries: CustomPolicyEntry[] = []; + for (const item of value) { + if ( + !isObjectRecord(item) || + typeof item.name !== "string" || + item.name.trim().length === 0 || + typeof item.content !== "string" || + (item.pendingContent !== undefined && typeof item.pendingContent !== "string") || + (item.sourcePath !== undefined && typeof item.sourcePath !== "string") || + (item.appliedAt !== undefined && typeof item.appliedAt !== "string") + ) { + throw new Error( + "Sandbox registry contains a malformed custom policy; repair the registry before rebuilding", + ); + } + let trustedPrivatePins; + try { + trustedPrivatePins = normalizeTrustedPrivatePolicyPinReceipt( + item.content, + item.trustedPrivatePins, + ); + } catch { + throw new Error( + `Sandbox registry custom policy '${item.name}' has invalid trusted-private pin authority; repair the registry before rebuilding`, + ); + } + entries.push({ + name: item.name, + content: item.content, + ...(item.pendingContent !== undefined ? { pendingContent: item.pendingContent } : {}), + ...(item.sourcePath !== undefined ? { sourcePath: item.sourcePath } : {}), + ...(item.appliedAt !== undefined ? { appliedAt: item.appliedAt } : {}), + ...(trustedPrivatePins ? { trustedPrivatePins } : {}), + }); + } + return entries.length > 0 ? entries : undefined; +} + function isCanonicalIsoTimestamp(value: string): boolean { const parsed = new Date(value); return !Number.isNaN(parsed.getTime()) && parsed.toISOString() === value; diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index a6b360998e..e13f9cd4c5 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -21,6 +21,7 @@ import { normalizeSandboxMcpState } from "./registry-mcp"; import { normalizeBaselineExclusions, normalizeBaselineExclusionTransition, + normalizeCustomPolicyEntries, retainedDefaultSandbox, } from "./registry-normalization"; import * as reversibleRemoval from "./registry-reversible-removal"; @@ -74,7 +75,6 @@ export type { SandboxWorkloadReceipt, } from "./registry/types"; export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; - export { getConfiguredMessagingChannelsFromEntry, getDisabledMessagingChannelsFromEntry, @@ -82,6 +82,7 @@ export { getMessagingPlanFromEntry, type SandboxMessagingState, } from "./registry-messaging"; +export { normalizeCustomPolicyEntries }; export type SandboxRemovalReceipt = reversibleRemoval.RegistryRemovalReceipt; diff --git a/src/lib/state/registry/persistence.ts b/src/lib/state/registry/persistence.ts index c0974507ea..17685af98b 100644 --- a/src/lib/state/registry/persistence.ts +++ b/src/lib/state/registry/persistence.ts @@ -14,6 +14,7 @@ import { import { normalizeBaselineExclusions, normalizeBaselineExclusionTransition, + normalizeCustomPolicyEntries, parseSandboxRegistryEntries, retainedDefaultSandbox, } from "../registry-normalization"; @@ -101,12 +102,14 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const baselineExclusionTransition = normalizeBaselineExclusionTransition( entry.baselineExclusionTransition, ); + const customPolicies = normalizeCustomPolicyEntries(entry.customPolicies); const { messaging: _messaging, workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, + customPolicies: _customPolicies, ...rest } = entry; return { @@ -116,6 +119,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), + ...(customPolicies ? { customPolicies } : {}), }; } @@ -146,12 +150,14 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { const baselineExclusionTransition = normalizeBaselineExclusionTransition( durable.baselineExclusionTransition, ); + const customPolicies = normalizeCustomPolicyEntries(durable.customPolicies); const { messaging: _messaging, workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, + customPolicies: _customPolicies, ...rest } = durable; return { @@ -162,5 +168,6 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), ...(baselineExclusionTransition ? { baselineExclusionTransition } : {}), + ...(customPolicies ? { customPolicies } : {}), }; } diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index 386bb16ff5..f2ce92c07f 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -4,6 +4,7 @@ import type { InferenceSelection } from "../../inference/selection"; import type { WebSearchProvider } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; +import type { TrustedPrivatePolicyPinReceipt } from "../../policy/trusted-private-endpoints"; import type { ToolDisclosure } from "../../tool-disclosure"; import type { OpenClawImagePluginInstall } from "../openclaw-plugin-restore"; import type { SandboxMcpState } from "../registry-mcp"; @@ -16,6 +17,8 @@ export interface CustomPolicyEntry { pendingContent?: string; sourcePath?: string; appliedAt?: string; + /** Content-bound authority for generated exact destination pins. */ + trustedPrivatePins?: TrustedPrivatePolicyPinReceipt; } export interface BaselineExclusionEntry { diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 05fed8025a..4509ba413d 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -281,18 +281,13 @@ function isInstanceBackup(value: unknown): value is InstanceBackup { } function isCustomPolicyEntryArray(value: unknown): value is CustomPolicyEntry[] { - return ( - Array.isArray(value) && - value.every( - (entry) => - typeof entry === "object" && - entry !== null && - typeof (entry as { name?: unknown }).name === "string" && - typeof (entry as { content?: unknown }).content === "string" && - ((entry as { pendingContent?: unknown }).pendingContent === undefined || - typeof (entry as { pendingContent?: unknown }).pendingContent === "string"), - ) - ); + if (!Array.isArray(value)) return false; + if (value.length === 0) return true; + try { + return registry.normalizeCustomPolicyEntries(value) !== undefined; + } catch { + return false; + } } function cloneOpenClawImagePluginInstalls( diff --git a/test/registry.test.ts b/test/registry.test.ts index e1e4970245..d405e8f584 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -252,6 +252,36 @@ describe("registry", () => { expect(raw.sandboxes.alpha.mcp.managedServerNames).toEqual(["github"]); }); + it("persists normalized trusted-private MCP intent and exact pins (#8267)", () => { + registry.registerSandbox({ + name: "private-mcp", + agent: "hermes", + mcp: { + bridges: { + local: { + server: "local", + agent: "hermes", + adapter: "hermes-config", + url: "https://mcp.corp.example/mcp", + env: ["LOCAL_MCP_TOKEN"], + trustedPrivateHost: "mcp.corp.example", + allowedIps: ["10.20.30.40", "fd00::40"], + providerName: "private-mcp-mcp-local", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-local", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + expect(registry.getSandbox("private-mcp").mcp.bridges.local).toMatchObject({ + trustedPrivateHost: "mcp.corp.example", + allowedIps: ["10.20.30.40", "fd00::40"], + }); + expect(fs.readFileSync(regFile, "utf-8")).not.toContain("host-only-secret"); + }); + it("retains sanitized managed MCP names after the active bridge map is emptied", () => { registry.registerSandbox({ name: "alpha", From 5f90e35e757c12dc201c9d1bd7e69da853081c41 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 17:33:57 -0700 Subject: [PATCH 02/32] fix(policy): preserve sequential directory apply --- src/lib/actions/sandbox/policy-channel.ts | 31 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 8f9f149f2f..669c149162 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -232,22 +232,39 @@ async function addSandboxPolicyUnlocked( console.error(` No .yaml/.yml preset files in ${dirPath}`); process.exit(1); } - const prepared = await prepareExternalPolicyPresets( - files, - trustedPrivateHosts, - commandTrustedPrivateHosts, - ); + if (commandTrustedPrivateHosts.length === 0) { + for (const file of files) { + const prepared = await prepareExternalPolicyPresets([file], trustedPrivateHosts, []); + const preset = prepared?.[0]; + if ( + !preset || + !(await applyExternalPreset(sandboxName, preset, { dryRun, yes: skipConfirm })) + ) { + console.error(` Aborting --from-dir: ${file} failed. Remaining presets not applied.`); + process.exit(1); + return; + } + } + return; + } + + // Command-line declarations are strict and must be consumed exactly once + // across the whole directory. Prepare that batch before mutating policy so + // an unused or duplicate declaration cannot partially apply the directory. + const prepared = await prepareExternalPolicyPresets(files, trustedPrivateHosts, [ + ...commandTrustedPrivateHosts, + ]); if (!prepared) { process.exit(1); return; } for (const preset of prepared) { - const ok = await applyExternalPreset(sandboxName, preset, { dryRun, yes: skipConfirm }); - if (!ok) { + if (!(await applyExternalPreset(sandboxName, preset, { dryRun, yes: skipConfirm }))) { console.error( ` Aborting --from-dir: ${preset.filePath} failed. Remaining presets not applied.`, ); process.exit(1); + return; } } return; From c9d7fd2159de067311aed333aeda6eddb2cef3d9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 17:46:16 -0700 Subject: [PATCH 03/32] fix(security): address trusted endpoint review --- .../actions/sandbox/mcp-bridge-contracts.ts | 2 + .../sandbox/mcp-bridge-input-targets.test.ts | 9 ++-- .../actions/sandbox/mcp-bridge-policy.test.ts | 25 ++++++----- src/lib/actions/sandbox/mcp-bridge-policy.ts | 14 ++---- .../mcp-bridge-rebuild-exec-unavailable.ts | 9 +++- .../sandbox/mcp-bridge-url-validation.ts | 19 +++----- .../actions/sandbox/mcp-bridge-validation.ts | 1 - .../inference/endpoint-ssrf-preflight.test.ts | 18 +++++++- .../policy/trusted-private-endpoints.test.ts | 17 ++++++++ src/lib/security/trusted-private-endpoint.ts | 41 +++++++++++++++--- test/mcp-policy-transition.test.ts | 6 +-- test/registry.test.ts | 43 ++++++++++++++++++- 12 files changed, 151 insertions(+), 53 deletions(-) diff --git a/src/lib/actions/sandbox/mcp-bridge-contracts.ts b/src/lib/actions/sandbox/mcp-bridge-contracts.ts index 474f1834bb..e17775bf0e 100644 --- a/src/lib/actions/sandbox/mcp-bridge-contracts.ts +++ b/src/lib/actions/sandbox/mcp-bridge-contracts.ts @@ -4,10 +4,12 @@ import type { AgentMcpAdapter } from "../../agent/defs"; export const MCP_BRIDGE_POLICY_SOURCE = "generated:nemoclaw-mcp-bridge"; +export type McpBridgeErrorReasonCode = "rejected" | "unresolved"; export class McpBridgeError extends Error { constructor( message: string, readonly exitCode = 1, + readonly reasonCode?: McpBridgeErrorReasonCode, ) { super(message); this.name = "McpBridgeError"; diff --git a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts index c2d36b5116..f581557729 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts @@ -10,7 +10,6 @@ import { inspectMcpRecordedTargetPins, preflightMcpServerUrlResolvedTarget, } from "./mcp-bridge-url-validation"; -import { validateMcpServerUrlResolvedTarget } from "./mcp-bridge-validation"; describe("MCP URL target validation", () => { it("sorts and deduplicates public DNS pins deterministically", async () => { @@ -21,8 +20,8 @@ describe("MCP URL target validation", () => { ] as never); try { await expect( - validateMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), - ).resolves.toEqual(["2606:4700:4700::1111", "8.8.8.8"]); + preflightMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), + ).resolves.toMatchObject({ addresses: ["2606:4700:4700::1111", "8.8.8.8"] }); } finally { lookup.mockRestore(); } @@ -34,10 +33,10 @@ describe("MCP URL target validation", () => { .mockResolvedValueOnce([{ address: "127.0.0.1", family: 4 }] as never); try { await expect( - validateMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), + preflightMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), ).rejects.toThrow(/resolves to private, local, or special-use address '127\.0\.0\.1'/); await expect( - validateMcpServerUrlResolvedTarget(new URL("https://host.openshell.internal:31337/mcp")), + preflightMcpServerUrlResolvedTarget(new URL("https://host.openshell.internal:31337/mcp")), ).rejects.toThrow(/does not expose an attested driver gateway address/); expect(lookup).toHaveBeenCalledOnce(); } finally { diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.test.ts b/src/lib/actions/sandbox/mcp-bridge-policy.test.ts index 4da9835006..a02e45ea00 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.test.ts @@ -50,7 +50,7 @@ describe("MCP OpenShell policy", () => { policyName: "mcp-bridge-github", addedAt: "2026-06-01T00:00:00.000Z", }, - [], + { addresses: [] }, ), ).toThrow(/without exact public address pins/); }); @@ -140,7 +140,7 @@ describe("MCP OpenShell policy", () => { policyName: "mcp-bridge-github", addedAt: "2026-06-01T00:00:00.000Z", }, - ["8.8.8.8"], + { addresses: ["8.8.8.8"] }, ); const [, , generatedContent, options] = applyPresetContent.mock.calls[0]; @@ -166,9 +166,9 @@ describe("MCP OpenShell policy", () => { .spyOn(policies, "getPresetContentGatewayState") .mockReturnValue("match"); - expect(assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", pins)).toEqual( - registration, - ); + expect( + assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", { addresses: pins }), + ).toEqual(registration); expect(gatewayState).toHaveBeenCalledWith("alpha", content); }); @@ -194,9 +194,9 @@ describe("MCP OpenShell policy", () => { ); const gatewayState = vi.spyOn(policies, "getPresetContentGatewayState"); - expect(() => assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", pins)).toThrow( - /ownership is missing or ambiguous/, - ); + expect(() => + assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", { addresses: pins }), + ).toThrow(/ownership is missing or ambiguous/); expect(gatewayState).not.toHaveBeenCalled(); }); @@ -267,7 +267,10 @@ describe("MCP OpenShell policy", () => { const gatewayState = vi.spyOn(policies, "getPresetContentGatewayState"); expect( - () => assertGeneratedPolicyExactReadOnly("alpha", candidateEntry, "mcporter", pins), + () => + assertGeneratedPolicyExactReadOnly("alpha", candidateEntry, "mcporter", { + addresses: pins, + }), mismatch.label, ).toThrow(/not canonical for its recorded bridge definition/); expect(gatewayState, mismatch.label).not.toHaveBeenCalled(); @@ -281,7 +284,9 @@ describe("MCP OpenShell policy", () => { let message = ""; try { - assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", ["8.8.8.8"]); + assertGeneratedPolicyExactReadOnly("alpha", entry, "mcporter", { + addresses: ["8.8.8.8"], + }); } catch (error) { message = error instanceof Error ? error.message : String(error); } diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 807ab0d238..18c7b5155f 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -98,9 +98,9 @@ function reconcileGeneratedPolicyRegistration( export function applyGeneratedPolicy( sandboxName: string, entry: McpBridgeEntry, - target: McpBridgeTargetValidation | readonly string[], + target: McpBridgeTargetValidation, ): void { - const resolvedAddresses = assertMcpBridgePolicyTarget(entry, asMcpBridgePolicyTarget(target)); + const resolvedAddresses = assertMcpBridgePolicyTarget(entry, target); if (resolvedAddresses.length === 0) { throw new McpBridgeError( `Refusing to apply generated MCP policy '${entry.policyName}' without exact public address pins.`, @@ -202,12 +202,6 @@ export function applyGeneratedPolicy( ); } -function asMcpBridgePolicyTarget( - value: McpBridgeTargetValidation | readonly string[], -): McpBridgeTargetValidation { - return Array.isArray(value) ? { addresses: [...value] } : (value as McpBridgeTargetValidation); -} - export function assertMcpBridgePolicyTarget( entry: McpBridgeEntry, target: McpBridgeTargetValidation, @@ -303,9 +297,9 @@ export function assertGeneratedPolicyExactReadOnly( sandboxName: string, entry: McpBridgeEntry, adapter: AgentMcpAdapter, - target: McpBridgeTargetValidation | readonly string[], + target: McpBridgeTargetValidation, ): registry.CustomPolicyEntry { - const resolvedAddresses = assertMcpBridgePolicyTarget(entry, asMcpBridgePolicyTarget(target)); + const resolvedAddresses = assertMcpBridgePolicyTarget(entry, target); const canonicalOwnershipError = (): McpBridgeError => new McpBridgeError( "Generated MCP policy ownership is not canonical for its recorded bridge definition. Refusing host-side rebuild recovery.", diff --git a/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts b/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts index e02501968d..521fcd2c83 100644 --- a/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts +++ b/src/lib/actions/sandbox/mcp-bridge-rebuild-exec-unavailable.ts @@ -149,7 +149,14 @@ async function inspectReadOnlyRecoveryState( const targetsByServer = new Map(); for (const entry of entries) { const target = resolvedTargets.get(entry.server); - const policy = assertGeneratedPolicyExactReadOnly(sandboxName, entry, adapter, target ?? []); + const policy = assertGeneratedPolicyExactReadOnly( + sandboxName, + entry, + adapter, + target ?? { + addresses: [], + }, + ); policyByServer.set(entry.server, policyFingerprint(policy)); const provider = inspectExactMcpDestroyProvider(entry, { allowMissing: false }); providerByServer.set(entry.server, providerFingerprint(provider)); diff --git a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts index e59e1b646f..5be6860a1e 100644 --- a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts @@ -239,21 +239,20 @@ export async function preflightMcpServerUrlResolvedTarget( { trustedPrivateHosts: normalizedTrustedHosts }, ); if (!result.ok) { - const privateAnswer = result.reason?.match( - /endpoint host "[^"]+" resolves to private\/internal address "([^"]+)"/, - )?.[1]; - if (privateAnswer) { - const guidance = isLoopbackHostname(privateAnswer) + if (result.reasonCode === "private-answer" && result.offendingAddress) { + const guidance = isLoopbackHostname(result.offendingAddress) ? " Sandbox loopback is not the host MCP service. Use an HTTPS reverse proxy on a stable routed private address." : ` Use a routed private HTTPS endpoint and pass --trusted-private-host ${parsed.hostname}.`; throw new McpBridgeError( - `MCP server URL host '${parsed.hostname}' resolves to private, local, or special-use address '${privateAnswer}'.${guidance}`, + `MCP server URL host '${parsed.hostname}' resolves to private, local, or special-use address '${result.offendingAddress}'.${guidance}`, 2, + "rejected", ); } throw new McpBridgeError( `MCP server URL target validation failed: ${result.reason ?? "the endpoint was rejected"}.`, 2, + result.reasonCode === "unresolved" ? "unresolved" : "rejected", ); } const literalAddress = /^\d{1,3}(?:\.\d{1,3}){3}$/.test(parsed.hostname) @@ -267,6 +266,7 @@ export async function preflightMcpServerUrlResolvedTarget( throw new McpBridgeError( `MCP server URL host '${parsed.hostname}' resolved without any addresses before policy registration.`, 2, + "unresolved", ); } const normalizedHostname = normalizeTrustedPrivateHost(parsed.hostname); @@ -305,10 +305,6 @@ export async function preflightMcpServerUrlResolvedTarget( return { addresses }; } -export async function validateMcpServerUrlResolvedTarget(parsed: URL): Promise { - return (await preflightMcpServerUrlResolvedTarget(parsed)).addresses; -} - export async function inspectMcpRecordedTargetPins( parsed: URL, trustedPrivateHost: string, @@ -331,8 +327,7 @@ export async function inspectMcpRecordedTargetPins( }; } catch (error) { const detail = error instanceof Error ? error.message : String(error); - const unresolved = - /cannot resolve|could not be resolved|did not resolve|without any addresses/i.test(detail); + const unresolved = error instanceof McpBridgeError && error.reasonCode === "unresolved"; return { state: unresolved ? "unresolved" : "drift", detail }; } } diff --git a/src/lib/actions/sandbox/mcp-bridge-validation.ts b/src/lib/actions/sandbox/mcp-bridge-validation.ts index e7106d59fd..5155ea52ee 100644 --- a/src/lib/actions/sandbox/mcp-bridge-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-validation.ts @@ -29,7 +29,6 @@ export { normalizeMcpServerUrl, parseMcpUrl, preflightMcpServerUrlResolvedTarget, - validateMcpServerUrlResolvedTarget, } from "./mcp-bridge-url-validation"; const VALID_SERVER_RE = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; diff --git a/src/lib/inference/endpoint-ssrf-preflight.test.ts b/src/lib/inference/endpoint-ssrf-preflight.test.ts index bdcbebfa29..30e11fad1b 100644 --- a/src/lib/inference/endpoint-ssrf-preflight.test.ts +++ b/src/lib/inference/endpoint-ssrf-preflight.test.ts @@ -40,7 +40,11 @@ describe("assertEndpointResolvesPublic (#6293)", () => { ])("refuses a public hostname that resolves to the private/reserved address %s (#6293)", async (privateAddress) => { const lookup = resolverTo(privateAddress); const result = await assertEndpointResolvesPublic("https://public-name.example/v1", lookup); - expect(result.ok).toBe(false); + expect(result).toMatchObject({ + ok: false, + reasonCode: "private-answer", + offendingAddress: privateAddress, + }); expect(result.reason).toContain(privateAddress); }); @@ -211,7 +215,7 @@ describe("assertEndpointResolvesPublic (#6293)", () => { throw new Error("ENOTFOUND"); }); const result = await assertEndpointResolvesPublic("https://unresolvable.example/v1", lookup); - expect(result.ok).toBe(false); + expect(result).toMatchObject({ ok: false, reasonCode: "unresolved" }); expect(result.reason).toContain("cannot resolve"); }); @@ -223,6 +227,7 @@ describe("assertEndpointResolvesPublic (#6293)", () => { expect(result).toEqual({ ok: false, reason: 'cannot resolve endpoint host "unresolvable.example": EAI_AGAIN', + reasonCode: "unresolved", }); }); @@ -244,6 +249,15 @@ describe("assertEndpointResolvesPublic (#6293)", () => { expect(result.ok).toBe(false); }); + it("returns a rejected result when URL parsing accepts a non-canonical hostname (#8176)", async () => { + const lookup = vi.fn(); + + await expect( + assertEndpointResolvesPublic("https://my_host.corp.example/v1", lookup), + ).resolves.toMatchObject({ ok: false, reasonCode: "rejected" }); + expect(lookup).not.toHaveBeenCalled(); + }); + it.each([ "https://inference.local/v1", "http://host.openshell.internal:8000/v1", diff --git a/src/lib/policy/trusted-private-endpoints.test.ts b/src/lib/policy/trusted-private-endpoints.test.ts index ad97cfc787..9322ded244 100644 --- a/src/lib/policy/trusted-private-endpoints.test.ts +++ b/src/lib/policy/trusted-private-endpoints.test.ts @@ -110,6 +110,23 @@ network_policies: ); }); + it("rejects durable replay receipts that pin only public addresses (#8176)", () => { + const content = `network_policies: + private: + endpoints: + - host: api.corp.example + allowed_ips: [93.184.216.34] +`; + const receipt = { + version: 1, + contentDigest: createHash("sha256").update(content).digest("hex"), + }; + + expect(() => replayTrustedPrivatePolicyPinCapability(content, receipt)).toThrow( + /non-canonical private pins/, + ); + }); + it("preserves the reviewed host-gateway exception beside generated private pins (#8176)", async () => { const input = preset(`preset: name: private diff --git a/src/lib/security/trusted-private-endpoint.ts b/src/lib/security/trusted-private-endpoint.ts index 7033cd2043..4c643274ea 100644 --- a/src/lib/security/trusted-private-endpoint.ts +++ b/src/lib/security/trusted-private-endpoint.ts @@ -93,6 +93,10 @@ export interface EndpointSsrfPreflightResult { ok: boolean; /** Human-readable reason, present only when `ok === false`. */ reason?: string; + /** Stable failure classification for callers that must not parse `reason`. */ + reasonCode?: "private-answer" | "rejected" | "unresolved"; + /** Exact rejected DNS answer when `reasonCode === "private-answer"`. */ + offendingAddress?: string; /** * Validated public addresses the endpoint host resolved to, for connection * pinning (curl `--resolve`) so a subsequent probe cannot re-resolve the name @@ -243,7 +247,11 @@ export async function assertEndpointResolvesPublic( try { hostname = new URL(String(endpointUrl)).hostname; } catch { - return { ok: false, reason: `"${String(endpointUrl)}" is not a valid URL` }; + return { + ok: false, + reason: `"${String(endpointUrl)}" is not a valid URL`, + reasonCode: "rejected", + }; } // Keep the capability and range helpers import-light for generic curl @@ -252,13 +260,14 @@ export async function assertEndpointResolvesPublic( const { isLoopbackHostname, isPrivateHostname, isPrivateIp } = require("../private-networks") as typeof import("../private-networks"); - const normalizedHostname = normalizeTrustedPrivateHost(hostname); + let normalizedHostname: string; let trustedPrivateHosts: string[]; try { + normalizedHostname = normalizeTrustedPrivateHost(hostname); trustedPrivateHosts = (options.trustedPrivateHosts ?? []).map(normalizeTrustedPrivateHost); } catch (error) { const message = error instanceof Error ? error.message : String(error); - return { ok: false, reason: message }; + return { ok: false, reason: message, reasonCode: "rejected" }; } const trustedPrivateHost = trustedPrivateHosts.includes(normalizedHostname); @@ -274,7 +283,11 @@ export async function assertEndpointResolvesPublic( // A literal private IP or reserved private name is refused without resolving. if (isPrivateHostname(hostname) && !trustedPrivateHost) { - return { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; + return { + ok: false, + reason: `endpoint host "${hostname}" is a private/internal address`, + reasonCode: "rejected", + }; } // A public IP literal needs neither DNS resolution nor connection pinning: @@ -289,7 +302,11 @@ export async function assertEndpointResolvesPublic( trustedPrivateCapability: issueTrustedPrivateEndpointCapability([bare]), trustedPrivateEndpoint: true, } - : { ok: false, reason: `endpoint host "${hostname}" is a private/internal address` }; + : { + ok: false, + reason: `endpoint host "${hostname}" is a private/internal address`, + reasonCode: "rejected", + }; } let addresses: Array<{ address: string; family?: number }>; @@ -297,10 +314,18 @@ export async function assertEndpointResolvesPublic( addresses = await lookup(bare, { all: true }); } catch (err) { const message = err instanceof Error ? err.message : String(err); - return { ok: false, reason: `cannot resolve endpoint host "${hostname}": ${message}` }; + return { + ok: false, + reason: `cannot resolve endpoint host "${hostname}": ${message}`, + reasonCode: "unresolved", + }; } if (!Array.isArray(addresses) || addresses.length === 0) { - return { ok: false, reason: `endpoint host "${hostname}" did not resolve to any address` }; + return { + ok: false, + reason: `endpoint host "${hostname}" did not resolve to any address`, + reasonCode: "unresolved", + }; } for (const { address } of addresses) { // A resolved private address — including loopback reached via a public name @@ -309,6 +334,8 @@ export async function assertEndpointResolvesPublic( return { ok: false, reason: `endpoint host "${hostname}" resolves to private/internal address "${address}"`, + reasonCode: "private-answer", + offendingAddress: address, }; } } diff --git a/test/mcp-policy-transition.test.ts b/test/mcp-policy-transition.test.ts index f5abe09276..636604740c 100644 --- a/test/mcp-policy-transition.test.ts +++ b/test/mcp-policy-transition.test.ts @@ -71,7 +71,7 @@ policies.applyPresetContent = () => { let firstError = ""; try { - generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); + generated.applyGeneratedPolicy("alpha", entry, { addresses: ["8.8.8.8"] }); } catch (error) { firstError = error instanceof Error ? error.message : String(error); } @@ -83,7 +83,7 @@ const afterPresence = registry.getCustomPolicies("alpha")[0]; let retryError = ""; if (mode !== "rejected") { try { - generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); + generated.applyGeneratedPolicy("alpha", entry, { addresses: ["8.8.8.8"] }); } catch (error) { retryError = error instanceof Error ? error.message : String(error); } @@ -148,7 +148,7 @@ try { if (${JSON.stringify(operation)} === "assert") { generated.assertGeneratedPolicyMutationSafe("alpha", entry); } else { - generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); + generated.applyGeneratedPolicy("alpha", entry, { addresses: ["8.8.8.8"] }); } } catch (error) { message = error instanceof Error ? error.message : String(error); diff --git a/test/registry.test.ts b/test/registry.test.ts index d405e8f584..a720621b20 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -252,7 +252,7 @@ describe("registry", () => { expect(raw.sandboxes.alpha.mcp.managedServerNames).toEqual(["github"]); }); - it("persists normalized trusted-private MCP intent and exact pins (#8267)", () => { + it("persists canonical trusted-private MCP intent and exact pins (#8267)", () => { registry.registerSandbox({ name: "private-mcp", agent: "hermes", @@ -279,7 +279,46 @@ describe("registry", () => { trustedPrivateHost: "mcp.corp.example", allowedIps: ["10.20.30.40", "fd00::40"], }); - expect(fs.readFileSync(regFile, "utf-8")).not.toContain("host-only-secret"); + }); + + it.each([ + { + label: "non-canonical host", + trustedPrivateHost: "MCP.CORP.EXAMPLE.", + allowedIps: ["10.20.30.40", "fd00::40"], + }, + { + label: "non-canonical pin order", + trustedPrivateHost: "mcp.corp.example", + allowedIps: ["fd00::40", "10.20.30.40"], + }, + ])("rejects $label from durable trusted-private MCP authority (#8267)", ({ + trustedPrivateHost, + allowedIps, + }) => { + registry.registerSandbox({ + name: "noncanonical-private-mcp", + agent: "hermes", + mcp: { + bridges: { + local: { + server: "local", + agent: "hermes", + adapter: "hermes-config", + url: "https://mcp.corp.example/mcp", + env: ["LOCAL_MCP_TOKEN"], + trustedPrivateHost, + allowedIps, + providerName: "noncanonical-private-mcp-mcp-local", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-local", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + expect(registry.getSandbox("noncanonical-private-mcp").mcp?.bridges?.local).toBeUndefined(); }); it("retains sanitized managed MCP names after the active bridge map is emptied", () => { From 6cc92268fb66e564a7f14c409257d77f04c8203d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 18:10:09 -0700 Subject: [PATCH 04/32] fix(shields): admit recorded private MCP pins --- .../sandbox/mcp-bridge-input-targets.test.ts | 159 ++++++++++++++++-- src/lib/actions/sandbox/mcp-bridge-policy.ts | 46 ++++- .../sandbox/mcp-bridge-url-validation.ts | 31 ++-- src/lib/security/trusted-private-endpoint.ts | 2 +- src/lib/shields/mcp-policy-transition.test.ts | 37 ++++ 5 files changed, 242 insertions(+), 33 deletions(-) diff --git a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts index f581557729..195da49311 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import dns from "node:dns/promises"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -21,7 +25,7 @@ describe("MCP URL target validation", () => { try { await expect( preflightMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), - ).resolves.toMatchObject({ addresses: ["2606:4700:4700::1111", "8.8.8.8"] }); + ).resolves.toEqual({ addresses: ["2606:4700:4700::1111", "8.8.8.8"] }); } finally { lookup.mockRestore(); } @@ -44,6 +48,24 @@ describe("MCP URL target validation", () => { } }); + it("rejects IPv6 literals before DNS until the pinned proxy parser supports them", async () => { + const lookup = vi.spyOn(dns, "lookup"); + try { + await expect( + preflightMcpServerUrlResolvedTarget(new URL("https://[2606:4700:4700::1111]/mcp")), + ).rejects.toThrow(/IPv6-literal MCP server URLs are not supported/); + await expect( + preflightMcpServerUrlResolvedTarget(new URL("https://[fd00::40]/mcp"), { + trustedPrivateHosts: ["fd00::40"], + requireTrustedPrivateEndpoint: true, + }), + ).rejects.toThrow(/IPv6-literal MCP server URLs are not supported/); + expect(lookup).not.toHaveBeenCalled(); + } finally { + lookup.mockRestore(); + } + }); + it("issues exact private pins only for the matching operator trust (#8176)", async () => { const lookup = vi.spyOn(dns, "lookup").mockResolvedValue([ { address: "10.20.30.41", family: 4 }, @@ -56,8 +78,11 @@ describe("MCP URL target validation", () => { trustedPrivateHosts: ["mcp.corp.example"], requireTrustedPrivateEndpoint: true, }), - ).resolves.toMatchObject({ + ).resolves.toEqual({ addresses: ["10.20.30.40", "10.20.30.41"], + trustedPrivateCapability: expect.objectContaining({ + addresses: ["10.20.30.40", "10.20.30.41"], + }), trustedPrivateHost: "mcp.corp.example", }); } finally { @@ -65,6 +90,106 @@ describe("MCP URL target validation", () => { } }); + it("persists exact normalized pins after successful trusted-private admission (#8267)", { + timeout: 15_000, + }, () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-mcp-add-success-")); + const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +process.env.LOCAL_MCP_TOKEN = "host-only-secret"; +require("node:dns/promises").lookup = async () => [ + { address: "10.20.30.41", family: 4 }, + { address: "10.20.30.40", family: 4 }, + { address: "10.20.30.40", family: 4 }, +]; +const replace = (module, name, value) => Object.defineProperty(module, name, { + configurable: true, enumerable: true, value, writable: true, +}); +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +const adapters = require("./src/lib/actions/sandbox/mcp-bridge-adapters.js"); +const policy = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); +const provider = require("./src/lib/actions/sandbox/mcp-bridge-provider.js"); +const state = require("./src/lib/actions/sandbox/mcp-bridge-state.js"); +const validation = require("./src/lib/actions/sandbox/mcp-bridge-validation.js"); +const trusted = require("./src/lib/security/trusted-private-endpoint.js"); +let admittedTarget; +replace(policies, "getPresetContentGatewayState", () => "absent"); +replace(adapters, "assertAgentMcpConfigMutationAllowed", () => {}); +replace(adapters, "assertAgentMcpMutationRuntimeCapability", () => {}); +replace(adapters, "inspectAgentAdapterRegistration", () => ({ state: "absent" })); +replace(adapters, "registerAgentAdapter", () => {}); +replace(policy, "applyGeneratedPolicy", (_sandbox, _entry, target) => { admittedTarget = target; }); +replace(state, "ensureSandboxGatewaySelected", async () => {}); +replace(validation, "assertMcpCredentialBoundaryRuntimeVersion", () => {}); +replace(provider, "assertNoAttachedProviderCredentialCollision", () => {}); +replace(provider, "inspectMcpProvider", () => ({ + credentialKeys: null, exists: false, id: null, resourceVersion: null, type: null, +})); +replace(provider, "upsertMcpProvider", () => ({ + action: "created", + inspection: { + credentialKeys: ["LOCAL_MCP_TOKEN"], exists: true, + id: "11111111-2222-4333-8444-555555555555", resourceVersion: "1", type: "generic", + }, +})); +replace(provider, "attachProvider", () => {}); +replace(provider, "waitForAttachedMcpCredential", () => {}); +registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +require("./src/lib/actions/sandbox/mcp-bridge.js").addMcpBridge("alpha", { + server: "local", + url: "https://mcp.corp.example/mcp", + env: [{ name: "LOCAL_MCP_TOKEN" }], + trustedPrivateHosts: ["MCP.CORP.EXAMPLE."], +}).then(() => { + const entry = registry.getSandbox("alpha").mcp.bridges.local; + process.stdout.write(JSON.stringify({ + entry, + target: { + addresses: admittedTarget.addresses, + capability: trusted.isTrustedPrivateEndpointCapability( + admittedTarget.trustedPrivateCapability, + ), + capabilityAddresses: admittedTarget.trustedPrivateCapability.addresses, + trustedPrivateHost: admittedTarget.trustedPrivateHost, + }, + })); +}, (error) => { process.stderr.write(error.stack || error.message); process.exitCode = 1; }); +`; + try { + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--require=${sourceRequireHook}`] + .filter(Boolean) + .join(" "), + }, + timeout: 12_000, + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const admission = JSON.parse(result.stdout) as { + entry: Record; + target: Record; + }; + expect(admission.entry).toMatchObject({ + allowedIps: ["10.20.30.40", "10.20.30.41"], + trustedPrivateHost: "mcp.corp.example", + }); + expect(admission.target).toEqual({ + addresses: ["10.20.30.40", "10.20.30.41"], + capability: true, + capabilityAddresses: ["10.20.30.40", "10.20.30.41"], + trustedPrivateHost: "mcp.corp.example", + }); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("rejects mixed answers and an unused trusted-private option (#8267)", async () => { const lookup = vi.spyOn(dns, "lookup"); try { @@ -93,27 +218,39 @@ describe("MCP URL target validation", () => { it("reports recorded private pins as match, drift, or unresolved without mutation (#8267)", async () => { const lookup = vi.spyOn(dns, "lookup"); + const matchingPins = ["10.20.30.40"]; + const driftedPins = ["10.20.30.40"]; + const unresolvedPins = ["10.20.30.40"]; try { lookup.mockResolvedValueOnce([{ address: "10.20.30.40", family: 4 }] as never); await expect( - inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ - "10.20.30.40", - ]), + inspectMcpRecordedTargetPins( + new URL("https://mcp.corp.example/mcp"), + "mcp.corp.example", + matchingPins, + ), ).resolves.toMatchObject({ state: "match", currentAddresses: ["10.20.30.40"] }); + expect(matchingPins).toEqual(["10.20.30.40"]); lookup.mockResolvedValueOnce([{ address: "10.20.30.41", family: 4 }] as never); await expect( - inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ - "10.20.30.40", - ]), + inspectMcpRecordedTargetPins( + new URL("https://mcp.corp.example/mcp"), + "mcp.corp.example", + driftedPins, + ), ).resolves.toMatchObject({ state: "drift", currentAddresses: ["10.20.30.41"] }); + expect(driftedPins).toEqual(["10.20.30.40"]); lookup.mockRejectedValueOnce(new Error("resolver unavailable")); await expect( - inspectMcpRecordedTargetPins(new URL("https://mcp.corp.example/mcp"), "mcp.corp.example", [ - "10.20.30.40", - ]), + inspectMcpRecordedTargetPins( + new URL("https://mcp.corp.example/mcp"), + "mcp.corp.example", + unresolvedPins, + ), ).resolves.toMatchObject({ state: "unresolved" }); + expect(unresolvedPins).toEqual(["10.20.30.40"]); } finally { lookup.mockRestore(); } diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 059ed39dfa..a17aaab584 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -9,7 +9,10 @@ import type { AgentMcpAdapter } from "../../agent/defs"; import { diagnosticPreview } from "../../name-validation"; import * as policies from "../../policy"; import { isBlockedMcpUrlTargetHost } from "../../security/mcp-url-target"; -import { isTrustedPrivateEndpointCapability } from "../../security/trusted-private-endpoint"; +import { + isTrustedPrivateEndpointCapability, + replayTrustedPrivateEndpoint, +} from "../../security/trusted-private-endpoint"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { @@ -85,7 +88,12 @@ function readManagedNetworkPolicies( return networkPolicies as Record; } -function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): readonly string[] { +function requireCanonicalAllowedIps( + networkPolicy: unknown, + policyName: string, + bridge: McpBridgeEntry, +): readonly string[] { + const addressKind = bridge.trustedPrivateHost ? "trusted-private" : "public"; if (!networkPolicy || typeof networkPolicy !== "object" || Array.isArray(networkPolicy)) { throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); } @@ -99,7 +107,7 @@ function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): } const allowedIps = (endpoint as Record).allowed_ips; if (!Array.isArray(allowedIps) || allowedIps.length === 0) { - throw new Error(`Managed MCP policy '${policyName}' has no exact public address pins`); + throw new Error(`Managed MCP policy '${policyName}' has no exact ${addressKind} address pins`); } if ( allowedIps.some( @@ -107,15 +115,37 @@ function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): typeof address !== "string" || address !== address.toLowerCase() || address.includes("%") || - isIP(address) === 0 || - isBlockedMcpUrlTargetHost(address), + isIP(address) === 0, ) ) { - throw new Error(`Managed MCP policy '${policyName}' has invalid public address pins`); + throw new Error(`Managed MCP policy '${policyName}' has invalid ${addressKind} address pins`); } const pins = allowedIps as string[]; if (new Set(pins).size !== pins.length || !isDeepStrictEqual(pins, [...pins].sort())) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical public address pins`); + throw new Error( + `Managed MCP policy '${policyName}' has non-canonical ${addressKind} address pins`, + ); + } + if (bridge.trustedPrivateHost) { + let replay; + try { + replay = replayTrustedPrivateEndpoint(bridge.trustedPrivateHost, bridge.allowedIps ?? []); + } catch { + throw new Error( + `Managed MCP policy '${policyName}' has invalid trusted-private address pins`, + ); + } + if ( + replay.host !== bridge.trustedPrivateHost || + !isDeepStrictEqual(replay.addresses, bridge.allowedIps) || + !isDeepStrictEqual(pins, bridge.allowedIps) + ) { + throw new Error( + `Managed MCP policy '${policyName}' does not match its recorded trusted-private address pins`, + ); + } + } else if (pins.some((address) => isBlockedMcpUrlTargetHost(address))) { + throw new Error(`Managed MCP policy '${policyName}' has invalid public address pins`); } return pins; } @@ -194,7 +224,7 @@ function requireCanonicalManagedPolicy( } const registeredNetworkPolicy = registeredPolicies[policyKey]; - const allowedIps = requireCanonicalAllowedIps(registeredNetworkPolicy, policyName); + const allowedIps = requireCanonicalAllowedIps(registeredNetworkPolicy, policyName, bridge); let expectedDocument: Record; try { expectedDocument = parseManagedPolicyDocument( diff --git a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts index 5be6860a1e..783c02b972 100644 --- a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts @@ -59,6 +59,22 @@ function rejectUnsupportedOpenShellMcpHostAlias(hostname: string): void { ); } +function rejectUnsupportedOpenShellMcpIpv6Literal(hostname: string): void { + if (!(hostname.startsWith("[") && hostname.endsWith("]"))) return; + // invalidState: an IPv6 literal reaches an OpenShell parser that cannot + // represent and enforce its exact proxy target safely. + // sourceBoundary: the pinned OpenShell proxy parser owns literal support. + // whyNotSourceFix: v0.0.85 does not support this target form. + // regressionTest: URL normalization and resolved-target preflight both reject + // private and public IPv6 literals with this capability-specific result. + // removalCondition: remove only with reviewed parser support and parity proof; + // never infer the capability from semver alone. + throw new McpBridgeError( + "IPv6-literal MCP server URLs are not supported by the current OpenShell proxy target parser. Use a DNS hostname with public A/AAAA records.", + 2, + ); +} + function validateMcpServerUrlTarget( parsed: URL, trustedPrivateHosts: readonly string[] = [], @@ -127,19 +143,7 @@ export function normalizeMcpServerUrl( 2, ); } - if (parsed.hostname.startsWith("[") && parsed.hostname.endsWith("]")) { - // invalidState: an IPv6 literal reaches an OpenShell parser that cannot - // represent and enforce its exact proxy target safely. - // sourceBoundary: the pinned OpenShell proxy parser owns literal support. - // whyNotSourceFix: v0.0.85 does not support this target form. - // regressionTest: host/Hermes parity rejects private and public IPv6 literals. - // removalCondition: remove only with reviewed parser support and parity proof; - // never infer the capability from semver alone. - throw new McpBridgeError( - "IPv6-literal MCP server URLs are not supported by the current OpenShell proxy target parser. Use a DNS hostname with public A/AAAA records.", - 2, - ); - } + rejectUnsupportedOpenShellMcpIpv6Literal(parsed.hostname); if (parsed.username || parsed.password) { throw new McpBridgeError( "MCP server URL must not embed credentials. Use --env KEY so OpenShell resolves host-only credentials.", @@ -226,6 +230,7 @@ export async function preflightMcpServerUrlResolvedTarget( // a 403 plus zero upstream requests for all three adapters. // removalCondition: revisit only when the pinned OpenShell implementation or // its allowed_ips resolve-validate-connect contract changes. + rejectUnsupportedOpenShellMcpIpv6Literal(parsed.hostname); rejectUnsupportedOpenShellMcpHostAlias(parsed.hostname); const normalizedTrustedHosts = (options.trustedPrivateHosts ?? []).map((host) => normalizeTrustedPrivateHost(host), diff --git a/src/lib/security/trusted-private-endpoint.ts b/src/lib/security/trusted-private-endpoint.ts index 4c643274ea..5b67f466a8 100644 --- a/src/lib/security/trusted-private-endpoint.ts +++ b/src/lib/security/trusted-private-endpoint.ts @@ -57,7 +57,7 @@ function issueTrustedPrivateEndpointCapability( addresses: readonly string[], ): TrustedPrivateEndpointCapability { const capability = Object.freeze({ - addresses: Object.freeze([...new Set(addresses)]), + addresses: Object.freeze([...new Set(addresses.map(normalizeIpLiteral))].sort()), }) as unknown as TrustedPrivateEndpointCapability; TRUSTED_PRIVATE_ENDPOINT_CAPABILITIES.add(capability); return capability; diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts index 951c6991a4..be00142d5a 100644 --- a/src/lib/shields/mcp-policy-transition.test.ts +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -126,6 +126,43 @@ describe("managed MCP Shields policy transitions (#7952)", () => { ]); }); + it("admits exact recorded private pins only for a trusted-private bridge", () => { + const alpha = registeredPolicy("alpha", "10.20.30.40"); + const sandbox = sandboxWithPolicies([alpha]); + Object.assign(sandbox.mcp!.bridges.alpha!, { + trustedPrivateHost: "alpha.example.com", + allowedIps: ["10.20.30.40"], + }); + + expect( + inspectExactManagedMcpPolicies( + sandbox, + livePolicy([{ content: alpha.content, server: "alpha" }]), + ), + ).toEqual([ + expect.objectContaining({ + key: "mcp_bridge_alpha", + server: "alpha", + }), + ]); + }); + + it("rejects a trusted-private policy that differs from its durable pins", () => { + const alpha = registeredPolicy("alpha", "10.20.30.40"); + const sandbox = sandboxWithPolicies([alpha]); + Object.assign(sandbox.mcp!.bridges.alpha!, { + trustedPrivateHost: "alpha.example.com", + allowedIps: ["10.20.30.41"], + }); + + expect(() => + inspectExactManagedMcpPolicies( + sandbox, + livePolicy([{ content: alpha.content, server: "alpha" }]), + ), + ).toThrow(/does not match its recorded trusted-private address pins/); + }); + it.each([ { label: "pending policy content", From 440705d95e4c25e9349dc597270f7d9dc2a31aac Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 18:29:03 -0700 Subject: [PATCH 05/32] fix(security): reject mixed trusted policy answers --- .../policy/trusted-private-endpoints.test.ts | 26 +++++++++++++++++-- src/lib/policy/trusted-private-endpoints.ts | 14 +++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/lib/policy/trusted-private-endpoints.test.ts b/src/lib/policy/trusted-private-endpoints.test.ts index 9322ded244..a627554446 100644 --- a/src/lib/policy/trusted-private-endpoints.test.ts +++ b/src/lib/policy/trusted-private-endpoints.test.ts @@ -40,14 +40,14 @@ network_policies: `); const [prepared] = await prepareTrustedPrivatePolicyPresets([input], ["API.CORP.EXAMPLE."], { - lookup: lookup({ "api.corp.example": ["10.20.30.40", "2001:4860:4860::8888"] }), + lookup: lookup({ "api.corp.example": ["fd00::40", "10.20.30.40"] }), }); const document = YAML.parse(prepared.content) as { network_policies: { services: { endpoints: Array<{ allowed_ips?: string[] }> } }; }; expect(document.network_policies.services.endpoints).toHaveLength(4); for (const endpoint of document.network_policies.services.endpoints) { - expect(endpoint.allowed_ips).toEqual(["10.20.30.40", "2001:4860:4860::8888"]); + expect(endpoint.allowed_ips).toEqual(["10.20.30.40", "fd00::40"]); } expect(prepared.trustedPrivatePins).toMatchObject({ version: 1, @@ -79,6 +79,28 @@ network_policies: expect(input.content).not.toContain("allowed_ips"); }); + it.each([ + "rest", + "websocket", + "jsonrpc", + "mcp", + ])("rejects mixed public and private DNS answers for %s endpoints (#8176)", async (protocol) => { + const input = preset(`preset: + name: private +network_policies: + services: + endpoints: + - { host: api.corp.example, port: 443, protocol: ${protocol} } +`); + + await expect( + prepareTrustedPrivatePolicyPresets([input], ["api.corp.example"], { + lookup: lookup({ "api.corp.example": ["10.20.30.40", "8.8.8.8"] }), + }), + ).rejects.toThrow(/mixed public and private addresses/); + expect(input.content).not.toContain("allowed_ips"); + }); + it("rejects a pin receipt that is stale, malformed, or unversioned (#8176)", () => { const content = "network_policies: {}\n"; for (const receipt of [ diff --git a/src/lib/policy/trusted-private-endpoints.ts b/src/lib/policy/trusted-private-endpoints.ts index db34952f29..45db11f452 100644 --- a/src/lib/policy/trusted-private-endpoints.ts +++ b/src/lib/policy/trusted-private-endpoints.ts @@ -314,7 +314,7 @@ export async function prepareTrustedPrivatePolicyPresets( } continue; } - const pins = [ + const resolvedPins = [ ...new Set( (result.addresses?.length ? result.addresses @@ -322,6 +322,18 @@ export async function prepareTrustedPrivatePolicyPresets( ).map((address) => address.toLowerCase()), ), ].sort(); + const capabilityPins = [...result.trustedPrivateCapability.addresses] + .map((address) => address.toLowerCase()) + .sort(); + if ( + resolvedPins.length !== capabilityPins.length || + resolvedPins.some((address, index) => address !== capabilityPins[index]) + ) { + throw new Error( + `Trusted private host '${host}' returned mixed public and private addresses. Trusted-private policy endpoints must resolve only to supported routed private addresses.`, + ); + } + const pins = capabilityPins; if (pins.length === 0) { throw new Error(`Trusted private host '${host}' produced no validated address pins.`); } From d1006cb420a4745111fd5d992b020b6cef84ee72 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 20:23:05 -0700 Subject: [PATCH 06/32] test(shields): bind restore runner mock --- src/lib/shields/index.test.ts | 15 +++++++++++++-- src/lib/shields/index.ts | 3 ++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 82d8a23e89..665cd13542 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -612,7 +612,7 @@ describe("shields — unit logic", () => { }); vi.spyOn(process, "kill").mockImplementation(routeProcessKill); const { applyShieldsPolicySnapshot } = await loadShieldsModule(); - const { buildPolicySetCommand } = await import("../policy"); + const { run } = await import("../runner"); const createTempDirectory = vi.spyOn(fs, "mkdtempSync").mockImplementation(() => { throw Object.assign(new Error("ENOSPC: simulated temporary storage full"), { code: "ENOSPC", @@ -627,7 +627,18 @@ describe("shields — unit logic", () => { expect(result.status).toBe(0); expect(createTempDirectory).not.toHaveBeenCalled(); - expect(buildPolicySetCommand).toHaveBeenCalledWith(snapshotPath, sandboxName); + expect(run).toHaveBeenCalledWith( + [ + expect.stringMatching(/(?:^|\/)openshell$/), + "policy", + "set", + "--policy", + snapshotPath, + "--wait", + sandboxName, + ], + { ignoreError: true }, + ); }); it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 379f6608ae..02145972ec 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -19,11 +19,12 @@ // in those modules, and a later decomposition must preserve the transition and // timer-bound lock tests before this facade can shrink safely. +import { run, runCapture, validateName } from "../runner"; + const fs = require("fs"); const path = require("path"); const { fork } = require("child_process"); const { randomBytes } = require("crypto"); -const { run, runCapture, validateName } = require("../runner"); const { CLI_NAME }: typeof import("../cli/branding") = require("../cli/branding"); const { isObjectRecord }: typeof import("../core/json-types") = require("../core/json-types"); const { From ea10bf280af1bf9029941e978f0a9cec5ffda90c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 21:12:39 -0700 Subject: [PATCH 07/32] ci: refresh exact-head validation gates Signed-off-by: Aaron Erickson From 96a26ac927c183bfc5241b23adadd6b91394cfba Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 21:23:44 -0700 Subject: [PATCH 08/32] fix(security): reject mixed inference DNS answers --- .../compatible-endpoint-context.test.ts | 21 +++++++++++++++++++ .../inference/endpoint-ssrf-preflight.test.ts | 18 ++++++++++++++++ src/lib/security/trusted-private-endpoint.ts | 21 +++++++++++++++++-- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/compatible-endpoint-context.test.ts b/src/lib/inference/compatible-endpoint-context.test.ts index d67b488257..fffd3e59c6 100644 --- a/src/lib/inference/compatible-endpoint-context.test.ts +++ b/src/lib/inference/compatible-endpoint-context.test.ts @@ -223,6 +223,27 @@ describe("compatible-endpoint context window", () => { expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); }); + it("does not probe an allowlisted endpoint with mixed private and public DNS answers (#8176)", async () => { + const fetchModels = vi.fn(() => ({ data: [{ id: "model-a", max_model_len: 65_536 }] })); + const messages: string[] = []; + const env: NodeJS.ProcessEnv = { + NEMOCLAW_TRUSTED_PRIVATE_HOSTS: "llm.corp.example", + }; + await applyCompatibleEndpointContextWindow("https://llm.corp.example/v1", "model-a", { + env, + fetchModels, + resolveHost: async () => [ + { address: "10.0.0.8", family: 4 }, + { address: "93.184.216.34", family: 4 }, + ], + logger: { log: (m) => messages.push(m), warn: (m) => messages.push(m) }, + }); + + expect(fetchModels).not.toHaveBeenCalled(); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined(); + expect(messages.some((message) => message.includes("93.184.216.34"))).toBe(true); + }); + it.each([ "http://127.0.0.1:8000/v1", "http://localhost:8000/v1", diff --git a/src/lib/inference/endpoint-ssrf-preflight.test.ts b/src/lib/inference/endpoint-ssrf-preflight.test.ts index 30e11fad1b..64b7dbf48c 100644 --- a/src/lib/inference/endpoint-ssrf-preflight.test.ts +++ b/src/lib/inference/endpoint-ssrf-preflight.test.ts @@ -70,6 +70,24 @@ describe("assertEndpointResolvesPublic (#6293)", () => { expect(lookup).toHaveBeenCalledWith("llm.corp.example", { all: true }); }); + it("rejects mixed public and trusted-private answers for an exact trusted hostname (#8176)", async () => { + const result = await assertEndpointResolvesPublic( + "https://llm.corp.example/v1", + async () => [ + { address: "10.0.0.8", family: 4 }, + { address: "93.184.216.34", family: 4 }, + ], + { trustedPrivateHosts: ["llm.corp.example"] }, + ); + + expect(result).toMatchObject({ + ok: false, + reasonCode: "mixed-answer", + offendingAddress: "93.184.216.34", + }); + expect(result.trustedPrivateCapability).toBeUndefined(); + }); + it("does not treat a trusted hostname as a suffix or wildcard allowlist (#6861)", async () => { const result = await assertEndpointResolvesPublic( "https://attacker.llm.corp.example/v1", diff --git a/src/lib/security/trusted-private-endpoint.ts b/src/lib/security/trusted-private-endpoint.ts index 5b67f466a8..f6b7ad858d 100644 --- a/src/lib/security/trusted-private-endpoint.ts +++ b/src/lib/security/trusted-private-endpoint.ts @@ -94,8 +94,8 @@ export interface EndpointSsrfPreflightResult { /** Human-readable reason, present only when `ok === false`. */ reason?: string; /** Stable failure classification for callers that must not parse `reason`. */ - reasonCode?: "private-answer" | "rejected" | "unresolved"; - /** Exact rejected DNS answer when `reasonCode === "private-answer"`. */ + reasonCode?: "mixed-answer" | "private-answer" | "rejected" | "unresolved"; + /** Exact rejected DNS answer when `reasonCode` identifies an address failure. */ offendingAddress?: string; /** * Validated public addresses the endpoint host resolved to, for connection @@ -341,6 +341,23 @@ export async function assertEndpointResolvesPublic( } const resolvedAddresses = addresses.map(({ address }) => address); const trustedPrivateAddresses = resolvedAddresses.filter((address) => isPrivateIp(address)); + if ( + trustedPrivateHost && + trustedPrivateAddresses.length > 0 && + trustedPrivateAddresses.length !== resolvedAddresses.length + ) { + const offendingAddress = resolvedAddresses.find( + (address) => !isOperatorTrustablePrivateIp(address), + ); + return { + ok: false, + reason: + `endpoint host "${hostname}" resolves to both trusted-private and public addresses` + + (offendingAddress ? `, including untrusted answer "${offendingAddress}"` : ""), + reasonCode: "mixed-answer", + offendingAddress, + }; + } return trustedPrivateHost && trustedPrivateAddresses.length > 0 ? { ok: true, From cb9780b4e223b1b2888dc9ec95dc23e8affc09f1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 21:37:04 -0700 Subject: [PATCH 09/32] fix(security): preserve mixed-answer diagnostic --- src/lib/security/trusted-private-endpoint.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/security/trusted-private-endpoint.ts b/src/lib/security/trusted-private-endpoint.ts index f6b7ad858d..2db7b63c97 100644 --- a/src/lib/security/trusted-private-endpoint.ts +++ b/src/lib/security/trusted-private-endpoint.ts @@ -352,7 +352,7 @@ export async function assertEndpointResolvesPublic( return { ok: false, reason: - `endpoint host "${hostname}" resolves to both trusted-private and public addresses` + + `endpoint host "${hostname}" resolves to mixed public and private addresses` + (offendingAddress ? `, including untrusted answer "${offendingAddress}"` : ""), reasonCode: "mixed-answer", offendingAddress, From f30d76c834402d36e159a6affb837a39ee690d41 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 21:58:06 -0700 Subject: [PATCH 10/32] test(e2e): prove trusted-private MCP rebinding --- test/e2e/live/mcp-bridge-onboard-env.ts | 4 + test/e2e/live/mcp-bridge-tool-discovery.ts | 8 +- test/e2e/live/mcp-bridge-trusted-private.ts | 244 ++++++++++++++++++ test/e2e/live/mcp-bridge.test.ts | 194 ++------------ test/e2e/setup-mcp-test-tls.sh | 11 +- .../support/mcp-bridge-onboard-env.test.ts | 9 + 6 files changed, 290 insertions(+), 180 deletions(-) create mode 100644 test/e2e/live/mcp-bridge-trusted-private.ts diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index 6b210bd8e0..aab4f763ed 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -32,6 +32,7 @@ export function buildMcpBridgeOnboardEnv(options: { baseEnv?: NodeJS.ProcessEnv; compatibleKey: string; compatibleModel: string; + corporateCaBundle?: string; endpointUrl: string; envOverlay?: NodeJS.ProcessEnv; sandboxName: string; @@ -40,6 +41,9 @@ export function buildMcpBridgeOnboardEnv(options: { ...buildMcpBridgeExactMainEnv(options), COMPATIBLE_API_KEY: options.compatibleKey, NVIDIA_INFERENCE_API_KEY: options.compatibleKey, + ...(options.corporateCaBundle + ? { NEMOCLAW_CORPORATE_CA_BUNDLE: options.corporateCaBundle } + : {}), NEMOCLAW_AGENT: options.agent, NEMOCLAW_ENDPOINT_URL: options.endpointUrl, NEMOCLAW_MODEL: options.compatibleModel, diff --git a/test/e2e/live/mcp-bridge-tool-discovery.ts b/test/e2e/live/mcp-bridge-tool-discovery.ts index 965735b1c5..cc22136a15 100644 --- a/test/e2e/live/mcp-bridge-tool-discovery.ts +++ b/test/e2e/live/mcp-bridge-tool-discovery.ts @@ -138,21 +138,25 @@ export async function assertAuthenticatedMcpToolDiscovery( options: { sandboxName: string; artifactPrefix: string; + credentialKey?: string; hostSecret: string; progress: Pick; + serverName?: string; }, ): Promise { + const credentialKey = options.credentialKey ?? "FAKE_MCP_SECRET"; + const serverName = options.serverName ?? "fake"; const requestOffset = fakeMcp.requests.length; let status: Awaited> | undefined; let statusJson: McpToolDiscoveryStatusJson | undefined; for (let attempt = 1; attempt <= MCP_TOOL_DISCOVERY_ATTEMPTS; attempt += 1) { status = await host.nemoclaw( - [options.sandboxName, "mcp", "status", "fake", "--tools", "--json"], + [options.sandboxName, "mcp", "status", serverName, "--tools", "--json"], { artifactName: `${options.artifactPrefix}-mcp-status-tools-json${attempt === 1 ? "" : `-retry-${attempt}`}`, env: { ...buildAvailabilityProbeEnv(), - FAKE_MCP_SECRET: options.hostSecret, + [credentialKey]: options.hostSecret, }, redactionValues: [options.hostSecret], timeoutMs: 60_000, diff --git a/test/e2e/live/mcp-bridge-trusted-private.ts b/test/e2e/live/mcp-bridge-trusted-private.ts new file mode 100644 index 0000000000..a828bef6af --- /dev/null +++ b/test/e2e/live/mcp-bridge-trusted-private.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { assertExitZero as expectExitZero } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { MCP_BRIDGE_TEST_CREDENTIALS } from "../fixtures/mcp-bridge-credentials.ts"; +import { + assertManagedMcpPolicySurvivedRemoval, + buildMcpDnsRebindingProbeScript, + captureManagedMcpPolicy, + hostPrivateAddressForSandbox, + isExpectedMcpCurlPolicyDenial, + type McpDnsRebindingAdapter, + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./mcp-bridge-sandbox.ts"; +import { startFakeMcpHttpsServer } from "./mcp-bridge-servers.ts"; +import { assertAuthenticatedMcpToolDiscovery } from "./mcp-bridge-tool-discovery.ts"; + +const SERVER_POLICY_KEY = "mcp_bridge_fake"; +const REBIND_SERVER_NAME = "rebind"; +const REBIND_POLICY_KEY = "mcp_bridge_rebind"; +const REBIND_HOSTNAME = "mcp-rebind.example.test"; +const REBIND_PUBLIC_IP = "1.1.1.1"; +const REBIND_CREDENTIAL_KEY = "REBIND_MCP_SECRET"; +const REBIND_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rebindHost; + +export async function assertTrustedPrivateMcpRebindingDenied( + host: HostCliClient, + sandbox: SandboxClient, + cleanup: CleanupRegistry, + options: { + adapter: McpDnsRebindingAdapter; + artifactPrefix: string; + assertSecretAbsent: ( + sandbox: SandboxClient, + sandboxName: string, + paths: string[], + secrets: string[], + artifactName: string, + ) => Promise; + cleanupBridge: ( + host: HostCliClient, + sandboxName: string, + server: string, + adapter: McpDnsRebindingAdapter, + ) => Promise; + mutationTimeoutMs: number; + progress: Parameters[2]["progress"]; + sandboxName: string; + secretPaths: string[]; + survivingMcpUrl: string; + }, +): Promise { + const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); + cleanup.add(`stop ${options.artifactPrefix} trusted-private fake MCP HTTPS server`, () => + rebindMcp.close(), + ); + cleanup.add(`remove ${options.artifactPrefix} trusted-private MCP bridge`, () => + options.cleanupBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), + ); + const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${rebindMcp.port}/mcp`; + const hostsFixture = await setupDnsRebindingHostsFixture( + host, + options.sandboxName, + REBIND_HOSTNAME, + ); + cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => + restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), + ); + const survivingPolicyBeforeAddResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-before-add`, + label: `${options.artifactPrefix} captures the surviving MCP policy before adding the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; + const trustedPrivateAddress = await hostPrivateAddressForSandbox(host); + expect(trustedPrivateAddress).not.toBe(REBIND_PUBLIC_IP); + await remapDnsRebindingHostname( + host, + options.sandboxName, + hostsFixture, + trustedPrivateAddress, + `${options.artifactPrefix}-mcp-trusted-private-map-before-add`, + ); + const add = await host.nemoclaw( + [ + options.sandboxName, + "mcp", + "add", + REBIND_SERVER_NAME, + "--url", + rebindMcpUrl, + "--env", + REBIND_CREDENTIAL_KEY, + "--trusted-private-host", + REBIND_HOSTNAME, + ], + { + artifactName: `${options.artifactPrefix}-mcp-trusted-private-add`, + env: { + ...buildAvailabilityProbeEnv(), + [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, + }, + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: options.mutationTimeoutMs, + }, + ); + expectExitZero( + add, + `${options.artifactPrefix} registers a routed-private MCP endpoint with explicit trust`, + ); + const status = await host.nemoclaw( + [options.sandboxName, "mcp", "status", REBIND_SERVER_NAME, "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-status-after-add`, + env: { + ...buildAvailabilityProbeEnv(), + [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, + }, + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(status, `${options.artifactPrefix} inspects trusted-private route after add`); + expect(JSON.parse(status.stdout)).toMatchObject({ + support: { supported: true, adapter: options.adapter }, + server: REBIND_SERVER_NAME, + url: rebindMcpUrl, + env: { names: [REBIND_CREDENTIAL_KEY], ready: true, missing: [] }, + provider: { attached: true, credentialReady: true }, + policy: { gatewayPresent: true }, + adapter: { registered: true }, + trustedPrivateTarget: { + host: REBIND_HOSTNAME, + recordedPins: [trustedPrivateAddress], + currentPins: [trustedPrivateAddress], + state: "match", + }, + }); + const rebindingPolicy = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-trusted-private-policy-pinned-address`, + label: `${options.artifactPrefix} validates the trusted-private add-time DNS pin`, + policyKey: REBIND_POLICY_KEY, + sandboxName: options.sandboxName, + url: rebindMcpUrl, + }); + expect(rebindingPolicy.policy.endpoints?.[0]).toMatchObject({ + host: REBIND_HOSTNAME, + allowed_ips: [trustedPrivateAddress], + }); + await options.assertSecretAbsent( + sandbox, + options.sandboxName, + options.secretPaths, + [REBIND_HOST_SECRET], + `${options.artifactPrefix}-dns-rebinding-secret-absent-from-sandbox`, + ); + await assertAuthenticatedMcpToolDiscovery(host, rebindMcp, { + sandboxName: options.sandboxName, + artifactPrefix: `${options.artifactPrefix}-trusted-private`, + credentialKey: REBIND_CREDENTIAL_KEY, + hostSecret: REBIND_HOST_SECRET, + progress: options.progress, + serverName: REBIND_SERVER_NAME, + }); + const requestsBeforeRebindingDenial = rebindMcp.requests.length; + + // OpenShell must retain the exact private address admitted at add time. A + // later DNS answer outside that set is drift for status and a hard denial + // for every adapter runtime identity. + await remapDnsRebindingHostname( + host, + options.sandboxName, + hostsFixture, + REBIND_PUBLIC_IP, + `${options.artifactPrefix}-mcp-trusted-private-map-public-unpinned-after-add`, + ); + const driftStatus = await host.nemoclaw( + [options.sandboxName, "mcp", "status", REBIND_SERVER_NAME, "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-trusted-private-status-drift`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expectExitZero(driftStatus, `${options.artifactPrefix} reports trusted-private pin drift`); + expect(JSON.parse(driftStatus.stdout)).toMatchObject({ + trustedPrivateTarget: { + host: REBIND_HOSTNAME, + recordedPins: [trustedPrivateAddress], + currentPins: [REBIND_PUBLIC_IP], + state: "drift", + }, + }); + const denial = await sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript( + buildMcpDnsRebindingProbeScript(options.adapter, rebindMcpUrl, REBIND_CREDENTIAL_KEY), + ), + { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-adapter-denied`, + env: buildAvailabilityProbeEnv(), + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: 90_000, + }, + ); + expect( + isExpectedMcpCurlPolicyDenial(denial), + `${options.artifactPrefix} adapter identity must receive an OpenShell policy denial after rebinding\nstdout:\n${denial.stdout}\nstderr:\n${denial.stderr}`, + ).toBe(true); + expect( + rebindMcp.requests.length, + `${options.artifactPrefix} rebound request must not reach the upstream MCP server`, + ).toBe(requestsBeforeRebindingDenial); + + // Restore before removal can reload policy and restart the sandbox. + await restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture); + const remove = await host.nemoclaw([options.sandboxName, "mcp", "remove", REBIND_SERVER_NAME], { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: options.mutationTimeoutMs, + }); + expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); + const survivingPolicyAfterRemoveResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-after-remove`, + label: `${options.artifactPrefix} inspects MCP policy after removing the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + assertManagedMcpPolicySurvivedRemoval( + survivingPolicyBeforeAdd, + survivingPolicyAfterRemoveResult, + REBIND_POLICY_KEY, + ); +} diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 84789a9161..669d8c2817 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -32,17 +32,12 @@ import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-brid import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts"; import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts"; import { - assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, captureManagedMcpPolicy, expectExitNonZero, hostAddressForSandbox, - hostPrivateAddressForSandbox, isExpectedMcpCurlPolicyDenial, type McpDnsRebindingAdapter, - remapDnsRebindingHostname, - restoreDnsRebindingHostsFixture, - setupDnsRebindingHostsFixture, } from "./mcp-bridge-sandbox.ts"; import { startCompatibleMock, @@ -54,6 +49,7 @@ import { assertAuthenticatedMcpRediscovery, assertAuthenticatedMcpToolDiscovery, } from "./mcp-bridge-tool-discovery.ts"; +import { assertTrustedPrivateMcpRebindingDenied } from "./mcp-bridge-trusted-private.ts"; import { MCP_PROVIDER_REWRITE_PROBE_SOURCE } from "./mcp-provider-rewrite-probe.ts"; import { assertRawOpenShellAllowedIpsRebindingDenied } from "./openshell-allowed-ips-rebinding.ts"; import { prepareExactMainMcpProof } from "./openshell-exact-main-mcp-proof.ts"; @@ -64,14 +60,8 @@ const DEEPAGENTS_SANDBOX_NAME = process.env.NEMOCLAW_MCP_DEEPAGENTS_SANDBOX_NAME const SERVER_NAME = "fake"; const SERVER_POLICY_KEY = "mcp_bridge_fake"; const CONCURRENT_SERVER_NAME = "concurrent"; -const REBIND_SERVER_NAME = "rebind"; -const REBIND_POLICY_KEY = "mcp_bridge_rebind"; -const REBIND_HOSTNAME = "mcp-rebind.example.test"; -const REBIND_PUBLIC_IP = "1.1.1.1"; -const REBIND_CREDENTIAL_KEY = "REBIND_MCP_SECRET"; const HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.host; const ROTATED_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rotatedHost; -const REBIND_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rebindHost; const COMPATIBLE_KEY = MCP_BRIDGE_TEST_CREDENTIALS.compatibleEndpoint; const COMPATIBLE_MODEL = "mock/mcp-bridge"; const TOOL_CHALLENGE = "nemoclaw-authenticated-mcp-proof"; @@ -118,6 +108,10 @@ async function onboardAgent( envOverlay?: NodeJS.ProcessEnv; }, ): Promise { + const corporateCaBundle = process.env.NEMOCLAW_MCP_TLS_CA_CERT; + if (!corporateCaBundle) { + throw new Error("NEMOCLAW_MCP_TLS_CA_CERT is required for routed-private MCP validation"); + } cleanup.trackSandbox(host, options.sandboxName, { artifactName: "cleanup-destroy-sandbox", timeoutMs: 15 * 60_000, @@ -134,6 +128,7 @@ async function onboardAgent( agent: options.agent, compatibleKey: COMPATIBLE_KEY, compatibleModel: COMPATIBLE_MODEL, + corporateCaBundle, endpointUrl, envOverlay: options.envOverlay, sandboxName: options.sandboxName, @@ -165,165 +160,6 @@ async function assertSecretAbsentFromSandbox( }); expectExitZero(result, "host MCP secret must not appear in sandbox files"); } -async function assertAdapterDnsRebindingDenied( - host: HostCliClient, - sandbox: SandboxClient, - cleanup: CleanupRegistry, - options: { - adapter: McpDnsRebindingAdapter; - artifactPrefix: string; - sandboxName: string; - secretPaths: string[]; - survivingMcpUrl: string; - }, -): Promise { - const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); - cleanup.add(`stop ${options.artifactPrefix} DNS rebinding fake MCP HTTPS server`, () => - rebindMcp.close(), - ); - cleanup.add(`remove ${options.artifactPrefix} DNS rebinding MCP bridge`, () => - cleanupMcpBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), - ); - const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${rebindMcp.port}/mcp`; - const hostsFixture = await setupDnsRebindingHostsFixture( - host, - options.sandboxName, - REBIND_HOSTNAME, - ); - cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => - restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), - ); - const survivingPolicyBeforeAddResult = await captureManagedMcpPolicy(sandbox, { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-before-add`, - label: `${options.artifactPrefix} captures the surviving MCP policy before adding the rebinding route`, - policyKey: SERVER_POLICY_KEY, - sandboxName: options.sandboxName, - url: options.survivingMcpUrl, - }); - const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; - await remapDnsRebindingHostname( - host, - options.sandboxName, - hostsFixture, - REBIND_PUBLIC_IP, - `${options.artifactPrefix}-mcp-dns-rebinding-map-public-before-add`, - ); - const add = await host.nemoclaw( - [ - options.sandboxName, - "mcp", - "add", - REBIND_SERVER_NAME, - "--url", - rebindMcpUrl, - "--env", - REBIND_CREDENTIAL_KEY, - ], - { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-add-with-public-resolution`, - env: { - ...buildAvailabilityProbeEnv(), - [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, - }, - redactionValues: [REBIND_HOST_SECRET], - timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], - }, - ); - expectExitZero( - add, - `${options.artifactPrefix} registers MCP route while its dedicated hostname resolves publicly`, - ); - const status = await host.nemoclaw( - [options.sandboxName, "mcp", "status", REBIND_SERVER_NAME, "--json"], - { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-status-after-add`, - env: { - ...buildAvailabilityProbeEnv(), - [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, - }, - redactionValues: [REBIND_HOST_SECRET], - timeoutMs: 60_000, - }, - ); - expectExitZero(status, `${options.artifactPrefix} inspects DNS rebinding route after add`); - expect(JSON.parse(status.stdout)).toMatchObject({ - support: { supported: true, adapter: options.adapter }, - server: REBIND_SERVER_NAME, - url: rebindMcpUrl, - env: { names: [REBIND_CREDENTIAL_KEY], ready: true, missing: [] }, - provider: { attached: true, credentialReady: true }, - policy: { gatewayPresent: true }, - adapter: { registered: true }, - }); - const rebindingPolicy = await captureManagedMcpPolicy(sandbox, { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, - label: `${options.artifactPrefix} validates the add-time DNS pin`, - policyKey: REBIND_POLICY_KEY, - sandboxName: options.sandboxName, - url: rebindMcpUrl, - }); - expect(rebindingPolicy.policy.endpoints?.[0]).toMatchObject({ - host: REBIND_HOSTNAME, - allowed_ips: [REBIND_PUBLIC_IP], - }); - await assertSecretAbsentFromSandbox( - sandbox, - options.sandboxName, - options.secretPaths, - [REBIND_HOST_SECRET], - `${options.artifactPrefix}-dns-rebinding-secret-absent-from-sandbox`, - ); - // OpenShell connects to the address list resolved and validated against allowed_ips. - const reboundAddress = await hostPrivateAddressForSandbox(host); - expect(reboundAddress).not.toBe(REBIND_PUBLIC_IP); - await remapDnsRebindingHostname( - host, - options.sandboxName, - hostsFixture, - reboundAddress, - `${options.artifactPrefix}-mcp-dns-rebinding-map-private-unpinned-after-add`, - ); - const denial = await sandbox.execShell( - options.sandboxName, - trustedSandboxShellScript( - buildMcpDnsRebindingProbeScript(options.adapter, rebindMcpUrl, REBIND_CREDENTIAL_KEY), - ), - { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-adapter-denied`, - env: buildAvailabilityProbeEnv(), - redactionValues: [REBIND_HOST_SECRET], - timeoutMs: 90_000, - }, - ); - expect( - isExpectedMcpCurlPolicyDenial(denial), - `${options.artifactPrefix} adapter identity must receive an OpenShell policy denial after rebinding\nstdout:\n${denial.stdout}\nstderr:\n${denial.stderr}`, - ).toBe(true); - expect( - rebindMcp.requests, - `${options.artifactPrefix} rebound request must not reach the upstream MCP server`, - ).toHaveLength(0); - // Restore before removal can reload policy and restart the sandbox. - await restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture); - const remove = await host.nemoclaw([options.sandboxName, "mcp", "remove", REBIND_SERVER_NAME], { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-remove`, - env: buildAvailabilityProbeEnv(), - timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], - }); - expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); - const survivingPolicyAfterRemoveResult = await captureManagedMcpPolicy(sandbox, { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-after-remove`, - label: `${options.artifactPrefix} inspects MCP policy after removing the rebinding route`, - policyKey: SERVER_POLICY_KEY, - sandboxName: options.sandboxName, - url: options.survivingMcpUrl, - }); - assertManagedMcpPolicySurvivedRemoval( - survivingPolicyBeforeAdd, - survivingPolicyAfterRemoveResult, - REBIND_POLICY_KEY, - ); -} async function addBridgeAndReadStatus( host: HostCliClient, options: { @@ -976,12 +812,16 @@ test("mcp-bridge", { }, ); - await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + await assertTrustedPrivateMcpRebindingDenied(host, sandbox, cleanup, { adapter: "mcporter", artifactPrefix: "openclaw", + assertSecretAbsent: assertSecretAbsentFromSandbox, + cleanupBridge: cleanupMcpBridge, + mutationTimeoutMs: MCP_MUTATION_TIMEOUT_MS.mcporter, sandboxName: OPENCLAW_SANDBOX_NAME, secretPaths: ["/sandbox/.openclaw", "/sandbox/.mcp.json"], survivingMcpUrl: mcpUrl, + progress, }); const requestCountBeforeAllowedNodeProof = fakeMcp.requests.length; @@ -1254,12 +1094,16 @@ mcpBridgeShardTest("hermes")( expectedSecret: HOST_SECRET, label: "Hermes MCP rediscovery after explicit restart", }; - await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + await assertTrustedPrivateMcpRebindingDenied(host, sandbox, cleanup, { adapter: "hermes-config", artifactPrefix: "hermes", + assertSecretAbsent: assertSecretAbsentFromSandbox, + cleanupBridge: cleanupMcpBridge, + mutationTimeoutMs: MCP_MUTATION_TIMEOUT_MS["hermes-config"], sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], survivingMcpUrl: mcpUrl, + progress, }); await assertHermesToolCall("hermes-real-mcp-tool-call-after-dns-rebinding-remove"); const survivingDiscoveryOffset = fakeMcp.requests.length; @@ -1418,12 +1262,16 @@ mcpBridgeShardTest("deepagents")( }); await assertDeepAgentsConfig(sandbox, DEEPAGENTS_SANDBOX_NAME, mcpUrl); await assertSecretAbsentFromSandbox(sandbox, DEEPAGENTS_SANDBOX_NAME, ["/sandbox/.deepagents"]); - await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + await assertTrustedPrivateMcpRebindingDenied(host, sandbox, cleanup, { adapter: "deepagents-config", artifactPrefix: "deepagents", + assertSecretAbsent: assertSecretAbsentFromSandbox, + cleanupBridge: cleanupMcpBridge, + mutationTimeoutMs: MCP_MUTATION_TIMEOUT_MS["deepagents-config"], sandboxName: DEEPAGENTS_SANDBOX_NAME, secretPaths: ["/sandbox/.deepagents"], survivingMcpUrl: mcpUrl, + progress, }); progress.phase("exercise lifecycle and confirm Deep Agents bridge removal"); await assertRealAdapterToolCall(sandbox, fakeMcp, { diff --git a/test/e2e/setup-mcp-test-tls.sh b/test/e2e/setup-mcp-test-tls.sh index 8c3c5c50d4..3651fedb4f 100755 --- a/test/e2e/setup-mcp-test-tls.sh +++ b/test/e2e/setup-mcp-test-tls.sh @@ -46,12 +46,13 @@ openssl x509 \ "subjectAltName=DNS:host.openshell.internal,DNS:mcp-rebind.example.test") \ -out "${tls_dir}/server.crt" -# The self-signed certificate secures only the loopback origin hop from -# cloudflared, which is launched with --no-tls-verify for that local fixture. -# Successful sandbox MCP connections use the public trycloudflare URL and its -# publicly trusted edge certificate. The direct DNS-rebinding fixture is denied -# by policy before TLS, so sandboxes never install or trust this private test CA. +# The self-signed certificate secures the loopback origin hop from cloudflared +# and the routed-private managed MCP fixture. The latter imports this bounded +# test CA through the normal corporate-CA onboarding path so all three managed +# agent runtimes can complete real TLS and MCP traffic before the DNS-rebinding +# denial proof. { + echo "NEMOCLAW_MCP_TLS_CA_CERT=${tls_dir}/ca.crt" echo "NEMOCLAW_MCP_TLS_CERT=${tls_dir}/server.crt" echo "NEMOCLAW_MCP_TLS_KEY=${tls_dir}/server.key" } >>"${GITHUB_ENV}" diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index b3b04464f2..dcbb9bcf13 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -62,6 +62,15 @@ describe("MCP bridge onboarding environment", () => { }); }); + it("passes the routed-private MCP test CA through the normal corporate CA input", () => { + const env = buildMcpBridgeOnboardEnv({ + ...ONBOARD_OPTIONS, + corporateCaBundle: "/tmp/nemoclaw-mcp-tls/ca.crt", + }); + + expect(env.NEMOCLAW_CORPORATE_CA_BUNDLE).toBe("/tmp/nemoclaw-mcp-tls/ca.crt"); + }); + it("rejects protected onboarding key collisions", () => { expect(() => buildMcpBridgeOnboardEnv({ From e24a3cf073e01e73b4fd660163e0d05dc8d7f7bc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 22:03:59 -0700 Subject: [PATCH 11/32] test(e2e): keep live MCP test linear --- test/e2e/live/mcp-bridge-onboard-env.ts | 8 ++++++++ test/e2e/live/mcp-bridge.test.ts | 11 ++++++----- test/e2e/support/mcp-bridge-onboard-env.test.ts | 8 ++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index aab4f763ed..22dc52f752 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -54,3 +54,11 @@ export function buildMcpBridgeOnboardEnv(options: { NEMOCLAW_RECREATE_SANDBOX: "1", }; } + +export function requireMcpBridgeTlsCaCert(env: NodeJS.ProcessEnv = process.env): string { + const corporateCaBundle = env.NEMOCLAW_MCP_TLS_CA_CERT; + if (!corporateCaBundle) { + throw new Error("NEMOCLAW_MCP_TLS_CA_CERT is required for routed-private MCP validation"); + } + return corporateCaBundle; +} diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 669d8c2817..45eb6927d3 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -28,7 +28,11 @@ import { assertHermesReloadRollback, assertHermesRemovalSurvivesGatewayRestart, } from "./mcp-bridge-hermes-lifecycle.ts"; -import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-bridge-onboard-env.ts"; +import { + buildMcpBridgeExactMainEnv, + buildMcpBridgeOnboardEnv, + requireMcpBridgeTlsCaCert, +} from "./mcp-bridge-onboard-env.ts"; import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts"; import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts"; import { @@ -108,10 +112,7 @@ async function onboardAgent( envOverlay?: NodeJS.ProcessEnv; }, ): Promise { - const corporateCaBundle = process.env.NEMOCLAW_MCP_TLS_CA_CERT; - if (!corporateCaBundle) { - throw new Error("NEMOCLAW_MCP_TLS_CA_CERT is required for routed-private MCP validation"); - } + const corporateCaBundle = requireMcpBridgeTlsCaCert(); cleanup.trackSandbox(host, options.sandboxName, { artifactName: "cleanup-destroy-sandbox", timeoutMs: 15 * 60_000, diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index dcbb9bcf13..480b97564f 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv, + requireMcpBridgeTlsCaCert, } from "../live/mcp-bridge-onboard-env.ts"; const ONBOARD_OPTIONS = { @@ -71,6 +72,13 @@ describe("MCP bridge onboarding environment", () => { expect(env.NEMOCLAW_CORPORATE_CA_BUNDLE).toBe("/tmp/nemoclaw-mcp-tls/ca.crt"); }); + it("requires the routed-private MCP test CA before onboarding", () => { + expect(requireMcpBridgeTlsCaCert({ NEMOCLAW_MCP_TLS_CA_CERT: "/tmp/ca.crt" })).toBe( + "/tmp/ca.crt", + ); + expect(() => requireMcpBridgeTlsCaCert({})).toThrow("NEMOCLAW_MCP_TLS_CA_CERT is required"); + }); + it("rejects protected onboarding key collisions", () => { expect(() => buildMcpBridgeOnboardEnv({ From 39c235b62e70cb10c6ce99cd98fd3e3399520a02 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 22:39:59 -0700 Subject: [PATCH 12/32] fix(mcp): trust private endpoint CAs in proxy --- Dockerfile | 10 +++++++--- agents/hermes/Dockerfile | 10 +++++++--- agents/langchain-deepagents-code/Dockerfile | 11 +++++++---- test/corporate-ca-dockerfile-decode.test.ts | 6 +++++- test/helpers/corporate-ca-support.ts | 6 ++++++ 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 33a31fefc7..4c02707aa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -259,15 +259,19 @@ ARG NEMOCLAW_CORPORATE_CA_B64 RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then \ command -v base64 >/dev/null 2>&1 || { echo "[nemoclaw] base64 is required to decode NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image" >&2; exit 1; }; \ command -v openssl >/dev/null 2>&1 || { echo "[nemoclaw] openssl is required to validate NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image (#6210)" >&2; exit 1; }; \ - mkdir -p /usr/local/share/nemoclaw \ + command -v update-ca-certificates >/dev/null 2>&1 || { echo "[nemoclaw] update-ca-certificates is required to anchor NEMOCLAW_CORPORATE_CA_B64 for the OpenShell proxy" >&2; exit 1; }; \ + case "${NEMOCLAW_CORPORATE_CA_B64}" in *[!A-Za-z0-9+/=]*) echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1 ;; esac; \ + mkdir -p /usr/local/share/nemoclaw /usr/local/share/ca-certificates \ && { printf '%s' "${NEMOCLAW_CORPORATE_CA_B64}" | base64 --decode > /tmp/nemoclaw-corporate-ca.decoded 2>/dev/null \ || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1; }; } \ && awk '/-----BEGIN CERTIFICATE-----/{f=1} f{print} /-----END CERTIFICATE-----/{f=0}' /tmp/nemoclaw-corporate-ca.decoded > /usr/local/share/nemoclaw/corporate-ca.pem \ && rm -f /tmp/nemoclaw-corporate-ca.decoded \ && { grep -qF -- "-----BEGIN CERTIFICATE-----" /usr/local/share/nemoclaw/corporate-ca.pem || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ && { openssl crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null 2>&1 || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ - && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem \ - && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem \ + && node -e 'const fs = require("node:fs"); const { X509Certificate } = require("node:crypto"); const pemPath = process.argv[1]; const anchorDir = process.argv[2]; const pem = fs.readFileSync(pemPath, "utf8"); const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g); if (!blocks?.length) process.exit(1); fs.writeFileSync(pemPath, blocks.map((block) => block.trim()).join("\n") + "\n"); blocks.forEach((block, index) => { if (!new X509Certificate(block).ca) process.exit(1); const name = anchorDir + "/nemoclaw-corporate-ca-" + String(index + 1).padStart(2, "0") + ".crt"; fs.writeFileSync(name, block.trim() + "\n"); });' /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates \ + && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && update-ca-certificates \ && echo "[nemoclaw] baked host corporate-proxy CA into image trust (#6210)"; \ fi diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 616f4e5ad9..d8f92edc20 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -676,15 +676,19 @@ RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-b RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then \ command -v base64 >/dev/null 2>&1 || { echo "[nemoclaw] base64 is required to decode NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image" >&2; exit 1; }; \ command -v openssl >/dev/null 2>&1 || { echo "[nemoclaw] openssl is required to validate NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image (#6210)" >&2; exit 1; }; \ - mkdir -p /usr/local/share/nemoclaw \ + command -v update-ca-certificates >/dev/null 2>&1 || { echo "[nemoclaw] update-ca-certificates is required to anchor NEMOCLAW_CORPORATE_CA_B64 for the OpenShell proxy" >&2; exit 1; }; \ + case "${NEMOCLAW_CORPORATE_CA_B64}" in *[!A-Za-z0-9+/=]*) echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1 ;; esac; \ + mkdir -p /usr/local/share/nemoclaw /usr/local/share/ca-certificates \ && { printf '%s' "${NEMOCLAW_CORPORATE_CA_B64}" | base64 --decode > /tmp/nemoclaw-corporate-ca.decoded 2>/dev/null \ || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1; }; } \ && awk '/-----BEGIN CERTIFICATE-----/{f=1} f{print} /-----END CERTIFICATE-----/{f=0}' /tmp/nemoclaw-corporate-ca.decoded > /usr/local/share/nemoclaw/corporate-ca.pem \ && rm -f /tmp/nemoclaw-corporate-ca.decoded \ && { grep -qF -- "-----BEGIN CERTIFICATE-----" /usr/local/share/nemoclaw/corporate-ca.pem || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ && { openssl crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null 2>&1 || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ - && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem \ - && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem \ + && node -e 'const fs = require("node:fs"); const { X509Certificate } = require("node:crypto"); const pemPath = process.argv[1]; const anchorDir = process.argv[2]; const pem = fs.readFileSync(pemPath, "utf8"); const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g); if (!blocks?.length) process.exit(1); fs.writeFileSync(pemPath, blocks.map((block) => block.trim()).join("\n") + "\n"); blocks.forEach((block, index) => { if (!new X509Certificate(block).ca) process.exit(1); const name = anchorDir + "/nemoclaw-corporate-ca-" + String(index + 1).padStart(2, "0") + ".crt"; fs.writeFileSync(name, block.trim() + "\n"); });' /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates \ + && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && update-ca-certificates \ && echo "[nemoclaw] baked host corporate-proxy CA into image trust (#6210)"; \ fi diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 2af5c6330c..959cfaeb84 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -110,15 +110,18 @@ ARG NEMOCLAW_CORPORATE_CA_B64 # hadolint ignore=DL3059,DL4006 RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then \ command -v base64 >/dev/null 2>&1 || { echo "[nemoclaw] base64 is required to decode NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image" >&2; exit 1; }; \ - install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ + command -v update-ca-certificates >/dev/null 2>&1 || { echo "[nemoclaw] update-ca-certificates is required to anchor NEMOCLAW_CORPORATE_CA_B64 for the OpenShell proxy" >&2; exit 1; }; \ + case "${NEMOCLAW_CORPORATE_CA_B64}" in *[!A-Za-z0-9+/=]*) echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1 ;; esac; \ + install -d -o root -g root -m 0755 /usr/local/share/nemoclaw /usr/local/share/ca-certificates \ && { printf '%s' "${NEMOCLAW_CORPORATE_CA_B64}" | base64 --decode > /tmp/nemoclaw-corporate-ca.decoded 2>/dev/null \ || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1; }; } \ && awk '/-----BEGIN CERTIFICATE-----/{f=1} f{print} /-----END CERTIFICATE-----/{f=0}' /tmp/nemoclaw-corporate-ca.decoded > /usr/local/share/nemoclaw/corporate-ca.pem \ && rm -f /tmp/nemoclaw-corporate-ca.decoded \ - && { node -e 'const fs = require("node:fs"); const { X509Certificate } = require("node:crypto"); const pem = fs.readFileSync(process.argv[1], "utf8"); const certificates = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g); if (!certificates?.length) process.exit(1); for (const certificate of certificates) if (!new X509Certificate(certificate).ca) process.exit(1);' /usr/local/share/nemoclaw/corporate-ca.pem \ + && { node -e 'const fs = require("node:fs"); const { X509Certificate } = require("node:crypto"); const pemPath = process.argv[1]; const anchorDir = process.argv[2]; const pem = fs.readFileSync(pemPath, "utf8"); const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g); if (!blocks?.length) process.exit(1); fs.writeFileSync(pemPath, blocks.map((block) => block.trim()).join("\n") + "\n"); blocks.forEach((block, index) => { if (!new X509Certificate(block).ca) process.exit(1); const name = anchorDir + "/nemoclaw-corporate-ca-" + String(index + 1).padStart(2, "0") + ".crt"; fs.writeFileSync(name, block.trim() + "\n"); });' /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates \ || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates with basicConstraints CA:TRUE (#6210)" >&2; exit 1; }; } \ - && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem \ - && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem \ + && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem /usr/local/share/ca-certificates/nemoclaw-corporate-ca-*.crt \ + && update-ca-certificates \ && echo "[nemoclaw] baked host corporate-proxy CA into DCode image trust (#6210)"; \ fi diff --git a/test/corporate-ca-dockerfile-decode.test.ts b/test/corporate-ca-dockerfile-decode.test.ts index 2cc6e2eec2..034741b955 100644 --- a/test/corporate-ca-dockerfile-decode.test.ts +++ b/test/corporate-ca-dockerfile-decode.test.ts @@ -152,12 +152,16 @@ for (const [label, dockerfile] of DOCKERFILES) { }); it("succeeds for a valid base64-encoded certificate", () => { + const dir = tmpDir(); const res = runDockerfileCorporateCaDecode( dockerfile, Buffer.from(CERT_PEM).toString("base64"), - tmpDir(), + dir, ); expect(res.status).toBe(0); + expect( + readFileSync(join(dir, "ca-certificates/nemoclaw-corporate-ca-01.crt"), "utf-8"), + ).toBe(CERT_PEM); }); it("strips trailing non-certificate content and bakes only the certificate", () => { diff --git a/test/helpers/corporate-ca-support.ts b/test/helpers/corporate-ca-support.ts index 4eebb6dc8c..7083907584 100644 --- a/test/helpers/corporate-ca-support.ts +++ b/test/helpers/corporate-ca-support.ts @@ -280,6 +280,12 @@ export function runDockerfileCorporateCaDecode( .join("\n") .replace(/^RUN /, "") .replaceAll("/usr/local/share/nemoclaw", outDir) + .replaceAll("/usr/local/share/ca-certificates", path.join(outDir, "ca-certificates")) + // The shipped block refreshes the image OS trust store. Keep this + // unprivileged extraction focused on the decode/split contract and prevent + // it from mutating the test host's trust store. + .replaceAll("command -v update-ca-certificates >/dev/null 2>&1", "true") + .replaceAll("&& update-ca-certificates \\", "&& true \\") // Redirect the fixed /tmp decode scratch path into the per-test dir so // concurrent test runs never collide. .replaceAll("/tmp/nemoclaw-corporate-ca.decoded", path.join(outDir, "decoded")) From a36d28932b08bf9de4ce38f809a0a8128526c84f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 22:44:04 -0700 Subject: [PATCH 13/32] docs(mcp): explain private endpoint CA trust --- docs/manage-sandboxes/add-mcp-server.mdx | 4 ++++ docs/reference/troubleshoot-mcp-servers.mdx | 2 ++ docs/security/configure-corporate-ca-trust.mdx | 7 +++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/manage-sandboxes/add-mcp-server.mdx b/docs/manage-sandboxes/add-mcp-server.mdx index 785f10f11c..e15dc0c3fe 100644 --- a/docs/manage-sandboxes/add-mcp-server.mdx +++ b/docs/manage-sandboxes/add-mcp-server.mdx @@ -61,6 +61,10 @@ Before registration, configure an HTTPS endpoint that meets these requirements: - A host firewall limits the listener to the OpenShell gateway or the required deployment subnet. - The endpoint exposes only the required MCP path and methods. +If the endpoint certificate chains to a private CA, provide that CA with `NEMOCLAW_CORPORATE_CA_BUNDLE` before onboarding. +Rebuild an existing sandbox after adding or changing the CA so both the OpenShell upstream proxy and sandbox clients receive the trust anchor. +For source validation and custom-image requirements, refer to [Configure Corporate CA Trust](../../security/configure-corporate-ca-trust). + For a host-local MCP process, keep the process bound to loopback when practical. Place an operator-managed HTTPS reverse proxy on one stable, routed private address. Bind the proxy to the exact private interface instead of every host interface. diff --git a/docs/reference/troubleshoot-mcp-servers.mdx b/docs/reference/troubleshoot-mcp-servers.mdx index 3da7c9c124..bfaa6a4ff6 100644 --- a/docs/reference/troubleshoot-mcp-servers.mdx +++ b/docs/reference/troubleshoot-mcp-servers.mdx @@ -193,6 +193,8 @@ A URL that the agent-native path accepts is therefore not registrable through ma For a host-local server, keep the MCP process on loopback when practical. Place an operator-managed HTTPS reverse proxy on a stable private address that the OpenShell gateway can route to. The certificate must match the original endpoint hostname and chain to a trust root available to the managed runtime. +For a private CA, set `NEMOCLAW_CORPORATE_CA_BUNDLE` before onboarding, or rebuild the existing sandbox after setting it. +The managed image installs that CA for both the OpenShell upstream proxy and sandbox TLS clients. Restrict the proxy listener to the OpenShell gateway or required deployment subnet with host firewall rules. Direct `127.0.0.1`, `::1`, and hostnames that resolve to loopback remain rejected. diff --git a/docs/security/configure-corporate-ca-trust.mdx b/docs/security/configure-corporate-ca-trust.mdx index 201d980846..df766fed47 100644 --- a/docs/security/configure-corporate-ca-trust.mdx +++ b/docs/security/configure-corporate-ca-trust.mdx @@ -30,6 +30,8 @@ Use `$$nemoclaw rebuild` after adding or changing the CA for an existing NemoClaw validates the selected bundle and bakes it into the sandbox image as `NEMOCLAW_CORPORATE_CA_B64`. The managed Dockerfile decodes it to the root-owned, read-only file `/usr/local/share/nemoclaw/corporate-ca.pem`. +It also installs each validated certificate as a separate operating-system trust anchor and refreshes the OS trust bundle. +This lets the in-sandbox OpenShell proxy validate TLS when it opens the upstream connection to an HTTPS inference, MCP, or custom-policy endpoint signed by that CA. @@ -50,7 +52,7 @@ This supports cold builds on hosts where the base image is not cached. -At runtime, NemoClaw appends the corporate CA to the OpenShell trust bundle instead of replacing it. +For sandbox child processes, NemoClaw also appends the corporate CA to the OpenShell client trust bundle instead of replacing it. It points `SSL_CERT_FILE`, `CURL_CA_BUNDLE`, `REQUESTS_CA_BUNDLE`, `GIT_SSL_CAINFO`, and `NODE_EXTRA_CA_CERTS` at the merged bundle so curl, Python, Git, and Node.js trust both roots. ## Understand Automatic Source Selection @@ -82,7 +84,8 @@ If no selection message appears, no source passed validation. ## Use a Custom Dockerfile Managed NemoClaw Dockerfiles implement the corporate CA build contract automatically. -A custom Dockerfile must declare `ARG NEMOCLAW_CORPORATE_CA_B64` and decode it into `/usr/local/share/nemoclaw/corporate-ca.pem` for runtime trust. +A custom Dockerfile must declare `ARG NEMOCLAW_CORPORATE_CA_B64`, validate and decode it into `/usr/local/share/nemoclaw/corporate-ca.pem`, install each certificate as an individual OS trust anchor, and refresh the OS trust bundle. +Decoding only the application bundle is insufficient for OpenShell-inspected HTTPS because the proxy verifies the upstream TLS connection itself. If its build needs network access behind the corporate proxy, establish build-time trust before the dependency operations that require TLS. Automatic fallback and host-store sources are a no-op when a custom Dockerfile omits that argument. From c0ecd8b4277e5b4a9d23d79bd8df7f453da05202 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 00:41:46 -0700 Subject: [PATCH 14/32] test(mcp): accept rejected public drift details --- test/e2e/live/mcp-bridge-trusted-private.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/mcp-bridge-trusted-private.ts b/test/e2e/live/mcp-bridge-trusted-private.ts index a828bef6af..1578788059 100644 --- a/test/e2e/live/mcp-bridge-trusted-private.ts +++ b/test/e2e/live/mcp-bridge-trusted-private.ts @@ -192,14 +192,16 @@ export async function assertTrustedPrivateMcpRebindingDenied( }, ); expectExitZero(driftStatus, `${options.artifactPrefix} reports trusted-private pin drift`); - expect(JSON.parse(driftStatus.stdout)).toMatchObject({ + const driftInspection = JSON.parse(driftStatus.stdout); + expect(driftInspection).toMatchObject({ trustedPrivateTarget: { host: REBIND_HOSTNAME, recordedPins: [trustedPrivateAddress], - currentPins: [REBIND_PUBLIC_IP], state: "drift", + detail: expect.stringContaining("did not resolve to supported routed private addresses"), }, }); + expect(driftInspection.trustedPrivateTarget.currentPins).toBeUndefined(); const denial = await sandbox.execShell( options.sandboxName, trustedSandboxShellScript( From 6d319becea566597c4d207a47cb532b20685d162 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 02:19:08 -0700 Subject: [PATCH 15/32] chore(ci): validate merged main From 605fe3fb4540965e57f34b31f618d1c95739939d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 02:30:57 -0700 Subject: [PATCH 16/32] chore(architecture): lower merged main budgets --- ci/source-architecture-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 4e3f1dacd4..dc2d40b910 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -27,7 +27,7 @@ "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, - "src/lib/state/registry.ts": 99, + "src/lib/state/registry.ts": 98, "src/lib/state/state-root.ts": 22, "src/lib/subprocess-env.ts": 24, "src/lib/validation.ts": 25 @@ -47,7 +47,7 @@ "src/lib/actions/uninstall/run-plan.ts": 26, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 21, - "src/lib/onboard.ts": 222, + "src/lib/onboard.ts": 219, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 22, "src/lib/shields/index.ts": 23 @@ -55,7 +55,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 308, + "src/lib/onboard": 307, "src/lib/actions": 19, "src/lib/actions/sandbox": 184, "src/lib/state": 37, From 1211a42842b48ab946e920996e57e0bc28444fd6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 04:29:08 -0700 Subject: [PATCH 17/32] chore(ci): retry hosted E2E port collision Signed-off-by: Aaron Erickson From 714131eab9ec02c1a97513b69f196e54af80da74 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 05:09:02 -0700 Subject: [PATCH 18/32] fix(ci): restore complete CLI build artifact Signed-off-by: Aaron Erickson --- .../restore-e2e-cli-artifact/action.yaml | 7 ++- .github/workflows/e2e.yaml | 5 +- .../cli-artifact-workflow-boundary.test.ts | 59 ++++++++++++++++++- tools/e2e/cli-artifact-workflow-boundary.mts | 5 ++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/.github/actions/restore-e2e-cli-artifact/action.yaml b/.github/actions/restore-e2e-cli-artifact/action.yaml index d3e7784e9b..b199170751 100644 --- a/.github/actions/restore-e2e-cli-artifact/action.yaml +++ b/.github/actions/restore-e2e-cli-artifact/action.yaml @@ -146,7 +146,7 @@ runs: { echo "::error::exact-commit CLI artifact payload digest mismatch"; exit 1; } while IFS= read -r member; do case "$member" in - dist | dist/*) ;; + dist | dist/* | nemoclaw | nemoclaw/dist | nemoclaw/dist/*) ;; *) echo "::error::CLI artifact contains an unsafe member: $member"; exit 1 ;; esac case "/$member/" in @@ -158,11 +158,15 @@ runs: { echo "::error::CLI artifact contains a link or special file"; exit 1; } [[ ! -e "$GITHUB_WORKSPACE/dist" && ! -L "$GITHUB_WORKSPACE/dist" ]] || { echo "::error::consumer unexpectedly built dist before artifact restore"; exit 1; } + [[ ! -e "$GITHUB_WORKSPACE/nemoclaw/dist" && ! -L "$GITHUB_WORKSPACE/nemoclaw/dist" ]] || + { echo "::error::consumer unexpectedly built nemoclaw/dist before artifact restore"; exit 1; } restore_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-cli-restore.XXXXXX")" trap 'rm -rf -- "$restore_dir"' EXIT tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir" test -s "$restore_dir/dist/nemoclaw.js" || { echo "::error::restored CLI artifact is missing dist/nemoclaw.js"; exit 1; } + test -s "$restore_dir/nemoclaw/dist/shared/sandbox-name.cjs" || + { echo "::error::restored CLI artifact is missing nemoclaw/dist/shared/sandbox-name.cjs"; exit 1; } jq -e --arg candidateSha "$CANDIDATE_SHA" ' type == "object" and (keys | sort) == ["nemoclawVersion", "sourceRevision"] and @@ -170,5 +174,6 @@ runs: .sourceRevision == $candidateSha ' "$restore_dir/dist/build-identity.json" >/dev/null || { echo "::error::restored CLI build identity does not match the candidate SHA"; exit 1; } + mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist" mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist" node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version >/dev/null diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8173c20f28..4adf2c4376 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -396,6 +396,8 @@ jobs: { echo "::error::candidate CLI build did not produce dist/nemoclaw.js"; exit 1; } test -s dist/build-identity.json || { echo "::error::candidate CLI build did not produce dist/build-identity.json"; exit 1; } + test -s nemoclaw/dist/shared/sandbox-name.cjs || + { echo "::error::candidate CLI build did not produce nemoclaw/dist/shared/sandbox-name.cjs"; exit 1; } jq -e --arg candidateSha "$CANDIDATE_SHA" ' type == "object" and (keys | sort) == ["nemoclawVersion", "sourceRevision"] and @@ -415,7 +417,8 @@ jobs: --group=0 \ --numeric-owner \ -cf "$payload" \ - dist + dist \ + nemoclaw/dist payload_sha256="$(sha256sum "$payload" | awk '{print $1}')" source_tree="$(git rev-parse 'HEAD^{tree}')" lockfile_sha256="$(sha256sum package-lock.json | awk '{print $1}')" diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index 1015e91df8..dd4b9e6ee5 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -67,11 +67,11 @@ function workflowFixture(): Workflow { } type RestoreFixtureOptions = { - archive?: "valid" | "non-dist" | "link" | "traversal"; + archive?: "valid" | "missing-shared" | "non-dist" | "link" | "traversal"; buildIdentitySha?: string; expectedPayloadSha256?: string; manifestCandidateSha?: string; - preexistingDist?: "dangling-symlink" | "directory"; + preexistingDist?: "dangling-symlink" | "directory" | "nested-directory"; }; type ArchiveFixtureContext = { @@ -87,6 +87,7 @@ function sha256File(file: string): string { function writeDistArchive( context: ArchiveFixtureContext, customizeDist: (dist: string) => void, + includeShared = true, ): void { const dist = path.join(context.payloadRoot, "dist"); fs.mkdirSync(dist); @@ -99,7 +100,14 @@ function writeDistArchive( })}\n`, ); customizeDist(dist); - execFileSync("tar", ["-cf", context.payload, "-C", context.payloadRoot, "dist"]); + const archiveMembers = ["dist"]; + if (includeShared) { + const shared = path.join(context.payloadRoot, "nemoclaw", "dist", "shared"); + fs.mkdirSync(shared, { recursive: true }); + fs.writeFileSync(path.join(shared, "sandbox-name.cjs"), "module.exports = {};\n"); + archiveMembers.push("nemoclaw/dist"); + } + execFileSync("tar", ["-cf", context.payload, "-C", context.payloadRoot, ...archiveMembers]); } function writeValidArchive(context: ArchiveFixtureContext): void { @@ -112,6 +120,10 @@ function writeLinkArchive(context: ArchiveFixtureContext): void { }); } +function writeMissingSharedArchive(context: ArchiveFixtureContext): void { + writeDistArchive(context, () => undefined, false); +} + function writeNonDistArchive(context: ArchiveFixtureContext): void { fs.writeFileSync(path.join(context.payloadRoot, "outside.txt"), "outside dist\n"); execFileSync("tar", ["-cf", context.payload, "-C", context.payloadRoot, "outside.txt"]); @@ -135,6 +147,7 @@ function writeTraversalArchive(context: ArchiveFixtureContext): void { const ARCHIVE_FIXTURE_WRITERS = { link: writeLinkArchive, + "missing-shared": writeMissingSharedArchive, "non-dist": writeNonDistArchive, traversal: writeTraversalArchive, valid: writeValidArchive, @@ -154,9 +167,16 @@ function writePreexistingDistDirectory(workspace: string): void { fs.writeFileSync(path.join(workspace, "dist", "existing.txt"), "preserve\n"); } +function writePreexistingNestedDistDirectory(workspace: string): void { + const dist = path.join(workspace, "nemoclaw", "dist"); + fs.mkdirSync(dist, { recursive: true }); + fs.writeFileSync(path.join(dist, "existing.txt"), "preserve\n"); +} + const PREEXISTING_DIST_WRITERS = { "dangling-symlink": writeDanglingDistSymlink, directory: writePreexistingDistDirectory, + "nested-directory": writePreexistingNestedDistDirectory, none: () => undefined, } satisfies Record< NonNullable | "none", @@ -171,6 +191,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { const payloadRoot = path.join(root, "payload-root"); const toolDirectory = path.join(root, "tools"); fs.mkdirSync(path.join(workspace, "bin"), { recursive: true }); + fs.mkdirSync(path.join(workspace, "nemoclaw"), { recursive: true }); fs.mkdirSync(artifactDirectory, { recursive: true }); fs.mkdirSync(payloadRoot, { recursive: true }); fs.mkdirSync(toolDirectory, { recursive: true }); @@ -180,6 +201,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { '#!/usr/bin/env node\nrequire("../dist/nemoclaw.js");\n', { mode: 0o755 }, ); + fs.writeFileSync(path.join(workspace, "nemoclaw", ".gitkeep"), ""); execFileSync("git", ["init", "--quiet"], { cwd: workspace }); execFileSync("git", ["add", "."], { cwd: workspace }); execFileSync( @@ -388,6 +410,11 @@ describe("exact-commit CLI artifact workflow boundary", () => { fs.readFileSync(path.join(fixture.workspace, "dist", "build-identity.json"), "utf8"), ), ).toEqual({ nemoclawVersion: "0.0.0", sourceRevision: fixture.candidateSha }); + expect( + fs.existsSync( + path.join(fixture.workspace, "nemoclaw", "dist", "shared", "sandbox-name.cjs"), + ), + ).toBe(true); expect( fs .readdirSync(fixture.runnerTemp) @@ -419,6 +446,13 @@ describe("exact-commit CLI artifact workflow boundary", () => { ); }); + it("rejects a payload missing the compiled shared dependency before activation (#7915)", () => { + expectRestoreFailure( + { archive: "missing-shared" }, + "restored CLI artifact is missing nemoclaw/dist/shared/sandbox-name.cjs", + ); + }); + it("rejects traversal through a dist-prefixed archive member before extraction (#7915)", () => { expectRestoreFailure({ archive: "traversal" }, "CLI artifact contains traversal"); }); @@ -452,6 +486,25 @@ describe("exact-commit CLI artifact workflow boundary", () => { } }); + it("does not overwrite a preexisting nemoclaw/dist directory (#7915)", () => { + const fixture = runRestoreValidation({ preexistingDist: "nested-directory" }); + try { + expect(fixture.result.status, fixture.output).not.toBe(0); + expect(fixture.output).toContain( + "consumer unexpectedly built nemoclaw/dist before artifact restore", + ); + expect( + fs.readFileSync( + path.join(fixture.workspace, "nemoclaw", "dist", "existing.txt"), + "utf8", + ), + ).toBe("preserve\n"); + expect(fs.existsSync(path.join(fixture.workspace, "dist"))).toBe(false); + } finally { + fixture.cleanup(); + } + }); + it("rejects a compiled identity mismatch before artifact activation (#7915)", () => { expectRestoreFailure( { buildIdentitySha: "e".repeat(40) }, diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 72272013c1..6e0122a2cb 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -164,9 +164,12 @@ export function validateCliArtifactRestoreAction( '*) echo "::error::CLI artifact contains an unsafe member', "CLI artifact contains a link or special file", '[[ ! -e "$GITHUB_WORKSPACE/dist" && ! -L "$GITHUB_WORKSPACE/dist" ]]', + '[[ ! -e "$GITHUB_WORKSPACE/nemoclaw/dist" && ! -L "$GITHUB_WORKSPACE/nemoclaw/dist" ]]', 'restore_dir="$(mktemp -d', 'tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir"', ".sourceRevision == $candidateSha", + 'test -s "$restore_dir/nemoclaw/dist/shared/sandbox-name.cjs"', + 'mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist"', 'mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist"', 'node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version', ]); @@ -224,10 +227,12 @@ function validateProducer(errors: string[], producer: WorkflowRecord): void { 'git rev-parse --verify HEAD)" == "$CANDIDATE_SHA"', "test -s dist/nemoclaw.js", "test -s dist/build-identity.json", + "test -s nemoclaw/dist/shared/sandbox-name.cjs", ".sourceRevision == $candidateSha", "candidate CLI build identity does not match the candidate commit SHA", "--sort=name", "--mtime=@0", + "nemoclaw/dist", "source_tree=\"$(git rev-parse 'HEAD^{tree}')\"", 'lockfile_sha256="$(sha256sum package-lock.json', 'artifact_name="nemoclaw-cli-${CANDIDATE_SHA}-${payload_sha256}"', From faeb93c5bbdb9d88125b44d96b7b4f97a61d2ec3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 05:09:48 -0700 Subject: [PATCH 19/32] ci(e2e): pin complete CLI artifact restore Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 124 +++++++++---------- tools/e2e/cli-artifact-workflow-boundary.mts | 4 +- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4adf2c4376..44bcc6f737 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -623,7 +623,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -801,7 +801,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -964,7 +964,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1015,7 +1015,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1289,7 +1289,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1421,7 +1421,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1543,7 +1543,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1655,7 +1655,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1738,7 +1738,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1783,7 +1783,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1862,7 +1862,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -1920,7 +1920,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Install OpenShell CLI @@ -2879,7 +2879,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Initialize runner comparison telemetry @@ -2953,7 +2953,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Install OpenShell CLI @@ -3040,7 +3040,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Initialize runner comparison telemetry @@ -3104,7 +3104,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3211,7 +3211,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3288,7 +3288,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3354,7 +3354,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3403,7 +3403,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3468,7 +3468,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3550,7 +3550,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3695,7 +3695,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3903,7 +3903,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3993,7 +3993,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4092,7 +4092,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4256,7 +4256,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4315,7 +4315,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4393,7 +4393,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4515,7 +4515,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4631,7 +4631,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4682,7 +4682,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4749,7 +4749,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4834,7 +4834,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4896,7 +4896,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -4959,7 +4959,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5021,7 +5021,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5215,7 +5215,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5313,7 +5313,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5374,7 +5374,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5436,7 +5436,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5496,7 +5496,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5555,7 +5555,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5671,7 +5671,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5729,7 +5729,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5841,7 +5841,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5944,7 +5944,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6010,7 +6010,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6082,7 +6082,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6157,7 +6157,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6236,7 +6236,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6305,7 +6305,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6401,7 +6401,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6483,7 +6483,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6538,7 +6538,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6601,7 +6601,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6678,7 +6678,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6778,7 +6778,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6871,7 +6871,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -6943,7 +6943,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -7008,7 +7008,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -7080,7 +7080,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 6e0122a2cb..63faabf7b0 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -19,7 +19,7 @@ export const CLI_ARTIFACT_DOWNLOAD_ACTION = export const CLI_ARTIFACT_UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; export const CLI_ARTIFACT_RESTORE_ACTION = - "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@4b911e40f3f41ce500d69330ca9280b79ceede0f"; + "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@714131eab9ec02c1a97513b69f196e54af80da74"; export const CLI_ARTIFACT_PACKAGE_STEP = "Package exact-commit CLI"; export const CLI_ARTIFACT_PUBLISH_STEP = "Publish content-addressed CLI artifact"; export const CLI_ARTIFACT_RESTORE_STEP = "Restore exact-commit CLI artifact"; @@ -33,7 +33,7 @@ const DEFAULT_RESTORE_ACTION_PATH = join( "action.yaml", ); const RESTORE_ACTION_CONTENT_SHA256 = - "bae2b6b99b46f7d007be9b792d7179fbef30c38d31623849e025d7341f179d0d"; + "89d52595cbe63a11c0ba8bb11930474c8ce605b3b3773ce007fcd82775c46683"; const CLI_ARTIFACT_DOWNLOAD_STEP = "Download exact-commit CLI artifact"; const CLI_ARTIFACT_VERIFY_STEP = "Verify and restore exact-commit CLI artifact"; const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; From 8998ef156a95d7992fa0e5ded352055a46f15765 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 05:19:48 -0700 Subject: [PATCH 20/32] test(ci): keep artifact fixtures branchless Signed-off-by: Aaron Erickson --- .../cli-artifact-workflow-boundary.test.ts | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index dd4b9e6ee5..a03c93ae77 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -84,10 +84,9 @@ function sha256File(file: string): string { return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); } -function writeDistArchive( +function writeDistTree( context: ArchiveFixtureContext, customizeDist: (dist: string) => void, - includeShared = true, ): void { const dist = path.join(context.payloadRoot, "dist"); fs.mkdirSync(dist); @@ -100,28 +99,40 @@ function writeDistArchive( })}\n`, ); customizeDist(dist); - const archiveMembers = ["dist"]; - if (includeShared) { - const shared = path.join(context.payloadRoot, "nemoclaw", "dist", "shared"); - fs.mkdirSync(shared, { recursive: true }); - fs.writeFileSync(path.join(shared, "sandbox-name.cjs"), "module.exports = {};\n"); - archiveMembers.push("nemoclaw/dist"); - } - execFileSync("tar", ["-cf", context.payload, "-C", context.payloadRoot, ...archiveMembers]); +} + +function writeSharedTree(context: ArchiveFixtureContext): void { + const shared = path.join(context.payloadRoot, "nemoclaw", "dist", "shared"); + fs.mkdirSync(shared, { recursive: true }); + fs.writeFileSync(path.join(shared, "sandbox-name.cjs"), "module.exports = {};\n"); +} + +function writeArchive(context: ArchiveFixtureContext, members: string[]): void { + execFileSync("tar", ["-cf", context.payload, "-C", context.payloadRoot, ...members]); +} + +function writeCompleteArchive( + context: ArchiveFixtureContext, + customizeDist: (dist: string) => void, +): void { + writeDistTree(context, customizeDist); + writeSharedTree(context); + writeArchive(context, ["dist", "nemoclaw/dist"]); } function writeValidArchive(context: ArchiveFixtureContext): void { - writeDistArchive(context, () => undefined); + writeCompleteArchive(context, () => undefined); } function writeLinkArchive(context: ArchiveFixtureContext): void { - writeDistArchive(context, (dist) => { + writeCompleteArchive(context, (dist) => { fs.symlinkSync("nemoclaw.js", path.join(dist, "linked-cli.js")); }); } function writeMissingSharedArchive(context: ArchiveFixtureContext): void { - writeDistArchive(context, () => undefined, false); + writeDistTree(context, () => undefined); + writeArchive(context, ["dist"]); } function writeNonDistArchive(context: ArchiveFixtureContext): void { @@ -494,10 +505,7 @@ describe("exact-commit CLI artifact workflow boundary", () => { "consumer unexpectedly built nemoclaw/dist before artifact restore", ); expect( - fs.readFileSync( - path.join(fixture.workspace, "nemoclaw", "dist", "existing.txt"), - "utf8", - ), + fs.readFileSync(path.join(fixture.workspace, "nemoclaw", "dist", "existing.txt"), "utf8"), ).toBe("preserve\n"); expect(fs.existsSync(path.join(fixture.workspace, "dist"))).toBe(false); } finally { From 5b2a9bb80906a419c8acc4cc4eaec8437f333008 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 06:28:56 -0700 Subject: [PATCH 21/32] test(snapshot): align private restore expectation Signed-off-by: Aaron Erickson --- .../snapshot-restore-observability-reconciliation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts b/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts index 09e3346710..a74ed9945b 100644 --- a/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-observability-reconciliation.test.ts @@ -41,7 +41,7 @@ describe("runSandboxSnapshot restore: observability policy reconciliation", () = "alpha", customPolicy.name, customPolicy.content, - { custom: { sourcePath: customPolicy.sourcePath } }, + { custom: { sourcePath: customPolicy.sourcePath }, nonFatal: true }, ); expect(consoleWarn.mock.calls.flat().join("\n")).toContain("private-api (apply failed)"); }); From 9998edab78e0c70636eba094a464498874a6e987 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 09:29:01 -0700 Subject: [PATCH 22/32] fix(hermes): recheck post-restore gateway health Signed-off-by: Aaron Erickson --- .../rebuild-hermes-post-restore.test.ts | 65 ++++++++++++++++++- .../sandbox/rebuild-hermes-post-restore.ts | 23 ++++--- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index e6aba20927..b0f3716811 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -1,12 +1,75 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createRebuildFlowHarness, resetRebuildFlowTestEnvironment, restoreRebuildFlowTestEnvironment, } from "../../../../test/helpers/rebuild-flow-harness"; +import { ensureHermesGatewayAfterStateRestore } from "./rebuild-hermes-post-restore"; + +describe("Hermes gateway post-restore recheck", () => { + it("accepts a gateway that becomes healthy after an inconclusive recovery check (#7084)", () => { + const checkAndRecoverSandboxProcesses = vi + .fn() + .mockReturnValueOnce({ + checked: false, + wasRunning: null, + recovered: false, + }) + .mockReturnValueOnce({ + checked: true, + wasRunning: true, + recovered: false, + }); + + expect( + ensureHermesGatewayAfterStateRestore("alpha", "hermes", { + checkAndRecoverSandboxProcesses, + }), + ).toBe("healthy"); + + expect(checkAndRecoverSandboxProcesses).toHaveBeenCalledTimes(2); + }); + + it.each([ + "forwardRecoveryFailed", + "secretBoundaryRefused", + "mcpReconciliationRefused", + ] as const)("fails immediately when recovery reports %s (#7084)", (failureFlag) => { + const checkAndRecoverSandboxProcesses = vi.fn(() => ({ + checked: true, + wasRunning: true, + recovered: false, + [failureFlag]: true, + })); + + expect( + ensureHermesGatewayAfterStateRestore("alpha", "hermes", { + checkAndRecoverSandboxProcesses, + }), + ).toBe("unverified"); + + expect(checkAndRecoverSandboxProcesses).toHaveBeenCalledOnce(); + }); + + it("fails closed after the bounded gateway recheck remains inconclusive (#7084)", () => { + const checkAndRecoverSandboxProcesses = vi.fn(() => ({ + checked: true, + wasRunning: false, + recovered: false, + })); + + expect( + ensureHermesGatewayAfterStateRestore("alpha", "hermes", { + checkAndRecoverSandboxProcesses, + }), + ).toBe("unverified"); + + expect(checkAndRecoverSandboxProcesses).toHaveBeenCalledTimes(2); + }); +}); describe("Hermes rebuild post-restore verification", () => { beforeEach(resetRebuildFlowTestEnvironment); diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts index d03284106f..5146ac599f 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -11,6 +11,7 @@ const RECEIPT_PREFIX = "NEMOCLAW_HERMES_CRON_RESTORE_V1:"; const BEGIN_TIMEOUT_MS = 70_000; const CONTROL_TIMEOUT_MS = 25_000; const RECOVERY_TIMEOUT_MS = BEGIN_TIMEOUT_MS + CONTROL_TIMEOUT_MS * 2 + 10_000; +const HERMES_GATEWAY_RECHECK_ATTEMPTS = 2; type HermesCronRestoreAction = "begin" | "validate" | "release" | "recover"; type HermesCronRestoreDisposition = @@ -91,17 +92,19 @@ export function ensureHermesGatewayAfterStateRestore( if (agentName !== "hermes") return "not-applicable"; const checkAndRecover = deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses; - const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); - if ( - !observation.checked || - observation.forwardRecoveryFailed === true || - observation.secretBoundaryRefused === true || - observation.mcpReconciliationRefused === true - ) { - return "unverified"; + for (let attempt = 1; attempt <= HERMES_GATEWAY_RECHECK_ATTEMPTS; attempt += 1) { + const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); + if ( + observation.forwardRecoveryFailed === true || + observation.secretBoundaryRefused === true || + observation.mcpReconciliationRefused === true + ) { + return "unverified"; + } + if (!observation.checked) continue; + if (observation.wasRunning === true) return "healthy"; + if (observation.recovered) return "recovered"; } - if (observation.wasRunning === true) return "healthy"; - if (observation.recovered) return "recovered"; return "unverified"; } From 5ee988d683f2f37b0f5dab3f28e729b420e9b909 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 09:52:18 -0700 Subject: [PATCH 23/32] test(hermes): accept flattened cron state Signed-off-by: Aaron Erickson --- test/e2e/live/rebuild-hermes-cron-restore.ts | 3 +- .../rebuild-hermes-cron-restore.test.ts | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/e2e/support/rebuild-hermes-cron-restore.test.ts diff --git a/test/e2e/live/rebuild-hermes-cron-restore.ts b/test/e2e/live/rebuild-hermes-cron-restore.ts index 96346d0c54..7aa1a53c77 100644 --- a/test/e2e/live/rebuild-hermes-cron-restore.ts +++ b/test/e2e/live/rebuild-hermes-cron-restore.ts @@ -121,7 +121,8 @@ function parseCronJob(text: string, jobId: string, label: string): JsonObject { ); } -function cronJobState(job: JsonObject, label: string): JsonObject { +export function cronJobState(job: JsonObject, label: string): JsonObject { + if (typeof job.state === "string") return job; return requireObject(job.state, `${label} state`); } diff --git a/test/e2e/support/rebuild-hermes-cron-restore.test.ts b/test/e2e/support/rebuild-hermes-cron-restore.test.ts new file mode 100644 index 0000000000..37a3225617 --- /dev/null +++ b/test/e2e/support/rebuild-hermes-cron-restore.test.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { cronJobState } from "../live/rebuild-hermes-cron-restore.ts"; + +describe("Hermes rebuild cron state compatibility", () => { + it("reads the historical nested cron state (#7806)", () => { + const state = { + last_run_at: null, + next_run_at: "2026-08-06T16:48:44.080564+00:00", + repeat: { completed: 0 }, + }; + + expect(cronJobState({ id: "job-1", state }, "cron job job-1")).toBe(state); + }); + + it("reads flattened cron state from the current Hermes runtime (#7806)", () => { + const job = { + id: "job-1", + last_run_at: null, + next_run_at: "2026-08-06T16:48:44.080564+00:00", + repeat: { completed: 0 }, + state: "scheduled", + }; + + expect(cronJobState(job, "cron job job-1")).toBe(job); + }); + + it("rejects cron state without either supported shape (#7806)", () => { + expect(() => cronJobState({ id: "job-1", state: null }, "cron job job-1")).toThrow( + "cron job job-1 state is not an object", + ); + }); +}); From 7a7160a3d9df81d835291e49b40b5f67e3e8078a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 10:17:28 -0700 Subject: [PATCH 24/32] test(hermes): preserve cron receipt redaction Signed-off-by: Aaron Erickson --- test/e2e/live/rebuild-hermes-cron-restore.ts | 48 +++++++++---------- .../rebuild-hermes-cron-restore.test.ts | 33 ++++++++++++- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/test/e2e/live/rebuild-hermes-cron-restore.ts b/test/e2e/live/rebuild-hermes-cron-restore.ts index 7aa1a53c77..2e5c3ab27c 100644 --- a/test/e2e/live/rebuild-hermes-cron-restore.ts +++ b/test/e2e/live/rebuild-hermes-cron-restore.ts @@ -59,7 +59,6 @@ interface CronControlReceipt { active_agents: number; disposition: string; drain_acquired: boolean; - drain_token: string; operator_drain_active: boolean; pid: number; start_time: number; @@ -86,6 +85,28 @@ function parseJsonObject(text: string, label: string): JsonObject { } } +export function parseHermesCronBeginReceipt(text: string): CronControlReceipt { + const lines = text.split(/\r?\n/u).filter((line) => line.startsWith(RECEIPT_PREFIX)); + if (lines.length !== 1) fail(`Hermes cron begin returned ${lines.length} receipts`); + const payload = parseJsonObject(lines[0].slice(RECEIPT_PREFIX.length), "cron begin receipt"); + expect(payload).toMatchObject({ + action: "begin", + active_agents: 0, + disposition: "drain-acquired", + drain_acquired: true, + operator_drain_active: false, + version: 1, + }); + if ( + !Number.isSafeInteger(payload.pid) || + !Number.isSafeInteger(payload.start_time) || + payload.drain_token !== "" + ) { + fail("Hermes cron begin receipt identity is invalid"); + } + return payload as unknown as CronControlReceipt; +} + function normalizeTimestampMs(value: unknown, label: string): number { if (typeof value === "number" && Number.isFinite(value)) { return value < 10_000_000_000 ? value * 1_000 : value; @@ -390,29 +411,6 @@ export function createRebuildHermesCronRestoreFixture({ fail(`Hermes gateway did not reach ${state}: ${lastEvidence}`); } - function parseBeginReceipt(text: string): CronControlReceipt { - const lines = text.split(/\r?\n/u).filter((line) => line.startsWith(RECEIPT_PREFIX)); - if (lines.length !== 1) fail(`Hermes cron begin returned ${lines.length} receipts`); - const payload = parseJsonObject(lines[0].slice(RECEIPT_PREFIX.length), "cron begin receipt"); - expect(payload).toMatchObject({ - action: "begin", - active_agents: 0, - disposition: "drain-acquired", - drain_acquired: true, - operator_drain_active: false, - version: 1, - }); - if ( - !Number.isSafeInteger(payload.pid) || - !Number.isSafeInteger(payload.start_time) || - typeof payload.drain_token !== "string" || - payload.drain_token.length !== 32 - ) { - fail("Hermes cron begin receipt identity is invalid"); - } - return payload as unknown as CronControlReceipt; - } - async function exerciseStateRootSubstitutionAttack( seed: SeededCronJob, expectedGateway: Pick, @@ -538,7 +536,7 @@ export function createRebuildHermesCronRestoreFixture({ "phase-8-acquire-stranded-hermes-cron-restore-gate", ); expectExitZero(begin, "acquire stranded Hermes cron restore gate"); - const receipt = parseBeginReceipt(begin.stdout); + const receipt = parseHermesCronBeginReceipt(begin.stdout); await assertControlMarker(true, "phase-8-verify-cron-restore-marker-before-restart"); const recoverySeed = await seedCronJob("recovery"); diff --git a/test/e2e/support/rebuild-hermes-cron-restore.test.ts b/test/e2e/support/rebuild-hermes-cron-restore.test.ts index 37a3225617..e766548ebd 100644 --- a/test/e2e/support/rebuild-hermes-cron-restore.test.ts +++ b/test/e2e/support/rebuild-hermes-cron-restore.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { cronJobState } from "../live/rebuild-hermes-cron-restore.ts"; +import { cronJobState, parseHermesCronBeginReceipt } from "../live/rebuild-hermes-cron-restore.ts"; describe("Hermes rebuild cron state compatibility", () => { it("reads the historical nested cron state (#7806)", () => { @@ -33,3 +33,34 @@ describe("Hermes rebuild cron state compatibility", () => { ); }); }); + +describe("Hermes rebuild cron begin receipt", () => { + const receipt = { + action: "begin", + active_agents: 0, + disposition: "drain-acquired", + drain_acquired: true, + drain_token: "", + operator_drain_active: false, + pid: 263, + start_time: 29_607, + version: 1, + }; + + it("accepts the canonically redacted ShellProbe receipt", () => { + expect( + parseHermesCronBeginReceipt(`NEMOCLAW_HERMES_CRON_RESTORE_V1:${JSON.stringify(receipt)}\n`), + ).toMatchObject({ pid: 263, start_time: 29_607 }); + }); + + it("rejects an unredacted drain token crossing the ShellProbe boundary", () => { + expect(() => + parseHermesCronBeginReceipt( + `NEMOCLAW_HERMES_CRON_RESTORE_V1:${JSON.stringify({ + ...receipt, + drain_token: "a".repeat(32), + })}\n`, + ), + ).toThrow("Hermes cron begin receipt identity is invalid"); + }); +}); From 5a5bb7006a7ee47f7c68f2bb429462e3de014f3e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 10:56:11 -0700 Subject: [PATCH 25/32] test(hermes): exercise scheduled cron drain Signed-off-by: Aaron Erickson --- test/e2e/live/rebuild-hermes-cron-restore.ts | 70 ++++++++++++++++++- .../rebuild-hermes-cron-restore.test.ts | 25 ++++++- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/rebuild-hermes-cron-restore.ts b/test/e2e/live/rebuild-hermes-cron-restore.ts index 2e5c3ab27c..97174304ae 100644 --- a/test/e2e/live/rebuild-hermes-cron-restore.ts +++ b/test/e2e/live/rebuild-hermes-cron-restore.ts @@ -14,6 +14,7 @@ import { buildHermesRuntimeExecArgs } from "./rebuild-hermes-runtime-exec.ts"; const HERMES_HOME = "/sandbox/.hermes"; const CRON_JOBS_FILE = `${HERMES_HOME}/cron/jobs.json`; const CRON_SCRIPTS_ROOT = `${HERMES_HOME}/scripts`; +const CRON_TICKER_LAST_SUCCESS = `${HERMES_HOME}/cron/ticker_last_success`; const CRON_CONTROL = "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py"; const NEMOCLAW_STATE_ROOT = "/sandbox/.nemoclaw"; const CRON_CONTROL_MARKER_NAME = "hermes-cron-restore-drain.json"; @@ -25,6 +26,15 @@ const EXECUTION_POLL_ATTEMPTS = 30; const POLL_INTERVAL_MS = 5_000; const SUBSTITUTION_PROBE_ATTEMPTS = 13; const RECEIPT_PREFIX = "NEMOCLAW_HERMES_CRON_RESTORE_V1:"; +const TRIGGER_CRON_JOB_PYTHON = [ + "import json", + "import sys", + "from cron.jobs import trigger_job", + "job = trigger_job(sys.argv[1])", + 'if not isinstance(job, dict) or job.get("id") != sys.argv[1]:', + ' raise SystemExit("cron job was not scheduled for the next ticker iteration")', + "print(json.dumps(job, sort_keys=True))", +].join("\n"); type JsonObject = Record; @@ -107,6 +117,14 @@ export function parseHermesCronBeginReceipt(text: string): CronControlReceipt { return payload as unknown as CronControlReceipt; } +export function parseCronTickerTimestamp(text: string, label: string): number { + const timestamp = Number(text.trim()); + if (!text.trim() || !Number.isFinite(timestamp) || timestamp < 0) { + fail(`${label} is invalid`); + } + return timestamp; +} + function normalizeTimestampMs(value: unknown, label: string): number { if (typeof value === "number" && Number.isFinite(value)) { return value < 10_000_000_000 ? value * 1_000 : value; @@ -322,6 +340,44 @@ export function createRebuildHermesCronRestoreFixture({ expectExitZero(run, `enqueue manual Hermes cron run ${seed.id}`); } + async function triggerCronJob(seed: SeededCronJob, artifactName: string): Promise { + const update = await dockerSandbox( + [HERMES_PYTHON, "-I", "-c", TRIGGER_CRON_JOB_PYTHON, seed.id], + artifactName, + ); + expectExitZero(update, `schedule Hermes cron job ${seed.id} for the next ticker iteration`); + const triggered = parseJsonObject(update.stdout, `${seed.name} trigger update`); + expect(triggered).toMatchObject({ + enabled: true, + id: seed.id, + state: "scheduled", + }); + expect( + normalizeTimestampMs(triggered.next_run_at, `${seed.name} trigger next_run_at`), + ).toBeLessThanOrEqual(Date.now() + POLL_INTERVAL_MS); + } + + async function tickerLastSuccess(artifactName: string): Promise { + const read = await dockerSandbox(["cat", CRON_TICKER_LAST_SUCCESS], artifactName); + expectExitZero(read, "read Hermes cron ticker last-success timestamp"); + return parseCronTickerTimestamp(read.stdout, "Hermes cron ticker last-success timestamp"); + } + + async function waitForTickerSuccessAfter( + baseline: number, + artifactPrefix: string, + ): Promise { + let lastSuccess = baseline; + for (let attempt = 1; attempt <= 15; attempt += 1) { + lastSuccess = await tickerLastSuccess(`${artifactPrefix}-attempt-${attempt}`); + if (lastSuccess > baseline) return lastSuccess; + await sleep(POLL_INTERVAL_MS); + } + return fail( + `Hermes cron ticker last-success timestamp did not advance: ${lastSuccess} <= ${baseline}`, + ); + } + async function waitForOneExecution(seed: SeededCronJob, artifactPrefix: string): Promise { let lastEvidence = "no scheduler evidence"; for (let attempt = 1; attempt <= EXECUTION_POLL_ATTEMPTS; attempt += 1) { @@ -540,8 +596,14 @@ export function createRebuildHermesCronRestoreFixture({ await assertControlMarker(true, "phase-8-verify-cron-restore-marker-before-restart"); const recoverySeed = await seedCronJob("recovery"); - await enqueueManualRun(recoverySeed, "phase-8-queue-due-hermes-cron-during-drain"); - await sleep(POLL_INTERVAL_MS); + await triggerCronJob(recoverySeed, "phase-8-trigger-hermes-cron-during-drain"); + const tickerSuccessBeforeRestart = await tickerLastSuccess( + "phase-8-read-cron-ticker-last-success-before-restart", + ); + const drainedTickerSuccess = await waitForTickerSuccessAfter( + tickerSuccessBeforeRestart, + "phase-8-wait-drained-cron-ticker-before-restart", + ); await assertExecutionMarkerAbsent( recoverySeed, "phase-8-verify-due-cron-blocked-before-restart", @@ -565,6 +627,10 @@ export function createRebuildHermesCronRestoreFixture({ "phase-8-verify-gateway-redrained-after-restart", beforeRestart, ); + await waitForTickerSuccessAfter( + drainedTickerSuccess, + "phase-8-wait-drained-cron-ticker-after-restart", + ); await assertControlMarker(true, "phase-8-verify-cron-restore-marker-after-restart"); await assertExecutionMarkerAbsent( recoverySeed, diff --git a/test/e2e/support/rebuild-hermes-cron-restore.test.ts b/test/e2e/support/rebuild-hermes-cron-restore.test.ts index e766548ebd..44f6530bae 100644 --- a/test/e2e/support/rebuild-hermes-cron-restore.test.ts +++ b/test/e2e/support/rebuild-hermes-cron-restore.test.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { cronJobState, parseHermesCronBeginReceipt } from "../live/rebuild-hermes-cron-restore.ts"; +import { + cronJobState, + parseCronTickerTimestamp, + parseHermesCronBeginReceipt, +} from "../live/rebuild-hermes-cron-restore.ts"; describe("Hermes rebuild cron state compatibility", () => { it("reads the historical nested cron state (#7806)", () => { @@ -64,3 +68,22 @@ describe("Hermes rebuild cron begin receipt", () => { ).toThrow("Hermes cron begin receipt identity is invalid"); }); }); + +describe("Hermes rebuild cron ticker timestamp", () => { + it("parses the ticker epoch", () => { + expect(parseCronTickerTimestamp("1785951799.098\n", "ticker timestamp")).toBe( + 1_785_951_799.098, + ); + }); + + it.each([ + "", + "not-an-epoch\n", + "Infinity\n", + "-1\n", + ])("rejects malformed ticker evidence %j", (evidence) => { + expect(() => parseCronTickerTimestamp(evidence, "ticker timestamp")).toThrow( + "ticker timestamp is invalid", + ); + }); +}); From 2b9515433e5738315af3df4995d28d7d4ec6f033 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 11:25:55 -0700 Subject: [PATCH 26/32] fix(hermes): stabilize scheduler and integrity probes Signed-off-by: Aaron Erickson --- .../mcp-bridge-hermes-reconciliation.test.ts | 30 +++++++++++++++++++ .../mcp-bridge-hermes-reconciliation.ts | 14 ++++++++- test/e2e/live/rebuild-hermes-cron-restore.ts | 12 ++++++-- .../rebuild-hermes-cron-restore.test.ts | 4 +++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts index 0eaf53a045..beafd68161 100644 --- a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts @@ -129,6 +129,36 @@ describe("Hermes MCP host reconciliation", () => { ); }); + it("retries a raced integrity snapshot with a fresh inspection", () => { + mocks.runOpenshellProviderCommand + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: "refusing raced Hermes MCP integrity snapshot", + }) + .mockReturnValueOnce({ + status: 0, + stdout: '{"ok":true,"state":"matched"}\n', + stderr: "", + }); + + expect(() => assertHermesMcpRuntimeIntent("alpha")).not.toThrow(); + expect(mocks.runOpenshellProviderCommand).toHaveBeenCalledTimes(2); + }); + + it("bounds raced integrity snapshot retries and still fails closed", () => { + mocks.runOpenshellProviderCommand.mockReturnValue({ + status: 1, + stdout: "", + stderr: "refusing raced Hermes MCP integrity snapshot", + }); + + expect(() => assertHermesMcpRuntimeIntent("alpha")).toThrow( + /refusing raced Hermes MCP integrity snapshot/, + ); + expect(mocks.runOpenshellProviderCommand).toHaveBeenCalledTimes(3); + }); + it("sanitizes thrown helper failures before returning or throwing them", () => { process.env.GITHUB_TOKEN = "host-only-secret"; mocks.runOpenshellProviderCommand.mockImplementation(() => { diff --git a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts index c95c19a2ea..951f116407 100644 --- a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts @@ -14,6 +14,8 @@ const HERMES_MCP_INSPECT_TIMEOUT_SECONDS = 45; const HERMES_MCP_INSPECT_TIMEOUT_MS = 60_000; const HERMES_MCP_RECONCILIATION_FAILURE = "Hermes MCP runtime does not match the persisted managed intent"; +const HERMES_MCP_RACED_SNAPSHOT_DETAIL = "refusing raced Hermes MCP integrity snapshot"; +const HERMES_MCP_RACED_SNAPSHOT_ATTEMPTS = 3; const ANSI_OR_UNSAFE_CONTROL_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])|[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/g; const DISPLAY_LINE_BREAK_RE = /[\r\n\u2028\u2029]+/g; @@ -186,7 +188,17 @@ export function assertHermesMcpRuntimeIntent( sandboxName: string, options: HermesMcpReconciliationOptions = {}, ): void { - const inspection = inspectHermesMcpRuntimeIntent(sandboxName, options); + let inspection = inspectHermesMcpRuntimeIntent(sandboxName, options); + for ( + let attempt = 1; + !inspection.ok && + inspection.state === "error" && + inspection.detail.includes(HERMES_MCP_RACED_SNAPSHOT_DETAIL) && + attempt < HERMES_MCP_RACED_SNAPSHOT_ATTEMPTS; + attempt += 1 + ) { + inspection = inspectHermesMcpRuntimeIntent(sandboxName, options); + } if (inspection.ok) return; throw new McpBridgeError( `${sanitizeHermesMcpReconciliationDetail( diff --git a/test/e2e/live/rebuild-hermes-cron-restore.ts b/test/e2e/live/rebuild-hermes-cron-restore.ts index 97174304ae..8a3a642446 100644 --- a/test/e2e/live/rebuild-hermes-cron-restore.ts +++ b/test/e2e/live/rebuild-hermes-cron-restore.ts @@ -14,7 +14,6 @@ import { buildHermesRuntimeExecArgs } from "./rebuild-hermes-runtime-exec.ts"; const HERMES_HOME = "/sandbox/.hermes"; const CRON_JOBS_FILE = `${HERMES_HOME}/cron/jobs.json`; const CRON_SCRIPTS_ROOT = `${HERMES_HOME}/scripts`; -const CRON_TICKER_LAST_SUCCESS = `${HERMES_HOME}/cron/ticker_last_success`; const CRON_CONTROL = "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py"; const NEMOCLAW_STATE_ROOT = "/sandbox/.nemoclaw"; const CRON_CONTROL_MARKER_NAME = "hermes-cron-restore-drain.json"; @@ -26,6 +25,12 @@ const EXECUTION_POLL_ATTEMPTS = 30; const POLL_INTERVAL_MS = 5_000; const SUBSTITUTION_PROBE_ATTEMPTS = 13; const RECEIPT_PREFIX = "NEMOCLAW_HERMES_CRON_RESTORE_V1:"; +const READ_CRON_TICKER_LAST_SUCCESS_PYTHON = [ + "from pathlib import Path", + "from cron.jobs import TICKER_SUCCESS_FILE", + "path = Path(TICKER_SUCCESS_FILE)", + 'print(path.read_text(encoding="utf-8").strip() if path.is_file() else "0")', +].join("\n"); const TRIGGER_CRON_JOB_PYTHON = [ "import json", "import sys", @@ -358,7 +363,10 @@ export function createRebuildHermesCronRestoreFixture({ } async function tickerLastSuccess(artifactName: string): Promise { - const read = await dockerSandbox(["cat", CRON_TICKER_LAST_SUCCESS], artifactName); + const read = await dockerSandbox( + [HERMES_PYTHON, "-I", "-c", READ_CRON_TICKER_LAST_SUCCESS_PYTHON], + artifactName, + ); expectExitZero(read, "read Hermes cron ticker last-success timestamp"); return parseCronTickerTimestamp(read.stdout, "Hermes cron ticker last-success timestamp"); } diff --git a/test/e2e/support/rebuild-hermes-cron-restore.test.ts b/test/e2e/support/rebuild-hermes-cron-restore.test.ts index 44f6530bae..79eb148049 100644 --- a/test/e2e/support/rebuild-hermes-cron-restore.test.ts +++ b/test/e2e/support/rebuild-hermes-cron-restore.test.ts @@ -70,6 +70,10 @@ describe("Hermes rebuild cron begin receipt", () => { }); describe("Hermes rebuild cron ticker timestamp", () => { + it("accepts the initial missing-file sentinel", () => { + expect(parseCronTickerTimestamp("0\n", "ticker timestamp")).toBe(0); + }); + it("parses the ticker epoch", () => { expect(parseCronTickerTimestamp("1785951799.098\n", "ticker timestamp")).toBe( 1_785_951_799.098, From 613f93355cf9f063738b8620ffb31e1d537baf6a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 11:51:54 -0700 Subject: [PATCH 27/32] test(hermes): tolerate restart pid gap Signed-off-by: Aaron Erickson --- test/e2e/live/rebuild-hermes-cron-restore.ts | 25 ++++++++------- .../rebuild-hermes-cron-restore.test.ts | 31 +++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/test/e2e/live/rebuild-hermes-cron-restore.ts b/test/e2e/live/rebuild-hermes-cron-restore.ts index 8a3a642446..06853834c9 100644 --- a/test/e2e/live/rebuild-hermes-cron-restore.ts +++ b/test/e2e/live/rebuild-hermes-cron-restore.ts @@ -61,11 +61,11 @@ interface SeededCronJob { scriptName: string; } -interface GatewayEvidence { +export interface GatewayEvidence { active_agents: number; gateway_state: string; pid: number; - running_pid: number; + running_pid: number | null; start_time: number; } @@ -130,6 +130,18 @@ export function parseCronTickerTimestamp(text: string, label: string): number { return timestamp; } +export function parseGatewayEvidence(text: string): GatewayEvidence { + const payload = parseJsonObject(text, "Hermes gateway status"); + for (const field of ["active_agents", "pid", "start_time"] as const) { + if (!Number.isSafeInteger(payload[field])) fail(`Hermes gateway ${field} is invalid`); + } + if (payload.running_pid !== null && !Number.isSafeInteger(payload.running_pid)) { + fail("Hermes gateway running_pid is invalid"); + } + if (typeof payload.gateway_state !== "string") fail("Hermes gateway state is invalid"); + return payload as unknown as GatewayEvidence; +} + function normalizeTimestampMs(value: unknown, label: string): number { if (typeof value === "number" && Number.isFinite(value)) { return value < 10_000_000_000 ? value * 1_000 : value; @@ -412,15 +424,6 @@ export function createRebuildHermesCronRestoreFixture({ if (present) expect(result.stdout.trim()).toMatch(/^0:0 400 [1-9]\d*$/u); } - function parseGatewayEvidence(text: string): GatewayEvidence { - const payload = parseJsonObject(text, "Hermes gateway status"); - for (const field of ["active_agents", "pid", "running_pid", "start_time"] as const) { - if (!Number.isSafeInteger(payload[field])) fail(`Hermes gateway ${field} is invalid`); - } - if (typeof payload.gateway_state !== "string") fail("Hermes gateway state is invalid"); - return payload as unknown as GatewayEvidence; - } - async function gatewayEvidence(artifactName: string): Promise { const script = [ "import json", diff --git a/test/e2e/support/rebuild-hermes-cron-restore.test.ts b/test/e2e/support/rebuild-hermes-cron-restore.test.ts index 79eb148049..13cc8a1369 100644 --- a/test/e2e/support/rebuild-hermes-cron-restore.test.ts +++ b/test/e2e/support/rebuild-hermes-cron-restore.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { cronJobState, parseCronTickerTimestamp, + parseGatewayEvidence, parseHermesCronBeginReceipt, } from "../live/rebuild-hermes-cron-restore.ts"; @@ -91,3 +92,33 @@ describe("Hermes rebuild cron ticker timestamp", () => { ); }); }); + +describe("Hermes rebuild gateway evidence", () => { + it("accepts a transient missing running process during restart", () => { + expect( + parseGatewayEvidence( + JSON.stringify({ + active_agents: 0, + gateway_state: "draining", + pid: 263, + running_pid: null, + start_time: 29_607, + }), + ), + ).toMatchObject({ pid: 263, running_pid: null }); + }); + + it("rejects malformed running process evidence", () => { + expect(() => + parseGatewayEvidence( + JSON.stringify({ + active_agents: 0, + gateway_state: "draining", + pid: 263, + running_pid: "263", + start_time: 29_607, + }), + ), + ).toThrow("Hermes gateway running_pid is invalid"); + }); +}); From f28ac2b50221e184409899efbd05926c1e423877 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 15:44:27 -0700 Subject: [PATCH 28/32] fix(mcp): retry provider detach conflicts Signed-off-by: Aaron Erickson --- .../mcp-bridge-provider-attachments.ts | 84 ++++++++----- test/mcp-provider-detach-retry.test.ts | 118 ++++++++++++++++++ 2 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 test/mcp-provider-detach-retry.test.ts diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts b/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts index dd44f1c1e8..c49f9afd66 100644 --- a/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts +++ b/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts @@ -105,6 +105,17 @@ export function providerDetachChangedState(status: number | null, output: string export type ProviderDetachOutcome = "detached" | "absent" | "unknown"; +const MCP_PROVIDER_DETACH_ATTEMPTS = 2; + +function isRetryableSandboxMutationConflict(status: number | null, output: string): boolean { + return ( + status !== 0 && + /Failed to detach provider:\s*sandbox was modified by another operation\.\s*Please retry the command\.?/i.test( + stripAnsi(output), + ) + ); +} + export function detachProvider( sandboxName: string, entry: McpBridgeEntry, @@ -118,43 +129,50 @@ export function detachProvider( `MCP server '${entry.server}' has no recorded provider ID for prechecked detach.`, ); } - const before = exactAttachment(sandboxName, entry); - if (!before.inspection.attachments) { - if (options.bestEffort) return "unknown"; - throw new McpBridgeError( - before.inspection.error ?? `Could not inspect provider attachment '${entry.providerName}'.`, - ); - } - if (!before.attachment) return "absent"; - if ( - before.attachment.providerId !== entry.providerId || - before.attachment.credentialKeys.length !== 1 || - before.attachment.credentialKeys[0] !== entry.env[0] - ) { + for (let attempt = 0; attempt < MCP_PROVIDER_DETACH_ATTEMPTS; attempt += 1) { + const before = exactAttachment(sandboxName, entry); + if (!before.inspection.attachments) { + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + before.inspection.error ?? `Could not inspect provider attachment '${entry.providerName}'.`, + ); + } + if (!before.attachment) return "absent"; + if (!attachmentMatchesCurrentProviderSnapshot(before.attachment, entry)) { + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + `Provider attachment '${entry.providerName}' does not match MCP server '${entry.server}'. Expected stable provider ID '${entry.providerId}', found '${before.attachment.providerId ?? "missing"}', with credential keys '${before.attachment.credentialKeys.join(", ") || "none"}'.`, + ); + } + const result = runOpenshellProviderCommand( + ["sandbox", "provider", "detach", sandboxName, entry.providerName], + { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + } as Record, + ) as OpenShellCommandResult; + const output = commandOutput(result); + const after = exactAttachment(sandboxName, entry); + if (after.inspection.attachments && !after.attachment) { + return providerDetachChangedState(result.status, output) ? "detached" : "absent"; + } + if ( + attempt + 1 < MCP_PROVIDER_DETACH_ATTEMPTS && + after.inspection.attachments && + attachmentMatchesCurrentProviderSnapshot(after.attachment, entry) && + isRetryableSandboxMutationConflict(result.status, output) + ) { + continue; + } if (options.bestEffort) return "unknown"; throw new McpBridgeError( - `Provider attachment '${entry.providerName}' does not match MCP server '${entry.server}'. Expected stable provider ID '${entry.providerId}', found '${before.attachment.providerId ?? "missing"}', with credential keys '${before.attachment.credentialKeys.join(", ") || "none"}'.`, + output || + after.inspection.error || + `OpenShell did not confirm removal of provider attachment '${entry.providerName}'.`, ); } - const result = runOpenshellProviderCommand( - ["sandbox", "provider", "detach", sandboxName, entry.providerName], - { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - suppressOutput: true, - } as Record, - ) as OpenShellCommandResult; - const output = commandOutput(result); - const after = exactAttachment(sandboxName, entry); - if (after.inspection.attachments && !after.attachment) { - return providerDetachChangedState(result.status, output) ? "detached" : "absent"; - } - if (options.bestEffort) return "unknown"; - throw new McpBridgeError( - output || - after.inspection.error || - `OpenShell did not confirm removal of provider attachment '${entry.providerName}'.`, - ); + return "unknown"; } /** diff --git a/test/mcp-provider-detach-retry.test.ts b/test/mcp-provider-detach-retry.test.ts new file mode 100644 index 0000000000..b151e9bd27 --- /dev/null +++ b/test/mcp-provider-detach-retry.test.ts @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +type DetachScenario = "success" | "drift" | "exhausted" | "other-error"; + +function runDetachScenario(scenario: DetachScenario) { + const script = String.raw` +const scenario = ${JSON.stringify(scenario)}; +const providerCommands = require("./src/lib/adapters/openshell/provider-command.js"); +const expectedId = "11111111-2222-4333-8444-555555555555"; +const foreignId = "99999999-8888-4777-8666-555555555555"; +let attached = true; +let liveId = expectedId; +let detachCalls = 0; +providerCommands.runOpenshellProviderCommand = (args) => { + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return attached + ? { status: 0, stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nalpha-mcp-fake generic 1 0\n", stderr: "" } + : { status: 0, stdout: "No providers attached to sandbox alpha.\n", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: " + liveId + "\nType: generic\nResource version: 4\nCredential keys: EXPECTED_TOKEN\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + detachCalls += 1; + if (scenario === "other-error") { + return { status: 1, stdout: "", stderr: "Failed to detach provider: permission denied" }; + } + if (detachCalls === 1 || scenario === "exhausted") { + if (scenario === "drift") liveId = foreignId; + return { + status: 1, + stdout: "", + stderr: "Failed to detach provider: sandbox was modified by another operation. Please retry the command.", + }; + } + attached = false; + return { status: 0, stdout: "Detached provider alpha-mcp-fake from sandbox alpha.", stderr: "" }; + } + throw new Error("unexpected call: " + args.join(" ")); +}; +const providerActions = require("./src/lib/actions/sandbox/mcp-bridge-provider.js"); +const entry = { + server: "fake", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["EXPECTED_TOKEN"], + providerName: "alpha-mcp-fake", + providerId: expectedId, + policyName: "mcp-bridge-fake", + addedAt: "2026-06-01T00:00:00.000Z", +}; +let outcome = null; +let message = null; +try { + outcome = providerActions.detachProvider("alpha", entry); +} catch (error) { + message = error.message; +} +process.stdout.write(JSON.stringify({ outcome, message, detachCalls, attached, liveId })); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + return JSON.parse(result.stdout) as { + outcome: string | null; + message: string | null; + detachCalls: number; + attached: boolean; + liveId: string; + }; +} + +describe("MCP provider detach retry", () => { + it("retries one exact OpenShell sandbox mutation conflict", () => { + expect(runDetachScenario("success")).toMatchObject({ + outcome: "detached", + message: null, + detachCalls: 2, + attached: false, + }); + }); + + it("refuses to retry when the attachment identity drifts", () => { + const result = runDetachScenario("drift"); + expect(result.outcome).toBeNull(); + expect(result.message).toContain("sandbox was modified by another operation"); + expect(result.detachCalls).toBe(1); + expect(result.attached).toBe(true); + }); + + it("bounds repeated sandbox mutation conflicts", () => { + const result = runDetachScenario("exhausted"); + expect(result.outcome).toBeNull(); + expect(result.message).toContain("sandbox was modified by another operation"); + expect(result.detachCalls).toBe(2); + expect(result.attached).toBe(true); + }); + + it("does not retry unrelated detach failures", () => { + const result = runDetachScenario("other-error"); + expect(result.outcome).toBeNull(); + expect(result.message).toContain("permission denied"); + expect(result.detachCalls).toBe(1); + expect(result.attached).toBe(true); + }); +}); From 59476e287c789f30d33e038ac163d30dc4870be0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 17:03:14 -0700 Subject: [PATCH 29/32] fix(ci): scale PR E2E evidence budget Signed-off-by: Aaron Erickson --- test/pr-e2e-gate-evidence-limits.test.ts | 15 +++++++++++++++ tools/e2e/pr-e2e-gate.mts | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/pr-e2e-gate-evidence-limits.test.ts diff --git a/test/pr-e2e-gate-evidence-limits.test.ts b/test/pr-e2e-gate-evidence-limits.test.ts new file mode 100644 index 0000000000..40b31f4088 --- /dev/null +++ b/test/pr-e2e-gate-evidence-limits.test.ts @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { evidenceEntryLimit } from "../tools/e2e/pr-e2e-gate.mts"; + +describe("PR E2E gate evidence traversal limits", () => { + it("scales the entry budget with expected signals under a hard ceiling", () => { + expect(evidenceEntryLimit(1)).toBe(1_280); + expect(evidenceEntryLimit(28)).toBe(8_192); + expect(evidenceEntryLimit(1_000)).toBe(16_384); + expect(() => evidenceEntryLimit(0)).toThrow(/signal count is invalid/u); + expect(() => evidenceEntryLimit(Number.NaN)).toThrow(/signal count is invalid/u); + }); +}); diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index ca02bdb9b9..23cdc6de71 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -139,7 +139,9 @@ const EVIDENCE_DOWNLOAD_TIMEOUT_MS = 10 * 60_000; const EVIDENCE_DOWNLOAD_KILL_GRACE_MS = 30_000; const EVIDENCE_LIMITS = { maxDepth: 8, - maxEntries: 4096, + baseEntries: 1024, + entriesPerExpectedSignal: 256, + maxEntries: 16_384, } as const; type ControllerPaths = { @@ -3634,6 +3636,16 @@ export function findSignalFiles( return files.sort((left, right) => left.localeCompare(right)); } +export function evidenceEntryLimit(expectedSignalCount: number): number { + if (!Number.isSafeInteger(expectedSignalCount) || expectedSignalCount < 1) { + throw new Error("E2E expected signal count is invalid"); + } + return Math.min( + EVIDENCE_LIMITS.maxEntries, + EVIDENCE_LIMITS.baseEntries + expectedSignalCount * EVIDENCE_LIMITS.entriesPerExpectedSignal, + ); +} + export async function finishPrGate(options: { statePath: string; stateHash: string; @@ -3737,7 +3749,8 @@ export async function finishPrGate(options: { throw error; } const signals = findSignalFiles(options.evidencePath, { - ...EVIDENCE_LIMITS, + maxDepth: EVIDENCE_LIMITS.maxDepth, + maxEntries: evidenceEntryLimit(expectedSignalCount), maxSignalFiles: expectedSignalCount + 1, }).map((file) => validateSignal(readRegularJson(file), state)); verdict = classifyPrGateEvidence({ From f8cf0f7490819ac9c5a350ab3d07d541bbd3be02 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 19:38:58 -0700 Subject: [PATCH 30/32] fix(ci): compact successful E2E evidence Signed-off-by: Aaron Erickson --- test/e2e-risk-signal-reporter.test.ts | 56 +++++++++++++++++++++++++++ test/e2e/risk-signal-reporter.ts | 31 +++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index d591da3cef..ceb8a5fe33 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -20,6 +20,7 @@ import { configuredEnvironment, default as E2eRiskSignalReporter, outcomeForRun, + pruneEmptyCommandArtifacts, RISK_SIGNAL_FILE, type RiskSignalEnvironment, writeRiskSignal, @@ -93,6 +94,61 @@ describe("E2E risk signal reporter", () => { expect(configuredEnvironment({})).toBeNull(); }); + it("prunes only empty command transcripts", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-prune-")); + const nested = path.join(dir, "shell"); + const outside = path.join(dir, "outside.txt"); + const linked = path.join(nested, "linked.stderr.txt"); + try { + fs.mkdirSync(nested); + fs.writeFileSync(path.join(nested, "empty.stdout.txt"), ""); + fs.writeFileSync(path.join(nested, "empty.stderr.txt"), ""); + fs.writeFileSync(path.join(nested, "nonempty.stdout.txt"), "diagnostic\n"); + fs.writeFileSync(path.join(nested, "result.json"), ""); + fs.writeFileSync(outside, ""); + fs.linkSync(outside, linked); + + expect(pruneEmptyCommandArtifacts(dir)).toBe(2); + expect(fs.existsSync(path.join(nested, "empty.stdout.txt"))).toBe(false); + expect(fs.existsSync(path.join(nested, "empty.stderr.txt"))).toBe(false); + expect(fs.readFileSync(path.join(nested, "nonempty.stdout.txt"), "utf8")).toBe( + "diagnostic\n", + ); + expect(fs.existsSync(path.join(nested, "result.json"))).toBe(true); + expect(fs.existsSync(linked)).toBe(true); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("prunes empty command transcripts only after a passing run", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-run-prune-")); + const shellDir = path.join(dir, "shell"); + const emptyTranscript = path.join(shellDir, "command.stderr.txt"); + try { + fs.mkdirSync(shellDir); + vi.stubEnv("E2E_ARTIFACT_DIR", dir); + vi.stubEnv("E2E_TARGET_ID", "onboard-resume"); + vi.stubEnv("NEMOCLAW_E2E_EXPECTED_SHA", EXPECTED_SHA); + vi.stubEnv("NEMOCLAW_E2E_PLAN_HASH", PLAN_HASH); + vi.stubEnv("NEMOCLAW_E2E_CORRELATION_ID", CORRELATION_ID); + vi.stubEnv("NEMOCLAW_E2E_SHARD", "default"); + + fs.writeFileSync(emptyTranscript, ""); + new E2eRiskSignalReporter().onTestRunEnd( + [moduleWithFailedError(new Error("failed"))], + [], + "failed", + ); + expect(fs.existsSync(emptyTranscript)).toBe(true); + + new E2eRiskSignalReporter().onTestRunEnd([moduleWithStates(["passed"])], [], "passed"); + expect(fs.existsSync(emptyTranscript)).toBe(false); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("fails closed when run metadata is incomplete", () => { expect(() => configuredEnvironment({ NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA })).toThrow( /E2E_ARTIFACT_DIR/u, diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 7afc30f3f7..e2acd668f2 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -121,6 +121,31 @@ function readPrevious(file: string): E2eRiskSignal | null { return contents === null ? null : (JSON.parse(contents) as E2eRiskSignal); } +const EMPTY_COMMAND_ARTIFACT_PATTERN = /[.](?:stdout|stderr)[.]txt$/u; + +export function pruneEmptyCommandArtifacts(root: string): number { + if (!fs.existsSync(root)) return 0; + let removed = 0; + const pending = [root]; + while (pending.length > 0) { + const directory = pending.pop(); + if (!directory) break; + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) { + pending.push(file); + continue; + } + if (!entry.isFile() || !EMPTY_COMMAND_ARTIFACT_PATTERN.test(entry.name)) continue; + const metadata = fs.lstatSync(file); + if (metadata.size !== 0 || metadata.nlink !== 1) continue; + fs.unlinkSync(file); + removed += 1; + } + } + return removed; +} + export function writeRiskSignal( environment: RiskSignalEnvironment, testModules: ReadonlyArray, @@ -184,5 +209,11 @@ export default class E2eRiskSignalReporter implements Reporter { outcomeForRun(testModules, unhandledErrors, reason, this.processTimedOut), ); } + if ( + this.environment && + outcomeForRun(testModules, unhandledErrors, reason, this.processTimedOut) === "none" + ) { + pruneEmptyCommandArtifacts(this.environment.artifactDir); + } } } From 6f26510059516325ab8448cf5e057f28aca94782 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 5 Aug 2026 22:05:49 -0700 Subject: [PATCH 31/32] fix(inference): reconcile serving resolver changes Signed-off-by: Aaron Erickson --- src/lib/inference/serving/resolver.test.ts | 11 +++++--- src/lib/inference/serving/resolver.ts | 31 ---------------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/src/lib/inference/serving/resolver.test.ts b/src/lib/inference/serving/resolver.test.ts index f7db87263a..81d017087f 100644 --- a/src/lib/inference/serving/resolver.test.ts +++ b/src/lib/inference/serving/resolver.test.ts @@ -195,8 +195,9 @@ function readinessSources(): ManagedInferenceReadinessSource[] { function storageRemediableReadinessReport( extraFindings: SystemReadinessReport["findings"] = [], + preset: ManagedInferenceServingPreset = shippedPreset(), ): SystemReadinessReport { - const report = readinessReport(); + const report = readinessReport({}, preset); return { ...report, capabilities: [ @@ -826,12 +827,14 @@ describe("managed inference resolver", () => { it("admits a storage conflict that the public lifecycle can remediate (#8246)", () => { const catalog = hostLocalFixtureCatalog(); - const presetId = catalog.presets[0]!.metadata.id; + const preset = catalog.presets[0]!; const result = resolveManagedInferenceServing( { - readinessReports: [{ nodeId: "spark-head", report: storageRemediableReadinessReport() }], + readinessReports: [ + { nodeId: "spark-head", report: storageRemediableReadinessReport([], preset) }, + ], topologyQualifications: [], - intent: { preset: presetId }, + intent: { preset: preset.metadata.id }, now: NOW, }, catalog, diff --git a/src/lib/inference/serving/resolver.ts b/src/lib/inference/serving/resolver.ts index d7fb743ea2..f8f06d83d5 100644 --- a/src/lib/inference/serving/resolver.ts +++ b/src/lib/inference/serving/resolver.ts @@ -185,37 +185,6 @@ function matchesOperator(actual: unknown, operator: SelectionOperator, expected: } } -function versionAtLeast(actual: unknown, minimum: string): boolean { - if (typeof actual !== "string") return false; - const versionPattern = /^\d+(?:\.\d+)*$/u; - if (!versionPattern.test(actual) || !versionPattern.test(minimum)) return false; - const actualParts = actual.split(".").map(Number); - const minimumParts = minimum.split(".").map(Number); - const width = Math.max(actualParts.length, minimumParts.length); - for (let index = 0; index < width; index += 1) { - const actualPart = actualParts[index] ?? 0; - const minimumPart = minimumParts[index] ?? 0; - if (actualPart !== minimumPart) return actualPart > minimumPart; - } - return true; -} - -function readinessComparisonMatches( - actual: unknown, - comparison: ServingReadinessComparison, -): boolean { - switch (comparison.operator) { - case "equals": - return scalarEquals(actual, comparison.value); - case "one-of": - return comparison.values.some((candidate) => scalarEquals(actual, candidate)); - case "at-least": - return typeof actual === "number" && actual >= comparison.value; - case "version-at-least": - return versionAtLeast(actual, comparison.value); - } -} - function readinessScopeMatches( scope: string, reports: readonly ManagedInferenceReadinessSource[], From cd31190e24b4337c8fbe0505ed77ed7dd94217fb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 6 Aug 2026 00:50:32 -0700 Subject: [PATCH 32/32] fix(ci): remove obsolete PR gate evidence code Signed-off-by: Aaron Erickson --- test/e2e-risk-signal-reporter.test.ts | 56 ------------------------ test/e2e/risk-signal-reporter.ts | 31 ------------- test/pr-e2e-gate-evidence-limits.test.ts | 15 ------- 3 files changed, 102 deletions(-) delete mode 100644 test/pr-e2e-gate-evidence-limits.test.ts diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index ceb8a5fe33..d591da3cef 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -20,7 +20,6 @@ import { configuredEnvironment, default as E2eRiskSignalReporter, outcomeForRun, - pruneEmptyCommandArtifacts, RISK_SIGNAL_FILE, type RiskSignalEnvironment, writeRiskSignal, @@ -94,61 +93,6 @@ describe("E2E risk signal reporter", () => { expect(configuredEnvironment({})).toBeNull(); }); - it("prunes only empty command transcripts", () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-prune-")); - const nested = path.join(dir, "shell"); - const outside = path.join(dir, "outside.txt"); - const linked = path.join(nested, "linked.stderr.txt"); - try { - fs.mkdirSync(nested); - fs.writeFileSync(path.join(nested, "empty.stdout.txt"), ""); - fs.writeFileSync(path.join(nested, "empty.stderr.txt"), ""); - fs.writeFileSync(path.join(nested, "nonempty.stdout.txt"), "diagnostic\n"); - fs.writeFileSync(path.join(nested, "result.json"), ""); - fs.writeFileSync(outside, ""); - fs.linkSync(outside, linked); - - expect(pruneEmptyCommandArtifacts(dir)).toBe(2); - expect(fs.existsSync(path.join(nested, "empty.stdout.txt"))).toBe(false); - expect(fs.existsSync(path.join(nested, "empty.stderr.txt"))).toBe(false); - expect(fs.readFileSync(path.join(nested, "nonempty.stdout.txt"), "utf8")).toBe( - "diagnostic\n", - ); - expect(fs.existsSync(path.join(nested, "result.json"))).toBe(true); - expect(fs.existsSync(linked)).toBe(true); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } - }); - - it("prunes empty command transcripts only after a passing run", () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-run-prune-")); - const shellDir = path.join(dir, "shell"); - const emptyTranscript = path.join(shellDir, "command.stderr.txt"); - try { - fs.mkdirSync(shellDir); - vi.stubEnv("E2E_ARTIFACT_DIR", dir); - vi.stubEnv("E2E_TARGET_ID", "onboard-resume"); - vi.stubEnv("NEMOCLAW_E2E_EXPECTED_SHA", EXPECTED_SHA); - vi.stubEnv("NEMOCLAW_E2E_PLAN_HASH", PLAN_HASH); - vi.stubEnv("NEMOCLAW_E2E_CORRELATION_ID", CORRELATION_ID); - vi.stubEnv("NEMOCLAW_E2E_SHARD", "default"); - - fs.writeFileSync(emptyTranscript, ""); - new E2eRiskSignalReporter().onTestRunEnd( - [moduleWithFailedError(new Error("failed"))], - [], - "failed", - ); - expect(fs.existsSync(emptyTranscript)).toBe(true); - - new E2eRiskSignalReporter().onTestRunEnd([moduleWithStates(["passed"])], [], "passed"); - expect(fs.existsSync(emptyTranscript)).toBe(false); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } - }); - it("fails closed when run metadata is incomplete", () => { expect(() => configuredEnvironment({ NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA })).toThrow( /E2E_ARTIFACT_DIR/u, diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index e2acd668f2..7afc30f3f7 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -121,31 +121,6 @@ function readPrevious(file: string): E2eRiskSignal | null { return contents === null ? null : (JSON.parse(contents) as E2eRiskSignal); } -const EMPTY_COMMAND_ARTIFACT_PATTERN = /[.](?:stdout|stderr)[.]txt$/u; - -export function pruneEmptyCommandArtifacts(root: string): number { - if (!fs.existsSync(root)) return 0; - let removed = 0; - const pending = [root]; - while (pending.length > 0) { - const directory = pending.pop(); - if (!directory) break; - for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { - const file = path.join(directory, entry.name); - if (entry.isDirectory()) { - pending.push(file); - continue; - } - if (!entry.isFile() || !EMPTY_COMMAND_ARTIFACT_PATTERN.test(entry.name)) continue; - const metadata = fs.lstatSync(file); - if (metadata.size !== 0 || metadata.nlink !== 1) continue; - fs.unlinkSync(file); - removed += 1; - } - } - return removed; -} - export function writeRiskSignal( environment: RiskSignalEnvironment, testModules: ReadonlyArray, @@ -209,11 +184,5 @@ export default class E2eRiskSignalReporter implements Reporter { outcomeForRun(testModules, unhandledErrors, reason, this.processTimedOut), ); } - if ( - this.environment && - outcomeForRun(testModules, unhandledErrors, reason, this.processTimedOut) === "none" - ) { - pruneEmptyCommandArtifacts(this.environment.artifactDir); - } } } diff --git a/test/pr-e2e-gate-evidence-limits.test.ts b/test/pr-e2e-gate-evidence-limits.test.ts deleted file mode 100644 index 40b31f4088..0000000000 --- a/test/pr-e2e-gate-evidence-limits.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import { evidenceEntryLimit } from "../tools/e2e/pr-e2e-gate.mts"; - -describe("PR E2E gate evidence traversal limits", () => { - it("scales the entry budget with expected signals under a hard ceiling", () => { - expect(evidenceEntryLimit(1)).toBe(1_280); - expect(evidenceEntryLimit(28)).toBe(8_192); - expect(evidenceEntryLimit(1_000)).toBe(16_384); - expect(() => evidenceEntryLimit(0)).toThrow(/signal count is invalid/u); - expect(() => evidenceEntryLimit(Number.NaN)).toThrow(/signal count is invalid/u); - }); -});