diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 93001165e35..fe2af1fb589 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1288,12 +1288,35 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + - name: Install and verify cloudflared prerequisite + # Keep the public HTTPS routing fixture on the same reviewed binary as + # the MCP and tunnel-lifecycle lanes. The checksum and package metadata + # checks prevent a mutable package source from entering PR-safe E2E. + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + - name: Run inference routing live test - # Direct E2E coverage. The always-on PR-safe slices prove invalid-key, - # unreachable-endpoint, and localhost-compatible gateway routing - # without spending live provider quota. Credential-backed isolation - # and provider smokes live in inference-routing-provider-smoke.test.ts; - # any future secret-bearing lane must run that file from trusted main. + # Direct PR-safe E2E coverage proves invalid-key, unreachable-endpoint, + # localhost-compatible routing, namespace-aware HTTPS pinning, DNS + # rebinding resistance, private-target redirect rejection, and + # credential-bearing URL state rejection without live provider quota. + # Provider smokes live in inference-routing-provider-smoke.test.ts; any + # future secret-bearing lane must run that file from trusted main. run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/inference-routing.test.ts diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index fdd062b1c22..aed046f4cbb 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -27,6 +27,22 @@ "name": "NEMOCLAW_BEDROCK_RUNTIME_REGION", "reason": "Internal child-process setting used only to pass the resolved Bedrock Runtime region to the hidden local adapter. Users should rely on the endpoint URL or standard AWS region environment variables." }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + "reason": "Internal child-process setting used only when launching the hidden HTTPS Pin Runtime adapter. The port is a fixed internal constant, not a public user-facing configuration knob." + }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN", + "reason": "Internal host-only child-process secret used to authenticate HTTPS Pin Runtime adapter control-plane calls. It is generated by NemoClaw, stored in a private local state file, and never registered with OpenShell or supplied by users." + }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ALLOWED_SOURCE_CIDRS", + "reason": "Internal child-process setting carrying only the JSON-encoded OpenShell Docker IPAM subnets allowed to reach the hidden HTTPS Pin Runtime adapter. It contains no endpoint or credential data and is never user-set." + }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTES", + "reason": "Internal child-process setting carrying a JSON-encoded map of opaque route ids to provider types and non-secret token generations (no URLs or credentials) that a fresh HTTPS Pin Runtime adapter respawn could not recover, so it can authenticate and respond to them distinctly from a route that never existed. Never user-set." + }, { "name": "NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE", "reason": "Internal installer sentinel exported only during OpenShell gateway replacement so onboard restores the pre-upgrade sandbox backup. Not user-facing." diff --git a/docs/inference/custom-endpoint-security.mdx b/docs/inference/custom-endpoint-security.mdx index e8519400c17..21c935cbbbd 100644 --- a/docs/inference/custom-endpoint-security.mdx +++ b/docs/inference/custom-endpoint-security.mdx @@ -9,7 +9,6 @@ keywords: ["nemoclaw endpoint security", "inference endpoint ssrf", "custom endp content: type: "concept" --- - NemoClaw keeps provider credentials on the host and validates explicit custom endpoint URLs before saving them through security-sensitive configuration paths. ## Protect Provider Credentials @@ -44,9 +43,37 @@ Legacy entries without a source, endpoints recorded by `inference set`, and diff For a public HTTP URL, NemoClaw stores the validated IP address so the downstream runtime cannot resolve the hostname again and reach another address. -NemoClaw rejects DNS-backed HTTPS URLs in these paths because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary. -Use an HTTPS IP-literal endpoint with a certificate valid for that address. -Use a public HTTP endpoint only when your deployment permits unencrypted traffic. +NemoClaw rejects DNS-backed HTTPS URLs for Hermes Provider setup, host-side `config set`, and a direct blueprint run because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary. +For those paths, use an HTTPS IP-literal endpoint with a certificate valid for that address, or use a public HTTP endpoint if your deployment permits unencrypted traffic. + + + +## Use a DNS-Backed HTTPS Endpoint with `inference set` + +`$$nemoclaw inference set --endpoint-url ` on an already-onboarded sandbox supports a DNS-backed HTTPS custom endpoint through the HTTPS Pin Runtime adapter. +After SSRF validation passes, NemoClaw starts a local reverse-proxy adapter on the host that terminates a pinned, SNI-correct outbound TLS connection to the real upstream hostname, re-validating that the resolved peer IP is still public. +The sandbox, its OpenShell provider configuration and network policy, and the persisted sandbox registry only ever see the opaque local base `http://host.openshell.internal:/route/`. +The real upstream hostname and path never reach the sandbox or the persisted registry; host recovery state stores only the opaque route ID, provider type, a non-secret token generation value, and timestamps. +Endpoint URLs containing userinfo, a query string, or a fragment are rejected rather than stripped or persisted. +For an OpenAI-compatible endpoint entered as a bare origin, the adapter preserves the incoming `/v1` request path. +For an endpoint with a path prefix, the adapter keeps forwarded requests beneath that prefix and rejects traversal-shaped paths. +One 30-second total upstream deadline covers connection setup, TLS negotiation, and the complete response; the adapter closes a response that arrives after the deadline instead of relaying it. + +Each opaque route has its own sandbox-facing adapter credential, distinct from both the real upstream credential and the host-only control credential; a credential issued for one route cannot authorize another route. +NemoClaw does not place upstream route credentials in the adapter child-process environment. +After startup, the host CLI registers each route and its credential in adapter memory through an authenticated loopback-only control plane. +Before starting or reusing the adapter, NemoClaw inspects the exact IPAM subnets assigned to the `openshell-docker` network. +The adapter accepts route-forwarding and non-control health requests only from loopback or those inspected subnets and returns a not-found response to peers on other private or LAN networks. +Authenticated control health and route-registration requests remain loopback-only. +NemoClaw refuses to expose the adapter when it cannot discover a valid bridge subnet. +Adapter reuse also requires an authenticated health proof for the same source-subnet policy, so a running process with a stale or different policy is replaced. +After an adapter restart, routes other than the one that triggered recovery return a recovery-needed response until their original `inference set --endpoint-url` command is rerun. +Switching away from a route or destroying its last sandbox reference revokes it; a scoped uninstall that leaves sibling gateways in place preserves the shared adapter and its remaining routes. + +This support is specific to `inference set` on an already-onboarded sandbox. +Hermes Provider setup, host-side `config set`, and a direct blueprint run still reject DNS-backed HTTPS URLs as described above. + + ## Use the Sandbox Host Alias diff --git a/docs/inference/switch-providers.mdx b/docs/inference/switch-providers.mdx index ad3a7839839..281b0cbd555 100644 --- a/docs/inference/switch-providers.mdx +++ b/docs/inference/switch-providers.mdx @@ -98,6 +98,11 @@ An explicit different API family is rejected for that route. To point a sandbox at a different custom endpoint, re-run onboarding with the new endpoint. A rebuild reuses the recorded endpoint and cannot change it. +If updating an existing compatible provider fails after OpenShell selects the new route, NemoClaw attempts to restore the previously recorded provider and model. +The command still exits nonzero because the provider binding might be partially updated. +Retry the switch or re-run onboarding to reconcile the provider. +If NemoClaw reports that it could not restore the previous selection, do not use the route until you re-run onboarding. + ## Account for Shared Gateways diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index fc07770498b..c88088c3cd9 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3091,7 +3091,8 @@ NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, For a same-provider model change, pass `--endpoint-url` with the exact canonical endpoint URL that the target sandbox registry identifies as onboarding-established. Missing or `inference set` provenance and every different URL remain subject to the full address validation above. For public HTTP URLs, NemoClaw stores the validated IP address to prevent DNS rebinding. -DNS-backed HTTPS URLs are rejected because NemoClaw cannot pin the downstream peer address while preserving TLS SNI and host validation across the OpenShell runtime boundary; HTTPS IP-literal URLs remain supported. +For a DNS-backed HTTPS URL, NemoClaw routes the endpoint through a local HTTPS Pin Runtime adapter that terminates a pinned, SNI-correct outbound connection to the real upstream hostname; the sandbox and the persisted registry only ever see a local `host.openshell.internal` route, never the real hostname. +HTTPS IP-literal URLs remain supported and do not need the adapter. NemoClaw accepts `http://host.openshell.internal:` only with an explicit port from `1024` through `65535`; this narrow exception supports NemoClaw's sandbox-to-host inference routes and is not a general private-endpoint bypass. `--credential-env` may also be supplied for compatible provider metadata; supported `--inference-api` values are `openai-completions`, `anthropic-messages`, and `openai-responses`. @@ -3402,8 +3403,8 @@ Passthrough commands do not consume flags intended for the downstream command as If a port value is not a valid integer or falls outside the allowed range, the CLI exits with an error. -`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, or OpenRouter runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `11434`, `11435`, and `11437`. -When you select OpenRouter, `NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT` must also be distinct from the gateway, vLLM, Ollama, and Ollama proxy ports. +`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, OpenRouter runtime adapter, or HTTPS Pin Runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `11434`, `11435`, `11437`, and `11438`. +When you select OpenRouter, `NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT` must also be distinct from the gateway, vLLM, Ollama, Ollama proxy, and HTTPS Pin Runtime adapter ports. When you run multiple NemoClaw gateways with different `NEMOCLAW_GATEWAY_PORT` values, NemoClaw derives a separate gateway name, state directory, and compatibility container name from the port so one gateway does not tear down another. Only port `8080` uses a NemoClaw-managed Linux systemd user service or macOS Homebrew service. NemoClaw-managed gateways on custom ports run as detached processes and do not change the default gateway service. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 972d93027b4..f46f2c78141 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1994,7 +1994,8 @@ After the upgrade, recreate the sandbox with `$$nemoclaw onboard`. NemoClaw rejects an explicit custom endpoint when it resolves a public HTTPS hostname but cannot pin the same peer address across the downstream OpenShell runtime boundary while preserving TLS SNI and host validation. This can appear during a direct blueprint run, custom-endpoint onboarding, or a host-side `config set` write. -It can also appear during a runtime `$$nemoclaw inference set` switch. +It does not appear during a runtime `$$nemoclaw inference set` switch on an already-onboarded sandbox; that command routes a DNS-backed HTTPS endpoint through a local HTTPS Pin Runtime adapter instead of rejecting it. +Refer to [Commands](commands) for details. Use an HTTPS IP-literal endpoint whose certificate is valid for that address. diff --git a/nemoclaw/src/blueprint/ssrf.ts b/nemoclaw/src/blueprint/ssrf.ts index 4e86936d65b..791f4b89f45 100644 --- a/nemoclaw/src/blueprint/ssrf.ts +++ b/nemoclaw/src/blueprint/ssrf.ts @@ -112,11 +112,12 @@ export async function validateEndpointUrl(url: string): Promise&2 + 'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 } if [[ "${latest_version}" != "${pinned_version}" ]]; then diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index add41abe538..692b16aec79 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { ensureHttpsPinRuntimeAdapter as realEnsureHttpsPinRuntimeAdapter } from "../inference/https-pin-runtime-adapter"; import type { ConfigObject } from "../security/credential-filter"; import { runInferenceSet } from "./inference-set"; import { baseSession, createDeps } from "./inference-set.test-support"; @@ -177,6 +178,27 @@ describe("runInferenceSet compatible providers", () => { }); it("preserves explicit inference API through the final registry and session sync", async () => { + let providerVersion = 1; + const captureOpenshell = vi.fn((args: string[]) => { + switch (`${args[0]}:${args[1]}`) { + case "provider:get": { + const output = [ + "Name: compatible-endpoint", + "Id: 11111111-2222-4333-8444-555555555555", + "Type: openai", + `Resource version: ${providerVersion}`, + "Credential keys: COMPATIBLE_API_KEY", + "Config keys: OPENAI_BASE_URL", + ].join("\n"); + return { status: 0, output, stdout: output, stderr: "" }; + } + case "provider:update": + providerVersion += 1; + return { status: 0, output: "", stdout: "", stderr: "" }; + default: + return { status: 0, output: "", stdout: "", stderr: "" }; + } + }); const config: ConfigObject = { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, models: { providers: { inference: { api: "openai-completions", models: [] } } }, @@ -196,6 +218,7 @@ describe("runInferenceSet compatible providers", () => { credentialEnv: "NVIDIA_INFERENCE_API_KEY", preferredInferenceApi: "openai-completions", }), + captureOpenshell, }); await runInferenceSet( @@ -221,12 +244,18 @@ describe("runInferenceSet compatible providers", () => { }, }, }); + // The DNS-backed HTTPS endpoint is pinned via the HTTPS-pin runtime + // adapter, so the persisted endpointUrl is the adapter's local route base + // URL, not the raw operator-supplied hostname — mirroring the existing + // HTTP precedent of persisting the validated/pinned address. The + // The canonical provider key stays stable while its invocation-local + // value is replaced by the route-scoped adapter token. expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ "alpha", expect.objectContaining({ provider: "compatible-endpoint", model: "mock-responses-model", - endpointUrl: "https://compatible.example/v1", + endpointUrl: "http://host.openshell.internal:11438/route/test-route", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-responses", }), @@ -234,7 +263,7 @@ describe("runInferenceSet compatible providers", () => { expect(deps.getSession()).toMatchObject({ provider: "compatible-endpoint", model: "mock-responses-model", - endpointUrl: "https://compatible.example/v1", + endpointUrl: "http://host.openshell.internal:11438/route/test-route", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-responses", }); @@ -320,6 +349,12 @@ describe("runInferenceSet compatible providers", () => { }, rewriteConfigUrlsWithDnsPinning: (value) => actualConfig.rewriteConfigUrlsWithDnsPinning(value, lookup), + // DNS-backed HTTPS endpoints (the "DNS-private" case below) route + // through the HTTPS-pin runtime adapter instead of + // rewriteConfigUrlsWithDnsPinning, so its real SSRF preflight is + // exercised here too, with the same injected DNS lookup. + ensureHttpsPinRuntimeAdapter: (adapterOptions) => + realEnsureHttpsPinRuntimeAdapter({ ...adapterOptions, lookup }), }); await expect( diff --git a/src/lib/actions/inference-set-degraded-state.test.ts b/src/lib/actions/inference-set-degraded-state.test.ts index be11b3ce0cb..20ec3b27276 100644 --- a/src/lib/actions/inference-set-degraded-state.test.ts +++ b/src/lib/actions/inference-set-degraded-state.test.ts @@ -147,7 +147,7 @@ describe("runInferenceSet degraded state handling", () => { const options = { provider: "compatible-endpoint", model: "openai/gpt-5.4-mini", - endpointUrl: "https://compatible.example/v1", + endpointUrl: "http://host.openshell.internal:11434/v1", credentialEnv: "COMPATIBLE_API_KEY", inferenceApi: "openai-completions", noVerify: true, diff --git a/src/lib/actions/inference-set-endpoint-security.test.ts b/src/lib/actions/inference-set-endpoint-security.test.ts index 586481cae2a..f325e9c119a 100644 --- a/src/lib/actions/inference-set-endpoint-security.test.ts +++ b/src/lib/actions/inference-set-endpoint-security.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { rewriteConfigUrlsWithDnsPinning } from "../sandbox/config"; +import type { ConfigValue } from "../security/credential-filter"; import { normalizeCustomEndpointUrl } from "./inference-set"; describe("custom inference endpoint DNS pinning", () => { @@ -48,6 +49,21 @@ describe("custom inference endpoint DNS pinning", () => { expect(lookup).toHaveBeenCalledWith("public-endpoint.example", { all: true }); }); + it.each([ + ["userinfo", "https://user:secret@public-endpoint.example/v1"], + ["query", "https://public-endpoint.example/v1?api_key=secret"], + ["fragment", "https://public-endpoint.example/v1#secret"], + ])("rejects a source endpoint with %s instead of silently stripping it", async (_kind, endpointUrl) => { + const rewriteUrl = vi.fn(async (value: ConfigValue) => value); + const ensureAdapter = vi.fn(async () => "http://host.openshell.internal:11438/route/test"); + + await expect( + normalizeCustomEndpointUrl(endpointUrl, rewriteUrl, ensureAdapter), + ).rejects.toThrow("without userinfo, query, or fragment components"); + expect(rewriteUrl).not.toHaveBeenCalled(); + expect(ensureAdapter).not.toHaveBeenCalled(); + }); + it("fails closed for DNS-backed HTTPS endpoints until runtime-aware pinning exists", async () => { const lookup = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]); @@ -57,4 +73,23 @@ describe("custom inference endpoint DNS pinning", () => { ), ).rejects.toThrow(/DNS-backed HTTPS URLs are not supported/); }); + + it("adds the HTTPS Pin Runtime adapter hint only at the inference-set call site, not in the generic config validator's own message (#6141)", async () => { + const lookup = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]); + + // The generic validator (also used by plain `config set` for arbitrary + // fields) must not mention inference set or the adapter -- it has no way + // to know the field it's validating is an inference endpoint. + await expect( + rewriteConfigUrlsWithDnsPinning("https://public-endpoint.example/v1/", lookup), + ).rejects.toThrow(/^(?!.*(?:inference set|HTTPS Pin Runtime adapter)).*$/is); + + // normalizeCustomEndpointUrl is only ever called for `inference set + // --endpoint-url`, so it appends the adapter-specific hint itself. + await expect( + normalizeCustomEndpointUrl("https://public-endpoint.example/v1/", (value) => + rewriteConfigUrlsWithDnsPinning(value, lookup), + ), + ).rejects.toThrow(/HTTPS Pin Runtime adapter/); + }); }); diff --git a/src/lib/actions/inference-set-gateway-route-containment.test.ts b/src/lib/actions/inference-set-gateway-route-containment.test.ts index 56b20907e3e..96eb0585606 100644 --- a/src/lib/actions/inference-set-gateway-route-containment.test.ts +++ b/src/lib/actions/inference-set-gateway-route-containment.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; +import { HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV } from "../inference/https-pin-runtime"; import type { ConfigObject } from "../security/credential-filter"; import type { SandboxEntry } from "../state/registry"; import { runInferenceSet } from "./inference-set"; @@ -28,6 +29,7 @@ const entry = (name: string, overrides: Partial = {}): SandboxEntr describe("runtime shared gateway route containment", () => { afterEach(() => { vi.unstubAllEnvs(); + delete process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]; }); it("rejects an ambient gateway endpoint before OpenShell prep or state mutation", async () => { @@ -213,7 +215,7 @@ describe("runtime shared gateway route containment", () => { const peer = entry("late-peer", { provider: "compatible-endpoint", model: "custom/model", - endpointUrl: "https://peer.example.test/v1", + endpointUrl: "http://peer.example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }); @@ -224,13 +226,16 @@ describe("runtime shared gateway route containment", () => { .mockReturnValue({ sandboxes: [alpha, peer], defaultSandbox: "alpha" }); deps.listSandboxes = listSandboxes; + // HTTP (not HTTPS) so this test exercises rewriteUrlWithDnsPinning directly + // to create the async validation gap; DNS-backed HTTPS endpoints route + // through the HTTPS-pin runtime adapter instead. await expect( runInferenceSet( { provider: "compatible-endpoint", model: "custom/model", sandboxName: "alpha", - endpointUrl: "https://alpha.example.test/v1", + endpointUrl: "http://alpha.example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", inferenceApi: "openai-completions", }, @@ -279,8 +284,11 @@ describe("runtime shared gateway route containment", () => { }); it("catches a DNS change between the preliminary and finalized gateway route checks", async () => { - const firstEndpoint = "https://first.example.test/v1"; - const secondEndpoint = "https://second.example.test/v1"; + // HTTP (not HTTPS) so this test exercises rewriteUrlWithDnsPinning directly; + // DNS-backed HTTPS endpoints route through the HTTPS-pin runtime adapter + // instead (see inference-set-https-pin-runtime.test.ts). + const firstEndpoint = "http://first.example.test/v1"; + const secondEndpoint = "http://second.example.test/v1"; const customRoute = { provider: "compatible-endpoint", model: "custom/model", @@ -304,6 +312,7 @@ describe("runtime shared gateway route containment", () => { sandboxes: [alpha, peer], }); const rewriteUrlWithDnsPinning = vi.fn().mockResolvedValueOnce(secondEndpoint); + const ensureHttpsPinRuntimeAdapter = vi.fn(); await expect( finalizeInferenceSetRoute({ @@ -315,11 +324,159 @@ describe("runtime shared gateway route containment", () => { onboardEndpointUrl: null, getSandboxes: () => [alpha, peer], rewriteUrlWithDnsPinning, + ensureHttpsPinRuntimeAdapter, }), ).rejects.toThrow("custom-peer"); expect(rewriteUrlWithDnsPinning).toHaveBeenCalledOnce(); expect(rewriteUrlWithDnsPinning).toHaveBeenCalledWith(firstEndpoint); + expect(ensureHttpsPinRuntimeAdapter).not.toHaveBeenCalled(); + }); + + it("matches a same-gateway peer by deterministic adapter route without persisting the source hostname (#6141)", async () => { + const sourceEndpoint = "https://shared.example.test/v1"; + const alpha = entry("alpha"); + const prepare = (sandboxes: SandboxEntry[]) => + prepareInferenceSetRoute({ + entry: alpha, + sandboxName: alpha.name, + provider: "compatible-endpoint", + model: "custom/model", + customRoute: { + endpointUrl: sourceEndpoint, + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + session: null, + sandboxes, + }); + const first = prepare([alpha]); + const adapterBaseUrl = first.preliminaryExplicitMetadata?.endpointUrl; + expect(adapterBaseUrl).toMatch(/^http:\/\/host\.openshell\.internal:/); + const peer = entry("custom-peer", { + provider: "compatible-endpoint", + model: "custom/model", + endpointUrl: adapterBaseUrl, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + + const prepared = prepare([alpha, peer]); + const result = await finalizeInferenceSetRoute({ + prepared, + sandboxName: alpha.name, + provider: "compatible-endpoint", + model: "custom/model", + canReuseRecordedRoute: false, + onboardEndpointUrl: null, + getSandboxes: () => [alpha, peer], + rewriteUrlWithDnsPinning: vi.fn(async (value: unknown) => value as string), + ensureHttpsPinRuntimeAdapter: vi.fn(async () => ({ + baseUrl: adapterBaseUrl as string, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: "route-token", + routeId: "route-id", + })), + }); + + expect(result.registryMetadata.endpointUrl).toBe(adapterBaseUrl); + expect(JSON.stringify(result)).not.toContain("shared.example.test"); + }); + + it("keeps concurrent HTTPS-pin route bindings isolated without shared process.env staging (#6141)", async () => { + const firstEndpoint = "https://race-a.example.test/v1"; + const secondEndpoint = "https://race-b.example.test/v1"; + const routeFor = (endpointUrl: string) => ({ + provider: "compatible-endpoint", + model: "custom/model", + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + const alpha = entry("alpha", routeFor(firstEndpoint)); + const beta = entry("beta", { + ...routeFor(secondEndpoint), + gatewayName: "nemoclaw-9091", + gatewayPort: 9091, + }); + const preparedFor = (target: SandboxEntry, endpointUrl: string) => + prepareInferenceSetRoute({ + entry: target, + sandboxName: target.name, + provider: "compatible-endpoint", + model: "custom/model", + customRoute: { + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + session: null, + sandboxes: [alpha, beta], + }); + + let releaseA: () => void = () => {}; + const aGate = new Promise((resolve) => { + releaseA = resolve; + }); + const callOrder: string[] = []; + const adapterBehaviorByEndpoint: Record< + string, + () => Promise<{ baseUrl: string; credentialEnv: string; token: string; routeId: string }> + > = { + [firstEndpoint]: async () => { + callOrder.push("a-start"); + await aGate; + callOrder.push("a-end"); + return { + baseUrl: "http://host.openshell.internal:1/route/a", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: "token-a", + routeId: "route-a", + }; + }, + [secondEndpoint]: async () => { + callOrder.push("b-start"); + return { + baseUrl: "http://host.openshell.internal:1/route/b", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: "token-b", + routeId: "route-b", + }; + }, + }; + const ensureHttpsPinRuntimeAdapter = vi.fn(async (options: { endpointUrl: string }) => + adapterBehaviorByEndpoint[options.endpointUrl](), + ); + const rewriteUrlWithDnsPinning = vi.fn(async (value: unknown) => value as string); + + const finalize = (target: SandboxEntry, endpointUrl: string) => + finalizeInferenceSetRoute({ + prepared: preparedFor(target, endpointUrl), + sandboxName: target.name, + provider: "compatible-endpoint", + model: "custom/model", + canReuseRecordedRoute: false, + onboardEndpointUrl: null, + getSandboxes: () => [alpha, beta], + rewriteUrlWithDnsPinning, + ensureHttpsPinRuntimeAdapter, + }); + + const callA = finalize(alpha, firstEndpoint); + await vi.waitFor(() => expect(callOrder).toContain("a-start")); + const callB = finalize(beta, secondEndpoint); + + await vi.waitFor(() => expect(callOrder).toContain("b-start")); + + releaseA(); + const [resultA, resultB] = await Promise.all([callA, callB]); + + expect(callOrder).toEqual(["a-start", "b-start", "a-end"]); + expect(resultA.registryMetadata.endpointUrl).toBe("http://host.openshell.internal:1/route/a"); + expect(resultB.registryMetadata.endpointUrl).toBe("http://host.openshell.internal:1/route/b"); + expect(resultA.httpsPinProviderBinding?.token).toBe("token-a"); + expect(resultB.httpsPinProviderBinding?.token).toBe("token-b"); + expect(process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]).toBeUndefined(); }); it("blocks an incomplete legacy custom target even without a peer (#6315)", async () => { diff --git a/src/lib/actions/inference-set-https-pin-provider.test.ts b/src/lib/actions/inference-set-https-pin-provider.test.ts new file mode 100644 index 00000000000..755f6d7dbf8 --- /dev/null +++ b/src/lib/actions/inference-set-https-pin-provider.test.ts @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { InferenceSetDeps } from "./inference-set"; +import { __test, prepareHttpsPinProviderBinding } from "./inference-set-https-pin-provider"; +import type { HttpsPinProviderBinding } from "./inference-set-route-containment"; + +const PROVIDER_ID = "11111111-2222-4333-8444-555555555555"; +function binding(overrides: Partial = {}): HttpsPinProviderBinding { + return { + baseUrl: "http://host.openshell.internal:11438/route/route-a/v1", + credentialEnv: "COMPATIBLE_API_KEY", + token: "route-token-a", + routeId: "route-a", + providerType: "openai", + ...overrides, + }; +} + +function providerOutput(options: { + id?: string; + resourceVersion: number; + providerName?: string; + type?: string; + credentialKey?: string; + configKey?: string; +}): string { + return [ + `Name: ${options.providerName ?? "compatible-endpoint"}`, + `Id: ${options.id ?? PROVIDER_ID}`, + `Type: ${options.type ?? "openai"}`, + `Resource version: ${options.resourceVersion}`, + `Credential keys: ${options.credentialKey ?? "COMPATIBLE_API_KEY"}`, + `Config keys: ${options.configKey ?? "OPENAI_BASE_URL"}`, + ].join("\n"); +} + +function captureSequence( + results: Array<{ status: number; stdout?: string; stderr?: string; output?: string }>, +): InferenceSetDeps["captureOpenshell"] & ReturnType { + return vi.fn( + () => + results.shift() ?? + (() => { + throw new Error("unexpected OpenShell call"); + })(), + ) as InferenceSetDeps["captureOpenshell"] & ReturnType; +} + +describe("HTTPS-pin provider binding", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("updates an owned provider with only the route token in invocation-local env", () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const before = providerOutput({ resourceVersion: 4 }); + const after = providerOutput({ resourceVersion: 5 }); + const capture = captureSequence([ + { status: 0, stdout: before, stderr: "", output: before }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: after, stderr: "", output: after }, + ]); + + const mutation = prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }); + mutation.commit(); + + expect(capture.mock.calls[1]).toEqual([ + [ + "provider", + "update", + "-g", + "nemoclaw", + "compatible-endpoint", + "--credential", + "COMPATIBLE_API_KEY", + "--config", + "OPENAI_BASE_URL=http://host.openshell.internal:11438/route/route-a/v1", + ], + expect.objectContaining({ env: { COMPATIBLE_API_KEY: "route-token-a" } }), + ]); + expect(JSON.stringify(capture.mock.calls)).not.toContain("real-upstream-secret"); + expect(process.env.COMPATIBLE_API_KEY).toBe("real-upstream-secret"); + expect(JSON.stringify(binding())).not.toContain("real-upstream-secret"); + }); + + it("creates an absent provider and verifies its new identity", () => { + const after = providerOutput({ resourceVersion: 1 }); + const capture = captureSequence([ + { status: 1, stdout: "", stderr: "Provider 'compatible-endpoint' not found" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: after, stderr: "" }, + ]); + + expect(() => + prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }), + ).not.toThrow(); + expect(capture.mock.calls[1][0]).toContain("create"); + }); + + it("removes a newly created provider when the caller rolls back", () => { + const after = providerOutput({ resourceVersion: 1 }); + const capture = captureSequence([ + { status: 1, stdout: "", stderr: "Provider 'compatible-endpoint' not found" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: after, stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 1, stdout: "", stderr: "Provider 'compatible-endpoint' not found" }, + ]); + + const mutation = prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }); + mutation.rollback(); + + expect(mutation.action).toBe("create"); + expect(capture.mock.calls[3][0]).toEqual([ + "provider", + "delete", + "-g", + "nemoclaw", + "compatible-endpoint", + ]); + }); + + it.each([ + ["same resource version", PROVIDER_ID, 4], + ["delete and recreate", "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", 5], + ])("fails closed on update identity drift: %s", (_label, id, resourceVersion) => { + const capture = captureSequence([ + { status: 0, stdout: providerOutput({ resourceVersion: 4 }), stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: providerOutput({ id, resourceVersion }), stderr: "" }, + ]); + + expect(() => + prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }).commit(), + ).toThrow("may be partial"); + }); + + it("fails closed when provider metadata is malformed or foreign", () => { + const malformed = providerOutput({ resourceVersion: 4, credentialKey: "FOREIGN_TOKEN" }); + const capture = captureSequence([{ status: 0, stdout: malformed, stderr: "" }]); + + expect(() => + prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }), + ).toThrow("malformed, foreign"); + expect(capture).toHaveBeenCalledTimes(1); + }); + + it("treats a nonzero mutation as ambiguous and never infers success from post-state", () => { + const before = providerOutput({ resourceVersion: 4 }); + const after = providerOutput({ resourceVersion: 5 }); + const capture = captureSequence([ + { status: 0, stdout: before, stderr: "" }, + { status: 1, stdout: "", stderr: "transient failure" }, + { status: 0, stdout: after, stderr: "" }, + ]); + + expect(() => + prepareHttpsPinProviderBinding({ + gatewayName: "nemoclaw", + providerName: "compatible-endpoint", + binding: binding(), + captureOpenshell: capture, + }).commit(), + ).toThrow("may have partially applied"); + }); + + it("keeps route credentials isolated across independent invocations", () => { + const mutations: Array = []; + const makeCapture = (id: string): InferenceSetDeps["captureOpenshell"] => { + let version = 1; + return (args, opts) => { + switch (args[1]) { + case "get": { + const output = providerOutput({ id, resourceVersion: version }); + return { status: 0, stdout: output, stderr: "", output }; + } + default: + mutations.push(opts?.env); + version += 1; + return { status: 0, stdout: "", stderr: "", output: "" }; + } + }; + }; + + prepareHttpsPinProviderBinding({ + gatewayName: "gateway-a", + providerName: "compatible-endpoint", + binding: binding({ token: "route-token-a" }), + captureOpenshell: makeCapture("aaaaaaaa-2222-4333-8444-555555555555"), + }).commit(); + prepareHttpsPinProviderBinding({ + gatewayName: "gateway-b", + providerName: "compatible-endpoint", + binding: binding({ token: "route-token-b", routeId: "route-b" }), + captureOpenshell: makeCapture("bbbbbbbb-2222-4333-8444-555555555555"), + }).commit(); + + expect(mutations).toEqual([ + { COMPATIBLE_API_KEY: "route-token-a" }, + { COMPATIBLE_API_KEY: "route-token-b" }, + ]); + }); + + it("parses styled identity fields but rejects duplicates and invalid versions", () => { + expect( + __test.parseProviderVersion( + "\u001b[2mId:\u001b[0m 11111111-2222-4333-8444-555555555555\n\u001b[2mResource version:\u001b[0m 7", + ), + ).toEqual({ id: PROVIDER_ID, resourceVersion: 7 }); + expect( + __test.parseProviderVersion(`Id: ${PROVIDER_ID}\nId: ${PROVIDER_ID}\nResource version: 7`), + ).toBeNull(); + expect(__test.parseProviderVersion(`Id: ${PROVIDER_ID}\nResource version: 0`)).toBeNull(); + }); +}); diff --git a/src/lib/actions/inference-set-https-pin-provider.ts b/src/lib/actions/inference-set-https-pin-provider.ts new file mode 100644 index 00000000000..058b5487a0d --- /dev/null +++ b/src/lib/actions/inference-set-https-pin-provider.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type CaptureOpenshellResult, stripAnsi } from "../adapters/openshell/client"; +import { + matchesGatewayProviderBinding, + parseGatewayProviderMetadata, +} from "../onboard/gateway-provider-metadata"; +import { + InferenceSetError, + OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + openshellReportsProviderNotFound, +} from "./inference-set-error"; +import type { HttpsPinProviderBinding } from "./inference-set-route-containment"; + +type CaptureProviderCommand = ( + args: string[], + options: { + ignoreError: true; + includeStreams: true; + maxBuffer: number; + env?: NodeJS.ProcessEnv; + }, +) => CaptureOpenshellResult; + +type ProviderSurface = { + type: "openai" | "anthropic"; + configKey: "OPENAI_BASE_URL" | "ANTHROPIC_BASE_URL"; +}; + +type ProviderObservation = + | { kind: "absent" } + | { + kind: "present"; + id: string; + resourceVersion: number; + metadata: NonNullable>; + } + | { kind: "error"; status: number | null }; + +function providerSurface(binding: HttpsPinProviderBinding): ProviderSurface { + return binding.providerType === "anthropic" + ? { type: "anthropic", configKey: "ANTHROPIC_BASE_URL" } + : { type: "openai", configKey: "OPENAI_BASE_URL" }; +} + +function resultText(result: CaptureOpenshellResult): string { + // includeStreams=true normally makes `output` a duplicate aggregate of + // stdout/stderr. Parse the split streams when present and use `output` only + // as the compatibility fallback so strict duplicate-field checks keep + // working on normal OpenShell results. + const hasStreams = result.stdout !== undefined || result.stderr !== undefined; + const combined = hasStreams + ? `${result.stdout ?? ""}\n${result.stderr ?? ""}` + : String(result.output ?? ""); + return Buffer.from(combined, "utf8") + .subarray(0, OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER) + .toString("utf8"); +} + +function parseProviderVersion(output: string): { id: string; resourceVersion: number } | null { + const clean = stripAnsi(output); + const ids = Array.from(clean.matchAll(/^\s*Id:\s*([A-Za-z0-9._:-]{1,128})\s*$/gimu)); + const versions = Array.from(clean.matchAll(/^\s*Resource version:\s*([0-9]+)\s*$/gimu)); + if (ids.length !== 1 || versions.length !== 1) return null; + const resourceVersion = Number(versions[0][1]); + if (!Number.isSafeInteger(resourceVersion) || resourceVersion < 1) return null; + return { id: ids[0][1], resourceVersion }; +} + +function inspectProvider( + captureOpenshell: CaptureProviderCommand, + gatewayName: string, + providerName: string, +): ProviderObservation { + const result = captureOpenshell(["provider", "get", "-g", gatewayName, providerName], { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }); + const output = resultText(result); + if (result.status !== 0) { + return openshellReportsProviderNotFound(output, providerName) + ? { kind: "absent" } + : { kind: "error", status: result.status }; + } + const metadata = parseGatewayProviderMetadata(output); + const version = parseProviderVersion(output); + if (!metadata || !version) return { kind: "error", status: result.status }; + return { kind: "present", ...version, metadata }; +} + +function expectedShape(providerName: string, surface: ProviderSurface, credentialEnv: string) { + return { + name: providerName, + type: surface.type, + credentialKey: credentialEnv, + configKey: surface.configKey, + }; +} + +function assertProviderOwnership(options: { + observation: ProviderObservation; + providerName: string; + surface: ProviderSurface; + binding: HttpsPinProviderBinding; +}): "create" | "update" { + const { observation, providerName, surface, binding } = options; + if (observation.kind === "absent") return "create"; + if (observation.kind === "error") { + throw new InferenceSetError( + `Could not inspect provider '${providerName}' (status ${observation.status ?? "unknown"}); no provider mutation was attempted.`, + 1, + ); + } + if ( + !matchesGatewayProviderBinding( + observation.metadata, + expectedShape(providerName, surface, binding.credentialEnv), + ) + ) { + throw new InferenceSetError( + `Refusing to replace provider '${providerName}': its live binding is malformed, foreign, or does not match this sandbox's durable custom-endpoint provenance. Re-run onboarding to reconcile the provider safely.`, + 2, + ); + } + return "update"; +} + +function mutationArgs(options: { + action: "create" | "update"; + gatewayName: string; + providerName: string; + surface: ProviderSurface; + credentialEnv: string; + baseUrl: string; +}): string[] { + const args = + options.action === "create" + ? [ + "provider", + "create", + "-g", + options.gatewayName, + "--name", + options.providerName, + "--type", + options.surface.type, + ] + : ["provider", "update", "-g", options.gatewayName, options.providerName]; + args.push( + "--credential", + options.credentialEnv, + "--config", + `${options.surface.configKey}=${options.baseUrl}`, + ); + return args; +} + +export function prepareHttpsPinProviderBinding(options: { + gatewayName: string; + providerName: string; + binding: HttpsPinProviderBinding; + captureOpenshell: CaptureProviderCommand; +}): { action: "create" | "update"; commit: () => void; rollback: () => void } { + const { gatewayName, providerName, binding, captureOpenshell } = options; + const surface = providerSurface(binding); + const before = inspectProvider(captureOpenshell, gatewayName, providerName); + const action = assertProviderOwnership({ + observation: before, + providerName, + surface, + binding, + }); + + const apply = (): void => { + const result = captureOpenshell( + mutationArgs({ + action, + gatewayName, + providerName, + surface, + credentialEnv: binding.credentialEnv, + baseUrl: binding.baseUrl, + }), + { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + env: { [binding.credentialEnv]: binding.token }, + }, + ); + const after = inspectProvider(captureOpenshell, gatewayName, providerName); + if (result.status !== 0) { + throw new InferenceSetError( + `Failed to ${action} HTTPS-pinned provider '${providerName}' on gateway '${gatewayName}' (status ${result.status ?? "unknown"}). ` + + `The provider command may have partially applied; retry this command or re-run onboarding to converge the safe adapter binding.`, + 1, + ); + } + if ( + after.kind !== "present" || + (action === "update" && + (before.kind !== "present" || + after.id !== before.id || + after.resourceVersion <= before.resourceVersion)) || + !matchesGatewayProviderBinding( + after.metadata, + expectedShape(providerName, surface, binding.credentialEnv), + ) + ) { + throw new InferenceSetError( + `Provider '${providerName}' did not converge to the expected HTTPS-pinned type and binding-key shape after ${action}. ` + + `Provider state may be partial; retry this command or re-run onboarding to reconcile it.`, + 1, + ); + } + }; + + if (action === "update") { + return { + action, + commit: apply, + rollback: () => {}, + }; + } + + apply(); + return { + action, + commit: () => {}, + rollback: () => { + const result = captureOpenshell(["provider", "delete", "-g", gatewayName, providerName], { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }); + const restored = inspectProvider(captureOpenshell, gatewayName, providerName); + if (result.status !== 0 || restored.kind !== "absent") { + throw new InferenceSetError( + `Failed to remove newly created provider '${providerName}' after inference selection failed.`, + 1, + ); + } + }, + }; +} + +export const __test = { + inspectProvider, + parseProviderVersion, + providerSurface, + mutationArgs, +}; diff --git a/src/lib/actions/inference-set-https-pin-runtime.test.ts b/src/lib/actions/inference-set-https-pin-runtime.test.ts new file mode 100644 index 00000000000..96adaf3f55e --- /dev/null +++ b/src/lib/actions/inference-set-https-pin-runtime.test.ts @@ -0,0 +1,537 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV } from "../inference/https-pin-runtime"; +import type { ConfigObject } from "../security/credential-filter"; +import type { InferenceSetDeps } from "./inference-set"; +import { runInferenceSet } from "./inference-set"; +import { baseSession, createDeps, HERMES_TARGET } from "./inference-set.test-support"; +import type { EnsureHttpsPinRuntimeAdapterOptions } from "./inference-set-route-containment"; + +const ADAPTER_TOKEN = "test-route-token"; +const NEW_ROUTE_ID = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const OLD_ROUTE_ID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const ADAPTER_BASE_URL = `http://host.openshell.internal:11438/route/${NEW_ROUTE_ID}`; +const OLD_ADAPTER_BASE_URL = `http://host.openshell.internal:11438/route/${OLD_ROUTE_ID}`; +const PROVIDER_ID = "11111111-2222-4333-8444-555555555555"; + +function mockAdapter() { + return vi.fn(async (_options: EnsureHttpsPinRuntimeAdapterOptions) => ({ + baseUrl: ADAPTER_BASE_URL, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: ADAPTER_TOKEN, + routeId: NEW_ROUTE_ID, + })); +} + +function providerCapture(options: { + providerName: string; + providerType: "openai" | "anthropic"; + credentialEnv: string; +}): InferenceSetDeps["captureOpenshell"] & ReturnType { + let resourceVersion = 4; + const configKey = options.providerType === "anthropic" ? "ANTHROPIC_BASE_URL" : "OPENAI_BASE_URL"; + const output = () => + [ + `Name: ${options.providerName}`, + `Id: ${PROVIDER_ID}`, + `Type: ${options.providerType}`, + `Resource version: ${resourceVersion}`, + `Credential keys: ${options.credentialEnv}`, + `Config keys: ${configKey}`, + ].join("\n"); + return vi.fn((args: string[]) => { + switch (`${args[0]}:${args[1]}`) { + case "provider:get": { + const text = output(); + return { status: 0, stdout: text, stderr: "", output: text }; + } + case "provider:update": + resourceVersion += 1; + return { status: 0, stdout: "", stderr: "", output: "" }; + default: + return { status: 0, stdout: "", stderr: "", output: "" }; + } + }) as InferenceSetDeps["captureOpenshell"] & ReturnType; +} + +function failRegistryRead(): never { + throw new Error("registry unavailable"); +} + +function failRegistryReadAfterTwoCalls( + originalListSandboxes: InferenceSetDeps["listSandboxes"], +): InferenceSetDeps["listSandboxes"] { + let listCalls = 0; + return () => (listCalls++ < 2 ? originalListSandboxes() : failRegistryRead()); +} + +describe("runInferenceSet HTTPS-pin route credential handoff (#6141)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + delete process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]; + }); + + it.each([ + [ + "compatible-endpoint", + "COMPATIBLE_API_KEY", + "openai-completions", + "openai", + "OPENAI_BASE_URL", + ], + [ + "compatible-anthropic-endpoint", + "COMPATIBLE_ANTHROPIC_API_KEY", + "anthropic-messages", + "anthropic", + "ANTHROPIC_BASE_URL", + ], + ] as const)("keeps the upstream secret host-only and binds a route token for %s", async (provider, credentialEnv, inferenceApi, providerType, configKey) => { + vi.stubEnv(credentialEnv, "real-upstream-secret"); + const adapter = mockAdapter(); + const capture = providerCapture({ providerName: provider, providerType, credentialEnv }); + const config: ConfigObject = { + agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, + models: { providers: { inference: { api: "openai-completions", models: [] } } }, + }; + const deps = createDeps({ + config, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "nvidia/model-a", + }, + session: baseSession({ provider: "nvidia-prod", model: "nvidia/model-a" }), + ensureHttpsPinRuntimeAdapter: adapter, + captureOpenshell: capture, + }); + + await runInferenceSet( + { + provider, + model: "mock-model", + endpointUrl: "https://compatible.example/v1", + credentialEnv, + inferenceApi, + }, + deps, + ); + + expect(adapter).toHaveBeenCalledWith( + expect.objectContaining({ credentialValue: "real-upstream-secret", providerType }), + ); + expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ + "alpha", + expect.objectContaining({ provider, endpointUrl: ADAPTER_BASE_URL, credentialEnv }), + ]); + expect(deps.getSession()).toMatchObject({ + provider, + endpointUrl: ADAPTER_BASE_URL, + credentialEnv, + }); + expect(JSON.stringify(deps.calls.updateSandbox.mock.calls)).not.toContain("compatible.example"); + expect(JSON.stringify(deps.calls.updateSandbox.mock.calls)).not.toContain("/v1"); + expect(process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]).toBeUndefined(); + + const mutation = capture.mock.calls.find( + ([args]) => args[0] === "provider" && args[1] === "update", + ); + expect(mutation?.[0]).toContain(`${configKey}=${ADAPTER_BASE_URL}`); + expect(mutation?.[1]).toEqual( + expect.objectContaining({ env: { [credentialEnv]: ADAPTER_TOKEN } }), + ); + expect(JSON.stringify(capture.mock.calls)).not.toContain("real-upstream-secret"); + expect(JSON.stringify(capture.mock.calls)).not.toContain("compatible.example"); + expect(capture).toHaveBeenCalledWith( + expect.arrayContaining(["inference", "set", "--no-verify"]), + expect.any(Object), + ); + const inferenceSetIndex = capture.mock.calls.findIndex( + ([args]) => args[0] === "inference" && args[1] === "set", + ); + const providerUpdateIndex = capture.mock.calls.findIndex( + ([args]) => args[0] === "provider" && args[1] === "update", + ); + expect(inferenceSetIndex).toBeGreaterThanOrEqual(0); + expect(providerUpdateIndex).toBeGreaterThan(inferenceSetIndex); + }); + + it("uses the OpenAI provider surface for a Hermes compatible-Anthropic route", async () => { + vi.stubEnv("COMPATIBLE_ANTHROPIC_API_KEY", "real-hermes-upstream-secret"); + const adapter = mockAdapter(); + const capture = providerCapture({ + providerName: "compatible-anthropic-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }); + const deps = createDeps({ + config: { model: {} }, + entry: { + name: "hermes", + agent: "hermes", + provider: "hermes-provider", + model: "old-model", + }, + defaultSandbox: "hermes", + target: HERMES_TARGET, + session: baseSession({ agent: "hermes", sandboxName: "hermes" }), + ensureHttpsPinRuntimeAdapter: adapter, + captureOpenshell: capture, + }); + + await runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "claude-proxy", + sandboxName: "hermes", + endpointUrl: "https://compatible.example/v1", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ); + + expect(adapter).toHaveBeenCalledWith(expect.objectContaining({ providerType: "openai" })); + const mutation = capture.mock.calls.find( + ([args]) => args[0] === "provider" && args[1] === "update", + ); + expect(mutation?.[0]).toContain(`OPENAI_BASE_URL=${ADAPTER_BASE_URL}`); + expect(mutation?.[1]).toEqual( + expect.objectContaining({ + env: { COMPATIBLE_ANTHROPIC_API_KEY: ADAPTER_TOKEN }, + }), + ); + }); + + it("leaves a distinct prior provider binding untouched when inference selection fails", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const capture = providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }); + const original = capture.getMockImplementation() as InferenceSetDeps["captureOpenshell"]; + capture.mockImplementation((args, opts) => + args[0] === "inference" && args[1] === "set" + ? { status: 1, stdout: "", stderr: "selection failed", output: "selection failed" } + : original(args, opts), + ); + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }, + ensureHttpsPinRuntimeAdapter: mockAdapter(), + captureOpenshell: capture, + }); + + await expect( + runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://compatible.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ), + ).rejects.toThrow( + "The existing OpenShell provider binding and inference selection were not changed.", + ); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + const providerUpdates = capture.mock.calls.filter( + ([args]) => args[0] === "provider" && args[1] === "update", + ); + expect(providerUpdates).toHaveLength(0); + }); + + it("restores the prior inference selection when the deferred provider update fails", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const capture = providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }); + const original = capture.getMockImplementation() as InferenceSetDeps["captureOpenshell"]; + capture.mockImplementation((args, opts) => + args[0] === "provider" && args[1] === "update" + ? { + status: 1, + stdout: "", + stderr: "provider update failed", + output: "provider update failed", + } + : original(args, opts), + ); + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "nvidia-prod", + model: "old-model", + }, + ensureHttpsPinRuntimeAdapter: mockAdapter(), + captureOpenshell: capture, + }); + + await expect( + runInferenceSet( + { + provider: "compatible-endpoint", + model: "new-model", + endpointUrl: "https://compatible.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ), + ).rejects.toThrow( + "The previous OpenShell inference selection was restored to 'nvidia-prod' / 'old-model'", + ); + expect(deps.calls.updateSandbox).not.toHaveBeenCalled(); + const selectionMutations = capture.mock.calls.filter( + ([args]) => args[0] === "inference" && args[1] === "set", + ); + expect(selectionMutations).toHaveLength(2); + expect(selectionMutations[0]?.[0]).toEqual( + expect.arrayContaining([ + "--provider", + "compatible-endpoint", + "--model", + "new-model", + "--no-verify", + ]), + ); + expect(selectionMutations[1]?.[0]).toEqual( + expect.arrayContaining(["--provider", "nvidia-prod", "--model", "old-model", "--no-verify"]), + ); + }); + + it("reports committed provider and selection state when registry convergence fails", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const capture = providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }); + const deps = createDeps({ + config: {}, + entry: { name: "alpha", agent: "openclaw", provider: "nvidia-prod", model: "old" }, + ensureHttpsPinRuntimeAdapter: mockAdapter(), + captureOpenshell: capture, + updateSandbox: () => false, + }); + + await expect( + runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://compatible.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ), + ).rejects.toThrow( + "provider and inference selection remain committed to the safer HTTPS-pinned adapter", + ); + expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled(); + }); + + it("revokes a superseded adapter route only after both registry commits", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }, + ensureHttpsPinRuntimeAdapter: mockAdapter(), + captureOpenshell: providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }), + }); + + await runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://new.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ); + + expect(deps.calls.updateSandbox).toHaveBeenCalledTimes(2); + expect(deps.calls.revokeHttpsPinRuntimeAdapterRoute).toHaveBeenCalledWith(OLD_ROUTE_ID); + expect( + deps.calls.revokeHttpsPinRuntimeAdapterRoute.mock.invocationCallOrder[0], + ).toBeGreaterThan(deps.calls.updateSandbox.mock.invocationCallOrder[1]); + }); + + it("revokes an adapter route when switching to a non-adapter provider", async () => { + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }, + }); + + await runInferenceSet({ provider: "nvidia-prod", model: "nvidia/new" }, deps); + + expect(deps.calls.revokeHttpsPinRuntimeAdapterRoute).toHaveBeenCalledWith(OLD_ROUTE_ID); + }); + + it("keeps a superseded route while another sandbox still references it", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const alpha = { + name: "alpha", + agent: "openclaw" as const, + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }; + const peer = { + name: "peer", + agent: "openclaw" as const, + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }; + const deps = createDeps({ + config: {}, + entries: [alpha], + ensureHttpsPinRuntimeAdapter: mockAdapter(), + captureOpenshell: providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }), + }); + let listCalls = 0; + deps.listSandboxes = () => ({ + sandboxes: listCalls++ < 2 ? [alpha] : [alpha, peer], + defaultSandbox: "alpha", + }); + + await runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://new.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ); + + expect(deps.calls.revokeHttpsPinRuntimeAdapterRoute).not.toHaveBeenCalled(); + }); + + it.each([ + ["peer registry read", "list"], + ["adapter DELETE", "revoke"], + ] as const)("keeps the committed route when post-commit %s fails", async (_name, failure) => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }, + ensureHttpsPinRuntimeAdapter: mockAdapter(), + revokeHttpsPinRuntimeAdapterRoute: + failure === "revoke" + ? async () => { + throw new Error("delete unavailable"); + } + : undefined, + captureOpenshell: providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }), + }); + deps.listSandboxes = + failure === "list" ? failRegistryReadAfterTwoCalls(deps.listSandboxes) : deps.listSandboxes; + + await expect( + runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://new.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ), + ).resolves.toMatchObject({ sandboxName: "alpha", provider: "compatible-endpoint" }); + expect(deps.calls.updateSandbox).toHaveBeenCalledTimes(2); + expect(deps.calls.log).toHaveBeenCalledWith(expect.stringContaining("could not be revoked")); + }); + + it("does not revoke when re-registration keeps the same route id", async () => { + vi.stubEnv("COMPATIBLE_API_KEY", "real-upstream-secret"); + const sameRouteAdapter = vi.fn(async () => ({ + baseUrl: OLD_ADAPTER_BASE_URL, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: ADAPTER_TOKEN, + routeId: OLD_ROUTE_ID, + })); + const deps = createDeps({ + config: {}, + entry: { + name: "alpha", + agent: "openclaw", + provider: "compatible-endpoint", + model: "old", + endpointUrl: OLD_ADAPTER_BASE_URL, + }, + ensureHttpsPinRuntimeAdapter: sameRouteAdapter, + captureOpenshell: providerCapture({ + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + }), + }); + + await runInferenceSet( + { + provider: "compatible-endpoint", + model: "new", + endpointUrl: "https://same.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ); + + expect(deps.calls.revokeHttpsPinRuntimeAdapterRoute).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/inference-set-provider-alias.test.ts b/src/lib/actions/inference-set-provider-alias.test.ts index 74c8b11a451..105d99f60f4 100644 --- a/src/lib/actions/inference-set-provider-alias.test.ts +++ b/src/lib/actions/inference-set-provider-alias.test.ts @@ -25,6 +25,7 @@ import { runInferenceSet, } from "./inference-set"; import { baseSession, createDeps } from "./inference-set.test-support"; +import type { EnsureHttpsPinRuntimeAdapterOptions } from "./inference-set-route-containment"; // eslint-disable-next-line @typescript-eslint/no-explicit-any const onboardProviders: any = @@ -279,6 +280,29 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { }); } + // A DNS-backed HTTPS endpoint (the shape every URL in this suite uses) never + // reaches rewriteConfigUrlsWithDnsPinning/ssrfGuard above — it is eligible for + // the HTTPS-pin runtime adapter, whose real implementation runs its own SSRF + // preflight (assertEndpointResolvesPublic) before registering a route. This + // stand-in mirrors that preflight against the same STUB_INTERNAL_HOSTS set. + function httpsPinAdapterGuard() { + return vi.fn(async (options: EnsureHttpsPinRuntimeAdapterOptions) => { + const host = new URL(options.endpointUrl).hostname; + return STUB_INTERNAL_HOSTS.has(host) + ? Promise.reject( + new Error( + `URL hostname "${host}" resolves to private/internal address "10.48.203.205". This could expose internal services to the sandbox.`, + ), + ) + : Promise.resolve({ + baseUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN", + token: "test-adapter-token", + routeId: "test-route", + }); + }); + } + it("keeps the SSRF guard when same-endpoint onboarding provenance is missing", async () => { // Legacy registry rows have no machine-checkable endpoint source. Exact // string equality is insufficient because inference set also persists the @@ -294,7 +318,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }, - rewriteConfigUrlsWithDnsPinning: ssrfGuard(), + ensureHttpsPinRuntimeAdapter: httpsPinAdapterGuard(), }); const attempt = runInferenceSet( @@ -323,6 +347,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { // the canonical supplied URL must match the URL whose registry source is // onboarding. DNS re-resolution is not required for that exact identity. const guard = ssrfGuard(); + const adapterGuard = httpsPinAdapterGuard(); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/anthropic/model-a" } } } }, entry: { @@ -336,6 +361,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { preferredInferenceApi: "anthropic-messages", }, rewriteConfigUrlsWithDnsPinning: guard, + ensureHttpsPinRuntimeAdapter: adapterGuard, }); await expect( runInferenceSet( @@ -349,6 +375,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { ), ).resolves.toBeTruthy(); expect(guard).not.toHaveBeenCalled(); + expect(adapterGuard).not.toHaveBeenCalled(); expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ "alpha", expect.objectContaining({ endpointSource: "onboard" }), @@ -357,6 +384,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { it("accepts the same onboard-provenanced internal endpoint after canonicalization (#6321)", async () => { const guard = ssrfGuard(); + const adapterGuard = httpsPinAdapterGuard(); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, @@ -373,6 +401,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { preferredInferenceApi: "openai-completions", }, rewriteConfigUrlsWithDnsPinning: guard, + ensureHttpsPinRuntimeAdapter: adapterGuard, }); await expect( @@ -387,10 +416,11 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { ), ).resolves.toBeTruthy(); expect(guard).not.toHaveBeenCalled(); + expect(adapterGuard).not.toHaveBeenCalled(); }); it("keeps the SSRF guard for an inference-set-authored endpoint", async () => { - const guard = ssrfGuard(); + const guard = httpsPinAdapterGuard(); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, @@ -406,7 +436,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }, - rewriteConfigUrlsWithDnsPinning: guard, + ensureHttpsPinRuntimeAdapter: guard, }); const attempt = runInferenceSet( @@ -500,7 +530,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { provider: "nvidia-prod", model: "nvidia/model-a", }, - rewriteConfigUrlsWithDnsPinning: ssrfGuard(), + ensureHttpsPinRuntimeAdapter: httpsPinAdapterGuard(), }); const attempt = runInferenceSet( diff --git a/src/lib/actions/inference-set-route-containment.ts b/src/lib/actions/inference-set-route-containment.ts index 132f41d6287..86d4fd637fa 100644 --- a/src/lib/actions/inference-set-route-containment.ts +++ b/src/lib/actions/inference-set-route-containment.ts @@ -5,7 +5,14 @@ import { checkGatewayRouteCompatibility, formatGatewayRouteConflict, } from "../inference/gateway-route-compatibility"; +import { + buildHttpsPinRouteBaseUrl, + computeHttpsPinRouteId, + type HttpsPinCredentialProviderType, + isHttpsPinRuntimeEligible, +} from "../inference/https-pin-runtime"; import { resolveSandboxGatewayName } from "../onboard/gateway-binding"; +import { ConfigUrlValidationError } from "../sandbox/config"; import type { ConfigValue } from "../security/credential-filter"; import type { Session } from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; @@ -36,9 +43,39 @@ export interface ExplicitCustomRouteOptions { type RewriteConfigUrlsWithDnsPinning = (value: ConfigValue) => Promise; +/** + * Resolves a DNS-backed HTTPS custom endpoint to a pinned, locally-terminated + * route base URL instead of the raw operator-supplied URL. OpenShell never + * sees the real hostname; the returned URL always targets the trusted + * `host.openshell.internal` bridge, matching the shape already exempted by + * {@link ALLOWED_PRIVATE_CUSTOM_ENDPOINT_HOSTS}. + */ +export interface EnsureHttpsPinRuntimeAdapterOptions { + gatewayName: string; + provider: string; + endpointUrl: string; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; +} +export type EnsureHttpsPinRuntimeAdapterFn = ( + options: EnsureHttpsPinRuntimeAdapterOptions, +) => Promise<{ baseUrl: string; credentialEnv: string; token: string; routeId: string }>; + +export interface HttpsPinProviderBinding { + baseUrl: string; + credentialEnv: string; + token: string; + routeId: string; + providerType: HttpsPinCredentialProviderType; +} + +type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise; + export interface PreparedInferenceSetRoute { gatewayName: string; preliminaryExplicitMetadata: RegistryInferenceMetadata | null; + /** Invocation-only source URL; never persisted for HTTPS-pin routes. */ + preliminaryExplicitSourceEndpointUrl: string | null; preliminaryRegistryMetadata: RegistryInferenceMetadata; } @@ -75,11 +112,15 @@ const ALLOWED_PRIVATE_CUSTOM_ENDPOINT_HOSTS = new Set(["host.openshell.internal" function normalizeEndpointUrlShape(value: string): { url: URL; normalized: string } { const url = new URL(value); - if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) { + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username || + url.password || + url.search || + url.hash + ) { throw new Error("unsupported URL shape"); } - url.search = ""; - url.hash = ""; const pathname = url.pathname.replace(/\/+$/, ""); url.pathname = pathname || "/"; return { @@ -96,7 +137,7 @@ function normalizeCustomEndpointUrlWithoutDns(value: string | null | undefined): return normalizeEndpointUrlShape(raw).normalized; } catch { throw new InferenceSetError( - "endpoint-url must be a valid http(s) URL without embedded credentials.", + "endpoint-url must be a valid http(s) URL without userinfo, query, or fragment components.", 2, ); } @@ -105,6 +146,7 @@ function normalizeCustomEndpointUrlWithoutDns(value: string | null | undefined): export async function normalizeCustomEndpointUrl( value: string | null | undefined, rewriteUrlWithDnsPinning: RewriteConfigUrlsWithDnsPinning, + ensureHttpsPinAdapterRoute?: EnsureHttpsPinAdapterRoute, ): Promise { const normalized = normalizeCustomEndpointUrlWithoutDns(value); const shaped = normalizeEndpointUrlShape(normalized); @@ -124,13 +166,42 @@ export async function normalizeCustomEndpointUrl( return normalized; } + // A DNS-backed HTTPS endpoint cannot be pinned by IP substitution alone: the + // TLS certificate requires the real hostname as SNI, so OpenShell's own + // re-resolution at request time would race the SSRF preflight (TOCTOU) if + // it saw that hostname directly. Route it through the local HTTPS-pin + // runtime adapter instead, which re-validates the address immediately + // before connecting and hides the real hostname from the OpenShell runtime + // boundary entirely. + if (ensureHttpsPinAdapterRoute && isHttpsPinRuntimeEligible(normalized)) { + try { + const effectiveRoute = await ensureHttpsPinAdapterRoute(normalized); + if (typeof effectiveRoute !== "string") + throw new Error("HTTPS pin adapter returned a non-string value"); + // Persist only the sandbox-facing adapter route. The source hostname is + // retained in invocation state long enough to validate and register the + // host adapter, but must not cross into the sandbox registry/session. + return normalizeEndpointUrlShape(effectiveRoute).normalized; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}`, 2); + } + } + try { const validated = await rewriteUrlWithDnsPinning(normalized); if (typeof validated !== "string") throw new Error("URL validator returned a non-string value"); return normalizeEndpointUrlShape(validated).normalized; } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}`, 2); + // The generic DNS-pinning validator's message stays scoped to arbitrary + // persisted config values; only this inference-set call site knows the + // rejected field is an inference endpoint, so it adds the adapter hint. + const hint = + error instanceof ConfigUrlValidationError && error.reason === "dns_backed_https_unsupported" + ? " This endpoint should have been routed through the HTTPS Pin Runtime adapter; retry, and report a bug if this persists." + : ""; + throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}${hint}`, 2); } } @@ -176,8 +247,13 @@ function normalizeExplicitInferenceApi(provider: string, value: string | null | function explicitCustomProviderMetadataWithoutDns( provider: string, options: ExplicitCustomRouteOptions, -): RegistryInferenceMetadata | null { - if (!hasExplicitCustomMetadata(options)) return null; + gatewayName: string, + onboardEndpointUrl: string | null, +): { + metadata: RegistryInferenceMetadata | null; + sourceEndpointUrl: string | null; +} { + if (!hasExplicitCustomMetadata(options)) return { metadata: null, sourceEndpointUrl: null }; if (!isCustomCompatibleProvider(provider)) { throw new InferenceSetError( "endpoint-url, credential-env, and inference-api are only supported for compatible-endpoint and compatible-anthropic-endpoint.", @@ -187,15 +263,28 @@ function explicitCustomProviderMetadataWithoutDns( // Source boundary: custom-compatible endpoint URLs are operator-supplied and // not discoverable from the gateway provider registry with a sandbox-scoped - // trust guarantee. Treat these explicit flags as the durable metadata source - // for this switch, after URL and credential-env validation, instead of - // borrowing from an unrelated onboard session or global OpenShell provider. + // trust guarantee. Treat these explicit flags as this invocation's source, + // after URL and credential-env validation, instead of borrowing from an + // unrelated onboard session or global OpenShell provider. + const sourceEndpointUrl = normalizeCustomEndpointUrlWithoutDns(options.endpointUrl); + const normalizedOnboardEndpoint = onboardEndpointUrl + ? normalizeCustomEndpointUrlWithoutDns(onboardEndpointUrl) + : null; + const reusesOnboardEndpoint = + normalizedOnboardEndpoint !== null && sourceEndpointUrl === normalizedOnboardEndpoint; + const endpointUrl = + !reusesOnboardEndpoint && isHttpsPinRuntimeEligible(sourceEndpointUrl) + ? buildHttpsPinRouteBaseUrl(computeHttpsPinRouteId(gatewayName, provider, sourceEndpointUrl)) + : sourceEndpointUrl; return { - endpointUrl: normalizeCustomEndpointUrlWithoutDns(options.endpointUrl), - endpointSource: "inference-set", - credentialEnv: normalizeExplicitCredentialEnv(provider, options.credentialEnv), - preferredInferenceApi: normalizeExplicitInferenceApi(provider, options.inferenceApi), - nimContainer: null, + metadata: { + endpointUrl, + endpointSource: reusesOnboardEndpoint ? "onboard" : "inference-set", + credentialEnv: normalizeExplicitCredentialEnv(provider, options.credentialEnv), + preferredInferenceApi: normalizeExplicitInferenceApi(provider, options.inferenceApi), + nimContainer: null, + }, + sourceEndpointUrl, }; } @@ -299,10 +388,15 @@ export function prepareInferenceSetRoute(options: { ); } - const preliminaryExplicitMetadata = explicitCustomProviderMetadataWithoutDns( + const explicit = explicitCustomProviderMetadataWithoutDns( options.provider, options.customRoute, + gatewayName, + options.entry.provider === options.provider && options.entry.endpointSource === "onboard" + ? (options.entry.endpointUrl ?? null) + : null, ); + const preliminaryExplicitMetadata = explicit.metadata; const preliminaryRegistryMetadata = registryMetadataForProviderSwitch({ entry: options.entry, provider: options.provider, @@ -319,7 +413,12 @@ export function prepareInferenceSetRoute(options: { metadata: preliminaryRegistryMetadata, sandboxes: options.sandboxes, }); - return { gatewayName, preliminaryExplicitMetadata, preliminaryRegistryMetadata }; + return { + gatewayName, + preliminaryExplicitMetadata, + preliminaryExplicitSourceEndpointUrl: explicit.sourceEndpointUrl, + preliminaryRegistryMetadata, + }; } export async function finalizeInferenceSetRoute(options: { @@ -331,22 +430,64 @@ export async function finalizeInferenceSetRoute(options: { onboardEndpointUrl: string | null; getSandboxes: () => SandboxEntry[]; rewriteUrlWithDnsPinning: RewriteConfigUrlsWithDnsPinning; + ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; + effectiveInferenceApi?: string | null; }): Promise<{ registryMetadata: RegistryInferenceMetadata; explicitPreferredInferenceApi: string | null; + httpsPinProviderBinding: HttpsPinProviderBinding | null; }> { const { prepared } = options; if (!prepared.preliminaryExplicitMetadata) { return { registryMetadata: prepared.preliminaryRegistryMetadata, explicitPreferredInferenceApi: null, + httpsPinProviderBinding: null, }; } + // Bound once per finalize call: the credential env var name is fixed per + // provider (normalizeExplicitCredentialEnv already enforced this), and the + // real credential value is read directly from the host process environment + // at invocation time, never persisted, and never returned to the caller. + const httpsPinCredentialEnv = CUSTOM_COMPATIBLE_CREDENTIAL_ENV[options.provider]; + // Set only when the adapter route is actually used. The canonical provider + // credential key stays stable; only its invocation-local value becomes the + // route-scoped adapter token. + let httpsPinProviderBinding: HttpsPinProviderBinding | null = null; + const ensureHttpsPinAdapterRoute: EnsureHttpsPinAdapterRoute = async (endpointUrl) => { + // The credential is held only for this invocation and handed directly + // to the adapter. It is never persisted, returned, or copied to a shared + // process.env slot. + const credentialValue = process.env[httpsPinCredentialEnv] ?? ""; + const providerType: HttpsPinCredentialProviderType = + (options.effectiveInferenceApi ?? + prepared.preliminaryExplicitMetadata?.preferredInferenceApi) === "anthropic-messages" + ? "anthropic" + : "openai"; + const adapter = await options.ensureHttpsPinRuntimeAdapter({ + gatewayName: prepared.gatewayName, + provider: options.provider, + endpointUrl, + providerType, + credentialValue, + }); + httpsPinProviderBinding = { + ...adapter, + // Keep the provider's one canonical credential key. Only its + // invocation-local value changes to the route-scoped token; using a + // second key risks OpenShell merging credential bindings on an attached + // provider instead of replacing the old key. + credentialEnv: httpsPinCredentialEnv, + providerType, + }; + return adapter.baseUrl; + }; let endpointUrl: string; let endpointSource: RegistryInferenceMetadata["endpointSource"]; try { const suppliedEndpoint = normalizeCustomEndpointUrlWithoutDns( - prepared.preliminaryExplicitMetadata.endpointUrl, + prepared.preliminaryExplicitSourceEndpointUrl ?? + prepared.preliminaryExplicitMetadata.endpointUrl, ); const onboardEndpoint = options.onboardEndpointUrl ? normalizeCustomEndpointUrlWithoutDns(options.onboardEndpointUrl) @@ -363,6 +504,7 @@ export async function finalizeInferenceSetRoute(options: { endpointUrl = await normalizeCustomEndpointUrl( suppliedEndpoint, options.rewriteUrlWithDnsPinning, + ensureHttpsPinAdapterRoute, ); endpointSource = "inference-set"; } @@ -401,5 +543,6 @@ export async function finalizeInferenceSetRoute(options: { return { registryMetadata, explicitPreferredInferenceApi: registryMetadata.preferredInferenceApi ?? null, + httpsPinProviderBinding, }; } diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 00de8f2a36c..581574af36c 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -8,6 +8,7 @@ import type { ConfigObject, ConfigValue } from "../security/credential-filter"; import type { Session } from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; import type { InferenceSetDeps } from "./inference-set"; +import type { EnsureHttpsPinRuntimeAdapterFn } from "./inference-set-route-containment"; export const OPENCLAW_TARGET: AgentConfigTarget = { agentName: "openclaw", @@ -79,12 +80,16 @@ export function createDeps(options: { target?: AgentConfigTarget; session?: Session | null; openshellStatus?: number; + captureOpenshell?: InferenceSetDeps["captureOpenshell"]; localValidation?: ValidationResult; localReachable?: boolean; contextWindow?: number | null; shieldsMutable?: boolean; prepareRunOpenshell?: () => void; rewriteConfigUrlsWithDnsPinning?: (value: ConfigValue) => Promise; + ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn; + revokeHttpsPinRuntimeAdapterRoute?: InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"]; + updateSandbox?: InferenceSetDeps["updateSandbox"]; restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"]; seedHermesDashboardConfigResult?: "converged" | "absent" | "failed"; withGatewayRouteMutationLock?: InferenceSetDeps["withGatewayRouteMutationLock"]; @@ -104,6 +109,8 @@ export function createDeps(options: { resolveContextWindowForModel: ReturnType; prepareRunOpenshell: ReturnType; rewriteConfigUrlsWithDnsPinning: ReturnType; + ensureHttpsPinRuntimeAdapter: ReturnType; + revokeHttpsPinRuntimeAdapterRoute: ReturnType; restartSandboxGateway: ReturnType; withGatewayRouteMutationLock: ReturnType; }; @@ -118,16 +125,19 @@ export function createDeps(options: { const defaultSandbox = options.defaultSandbox === undefined ? (entries[0]?.name ?? null) : options.defaultSandbox; const calls = { - captureOpenshell: vi.fn(() => ({ - status: options.openshellStatus ?? 0, - output: "", - stdout: "", - stderr: "", - })), + captureOpenshell: vi.fn( + options.captureOpenshell ?? + (() => ({ + status: options.openshellStatus ?? 0, + output: "", + stdout: "", + stderr: "", + })), + ), writeSandboxConfig: vi.fn(), recomputeSandboxConfigHash: vi.fn(), seedHermesDashboardConfig: vi.fn(() => options.seedHermesDashboardConfigResult ?? "converged"), - updateSandbox: vi.fn(() => true), + updateSandbox: vi.fn(options.updateSandbox ?? (() => true)), readSandboxConfig: vi.fn(() => options.config), updateSession: vi.fn((mutator: (value: Session) => Session | void) => { const current = session ?? baseSession(); @@ -145,6 +155,18 @@ export function createDeps(options: { rewriteConfigUrlsWithDnsPinning: vi.fn( options.rewriteConfigUrlsWithDnsPinning ?? (async (value: ConfigValue) => value), ), + ensureHttpsPinRuntimeAdapter: vi.fn( + options.ensureHttpsPinRuntimeAdapter ?? + (async () => ({ + baseUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN", + token: "test-adapter-token", + routeId: "test-route", + })), + ), + revokeHttpsPinRuntimeAdapterRoute: vi.fn( + options.revokeHttpsPinRuntimeAdapterRoute ?? (async () => true), + ), restartSandboxGateway: vi.fn( options.restartSandboxGateway ?? ((): ReturnType => ({ @@ -184,6 +206,10 @@ export function createDeps(options: { resolveContextWindowForModel: calls.resolveContextWindowForModel, isSandboxConfigMutable: () => options.shieldsMutable ?? true, rewriteConfigUrlsWithDnsPinning: calls.rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter: + calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn, + revokeHttpsPinRuntimeAdapterRoute: + calls.revokeHttpsPinRuntimeAdapterRoute as InferenceSetDeps["revokeHttpsPinRuntimeAdapterRoute"], withGatewayRouteMutationLock: calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"], restartSandboxGateway: calls.restartSandboxGateway, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 86f70df3def..ac82cd605f7 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -15,6 +15,11 @@ import { } from "../inference/config"; import { resolveContextWindowForModel } from "../inference/context-window"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; +import { parseHttpsPinRouteId } from "../inference/https-pin-runtime"; +import { + ensureHttpsPinRuntimeAdapter, + revokeHttpsPinRuntimeAdapterRoute, +} from "../inference/https-pin-runtime-adapter"; import { type ValidationResult, validateLocalProvider } from "../inference/local"; import { inferenceSelectionRegistryFields } from "../inference/selection"; import { resolveSandboxGatewayName } from "../onboard/gateway-binding"; @@ -57,12 +62,14 @@ import { type InferenceMutation, readPreviousOpenClawInferenceApi, } from "./inference-set-gateway-restart"; +import { prepareHttpsPinProviderBinding } from "./inference-set-https-pin-provider"; import { buildInferenceSetFailure } from "./inference-set-provider-diagnostics"; import { applyOpenClawAnthropicReplyBudget, readOpenClawPrimaryReplyBudget, } from "./inference-set-reply-budget"; import { + type EnsureHttpsPinRuntimeAdapterFn, finalizeInferenceSetRoute, prepareInferenceSetRoute, type RegistryInferenceMetadata, @@ -127,7 +134,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { args: string[], opts?: Pick< CaptureOpenshellOptions, - "ignoreError" | "includeStreams" | "maxBuffer" | "timeout" + "env" | "ignoreError" | "includeStreams" | "maxBuffer" | "timeout" >, ) => CaptureOpenshellResult; isLocalInferenceProvider: (provider: string) => boolean; @@ -136,6 +143,8 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { resolveContextWindowForModel: (provider: string, model: string) => number | null; isSandboxConfigMutable: (sandboxName: string) => boolean; rewriteConfigUrlsWithDnsPinning: (value: ConfigValue) => Promise; + ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; + revokeHttpsPinRuntimeAdapterRoute: (routeId: string) => Promise; withGatewayRouteMutationLock: typeof withGatewayRouteMutationLock; } @@ -237,6 +246,8 @@ function defaultDeps(): InferenceSetDeps { ensureLocalProviderReachable, resolveContextWindowForModel, rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter, + revokeHttpsPinRuntimeAdapterRoute, withGatewayRouteMutationLock, restartSandboxGateway: defaultInferenceGatewayRestart, isSandboxConfigMutable: (sandboxName) => { @@ -566,6 +577,7 @@ function assertHermesCompatibleAnthropicOpenAiProvider( provider: string, endpointUrl: string | null, deps: InferenceSetDeps, + httpsPinProviderBinding: { providerType: "openai" | "anthropic" } | null = null, ): void { if ( agentName !== "hermes" || @@ -574,6 +586,7 @@ function assertHermesCompatibleAnthropicOpenAiProvider( ) { return; } + if (httpsPinProviderBinding?.providerType === "openai") return; const result = deps.captureOpenshell(["provider", "get", "-g", gatewayName, provider], { ignoreError: true, @@ -649,6 +662,7 @@ async function runInferenceSetWithoutHostLock( } const { sandboxName, entry, agentName } = resolveTargetSandbox(options.sandboxName, deps); + const priorHttpsPinRouteId = parseHttpsPinRouteId(entry.endpointUrl); if (agentName !== "openclaw" && agentName !== "hermes") { // #6321: Deep Agents Code (langchain-deepagents-code) bakes its model into // the sandbox image at build time (agents/langchain-deepagents-code/Dockerfile @@ -751,24 +765,28 @@ async function runInferenceSetWithoutHostLock( 2, ); } - const { registryMetadata, explicitPreferredInferenceApi } = await finalizeInferenceSetRoute({ - prepared: preparedRoute, - sandboxName, - provider, - model, - canReuseRecordedRoute: - entry.provider === provider && - typeof entry.endpointUrl === "string" && - entry.endpointUrl.trim().length > 0 && - typeof entry.preferredInferenceApi === "string" && - entry.preferredInferenceApi.trim().length > 0, - onboardEndpointUrl: - entry.provider === provider && entry.endpointSource === "onboard" - ? (entry.endpointUrl ?? null) - : null, - getSandboxes: () => deps.listSandboxes().sandboxes, - rewriteUrlWithDnsPinning: deps.rewriteConfigUrlsWithDnsPinning, - }); + const { registryMetadata, explicitPreferredInferenceApi, httpsPinProviderBinding } = + await finalizeInferenceSetRoute({ + prepared: preparedRoute, + sandboxName, + provider, + model, + canReuseRecordedRoute: + entry.provider === provider && + typeof entry.endpointUrl === "string" && + entry.endpointUrl.trim().length > 0 && + typeof entry.preferredInferenceApi === "string" && + entry.preferredInferenceApi.trim().length > 0, + onboardEndpointUrl: + entry.provider === provider && entry.endpointSource === "onboard" + ? (entry.endpointUrl ?? null) + : null, + getSandboxes: () => deps.listSandboxes().sandboxes, + rewriteUrlWithDnsPinning: deps.rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter: deps.ensureHttpsPinRuntimeAdapter, + effectiveInferenceApi: + preparedRoute.preliminaryExplicitMetadata?.preferredInferenceApi ?? null, + }); // Local providers (ollama-local, vllm-local) route through the sandbox-facing // host.openshell.internal hostname, which the host-side `openshell inference set` @@ -777,6 +795,10 @@ async function runInferenceSetWithoutHostLock( // verify. Only a genuinely-unreachable host stack hard-fails here, before the // route is touched. let effectiveNoVerify = options.noVerify === true; + // The adapter origin resolves only from inside the sandbox network. The + // host-side OpenShell verifier cannot resolve host.openshell.internal, so + // adapter registration + local health are the verification boundary. + if (httpsPinProviderBinding) effectiveNoVerify = true; if (deps.isLocalInferenceProvider(provider)) { const localValidation = deps.validateLocalProvider(provider); if (localValidation.ok) { @@ -809,6 +831,7 @@ async function runInferenceSetWithoutHostLock( provider, registryMetadata.endpointUrl ?? null, deps, + httpsPinProviderBinding, ); // Read the in-sandbox config *before* mutating the gateway route or registry. @@ -817,184 +840,304 @@ async function runInferenceSetWithoutHostLock( // `rebuild`-recoverable), so gate on the read here to abort cleanly instead of // leaving a half-applied switch across the three config layers (#6997). const config = readInSandboxConfigOrFail(deps, sandboxName, target); + const previousProvider = typeof entry.provider === "string" ? entry.provider.trim() : ""; + const previousModel = typeof entry.model === "string" ? entry.model.trim() : ""; - deps.log(` Setting OpenShell inference route: ${provider} / ${model}`); - const setResult = deps.captureOpenshell( - openshellInferenceSetArgs({ - gatewayName: preparedRoute.gatewayName, - provider, - model, - noVerify: effectiveNoVerify, - }), - { - ignoreError: true, - includeStreams: true, - maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, - }, - ); - if (setResult.status !== 0) { - const failure = buildInferenceSetFailure(setResult, provider, deps); - throw new InferenceSetError(failure.message, failure.exitCode); - } + let appliedHttpsPinProvider = false; + let appliedInferenceSelection = false; + let restoredSelectionAfterProviderFailure = false; + let httpsPinProviderMutation: ReturnType | null = null; + try { + if (httpsPinProviderBinding) { + httpsPinProviderMutation = prepareHttpsPinProviderBinding({ + gatewayName: preparedRoute.gatewayName, + providerName: provider, + binding: httpsPinProviderBinding, + captureOpenshell: deps.captureOpenshell, + }); + appliedHttpsPinProvider = httpsPinProviderMutation.action === "create"; + if (httpsPinProviderMutation.action === "update" && (!previousProvider || !previousModel)) { + throw new InferenceSetError( + `Cannot update existing HTTPS-pinned provider '${provider}' because sandbox '${sandboxName}' ` + + `does not record the previous provider and model needed to restore its inference selection.`, + 2, + ); + } + } - // Write minimal registry state before any sandbox-facing config read so the - // gateway and registry cannot split if the in-sandbox layer is unavailable. - const registryFields = (preferredInferenceApi: string | null) => - inferenceSelectionRegistryFields({ - provider, - model, - endpointUrl: registryMetadata.endpointUrl ?? null, - endpointSource: registryMetadata.endpointSource ?? null, - credentialEnv: registryMetadata.credentialEnv ?? null, - preferredInferenceApi, - nimContainer: registryMetadata.nimContainer ?? null, - }); - if ( - !deps.updateSandbox( - sandboxName, - registryFields( - resolveAgentInferenceApi( - agentName, - provider, - registryMetadata.preferredInferenceApi ?? null, + deps.log(` Setting OpenShell inference route: ${provider} / ${model}`); + const setResult = deps.captureOpenshell( + openshellInferenceSetArgs({ + gatewayName: preparedRoute.gatewayName, + provider, + model, + noVerify: effectiveNoVerify, + }), + { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }, + ); + if (setResult.status !== 0) { + const failure = buildInferenceSetFailure(setResult, provider, deps); + throw new InferenceSetError(failure.message, failure.exitCode); + } + appliedInferenceSelection = true; + if (httpsPinProviderMutation) { + try { + httpsPinProviderMutation.commit(); + appliedHttpsPinProvider = true; + } catch (providerError) { + const providerDetail = + providerError instanceof Error ? providerError.message : String(providerError); + const providerExitCode = + providerError instanceof InferenceSetError ? providerError.exitCode : 1; + const restoreResult = deps.captureOpenshell( + openshellInferenceSetArgs({ + gatewayName: preparedRoute.gatewayName, + provider: previousProvider, + model: previousModel, + noVerify: true, + }), + { + ignoreError: true, + includeStreams: true, + maxBuffer: OPEN_SHELL_FAILURE_CAPTURE_MAX_BUFFER, + }, + ); + if (restoreResult.status !== 0) { + throw new InferenceSetError( + `${providerDetail}\n Failed to restore the previous OpenShell inference selection ` + + `'${previousProvider}' / '${previousModel}' (status ${restoreResult.status ?? "unknown"}). ` + + `The live selection and provider binding may be split; re-run onboarding before using this route.`, + providerExitCode, + ); + } + appliedInferenceSelection = false; + restoredSelectionAfterProviderFailure = true; + throw new InferenceSetError( + `${providerDetail}\n The previous OpenShell inference selection was restored to ` + + `'${previousProvider}' / '${previousModel}'. Provider state may still be partial; ` + + `retry this command or re-run onboarding to reconcile it.`, + providerExitCode, + ); + } + } + + // Write minimal registry state before any sandbox-facing config read so the + // gateway and registry cannot split if the in-sandbox layer is unavailable. + const registryFields = (preferredInferenceApi: string | null) => + inferenceSelectionRegistryFields({ + provider, + model, + endpointUrl: registryMetadata.endpointUrl ?? null, + endpointSource: registryMetadata.endpointSource ?? null, + credentialEnv: registryMetadata.credentialEnv ?? null, + preferredInferenceApi, + nimContainer: registryMetadata.nimContainer ?? null, + }); + if ( + !deps.updateSandbox( + sandboxName, + registryFields( + resolveAgentInferenceApi( + agentName, + provider, + registryMetadata.preferredInferenceApi ?? null, + ), ), - ), - ) - ) { - throw new InferenceSetError(`Failed to update NemoClaw registry for sandbox '${sandboxName}'.`); - } + ) + ) { + throw new InferenceSetError( + `Failed to update NemoClaw registry for sandbox '${sandboxName}'.`, + ); + } - const previousOpenClawInferenceApi = readPreviousOpenClawInferenceApi(agentName, config); - const preferredInferenceApi = - explicitPreferredInferenceApi ?? - resolveRuntimeInferenceApi({ - agentName, - config, - currentProvider: entry.provider, - provider, - sandboxName, - session, - }); - const effectiveRegistryMetadata: RegistryInferenceMetadata = { - ...registryMetadata, - preferredInferenceApi, - }; - // Refresh the registry with config-derived API-family metadata before the - // crash-prone in-sandbox sync (#3725/#3726). Explicit operator-supplied - // metadata remains authoritative when present. - if (!deps.updateSandbox(sandboxName, registryFields(preferredInferenceApi))) { - throw new InferenceSetError(`Failed to update NemoClaw registry for sandbox '${sandboxName}'.`); - } + const previousOpenClawInferenceApi = readPreviousOpenClawInferenceApi(agentName, config); + const preferredInferenceApi = + explicitPreferredInferenceApi ?? + resolveRuntimeInferenceApi({ + agentName, + config, + currentProvider: entry.provider, + provider, + sandboxName, + session, + }); + const effectiveRegistryMetadata: RegistryInferenceMetadata = { + ...registryMetadata, + preferredInferenceApi, + }; + // Refresh the registry with config-derived API-family metadata before the + // crash-prone in-sandbox sync (#3725/#3726). Explicit operator-supplied + // metadata remains authoritative when present. + if (!deps.updateSandbox(sandboxName, registryFields(preferredInferenceApi))) { + throw new InferenceSetError( + `Failed to update NemoClaw registry for sandbox '${sandboxName}'.`, + ); + } - let patched: { changed: boolean; route: SandboxInferenceConfig }; - if (agentName === "hermes") { - patched = patchHermesInferenceConfig(config, provider, model, preferredInferenceApi); - } else { - // Recompute the context window for the model being switched to, so it does - // not inherit the prior model's window (#context-window-on-switch). - const contextWindow = deps.resolveContextWindowForModel(provider, model); - if (contextWindow != null) { - deps.log(` Context window for '${model}': ${contextWindow} tokens`); + const currentHttpsPinRouteId = parseHttpsPinRouteId(registryMetadata.endpointUrl); + if (priorHttpsPinRouteId && priorHttpsPinRouteId !== currentHttpsPinRouteId) { + try { + const peerStillReferencesRoute = deps + .listSandboxes() + .sandboxes.some( + (candidate) => + candidate.name !== sandboxName && + parseHttpsPinRouteId(candidate.endpointUrl) === priorHttpsPinRouteId, + ); + if (!peerStillReferencesRoute) { + const revoked = await deps.revokeHttpsPinRuntimeAdapterRoute(priorHttpsPinRouteId); + if (!revoked) throw new Error("the adapter did not confirm route revocation"); + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + deps.log( + ` Warning: the new inference route is committed, but superseded HTTPS Pin Runtime route ` + + `'${priorHttpsPinRouteId}' could not be revoked: ${detail}. The raw upstream endpoint was not restored; ` + + `uninstall NemoClaw to stop the adapter and purge its in-memory credentials if this persists.`, + ); + } + } + + let patched: { changed: boolean; route: SandboxInferenceConfig }; + if (agentName === "hermes") { + patched = patchHermesInferenceConfig(config, provider, model, preferredInferenceApi); } else { - deps.log( - ` Warning: could not determine the context window for '${model}'; keeping the ` + - `existing value. Run '${CLI_NAME} ${sandboxName} rebuild' to re-probe it.`, + // Recompute the context window for the model being switched to, so it does + // not inherit the prior model's window (#context-window-on-switch). + const contextWindow = deps.resolveContextWindowForModel(provider, model); + if (contextWindow != null) { + deps.log(` Context window for '${model}': ${contextWindow} tokens`); + } else { + deps.log( + ` Warning: could not determine the context window for '${model}'; keeping the ` + + `existing value. Run '${CLI_NAME} ${sandboxName} rebuild' to re-probe it.`, + ); + } + patched = patchOpenClawInferenceConfig( + config, + provider, + model, + preferredInferenceApi || getPreferredInferenceApi(config), + contextWindow ?? undefined, + provider, ); } - patched = patchOpenClawInferenceConfig( - config, - provider, - model, - preferredInferenceApi || getPreferredInferenceApi(config), - contextWindow ?? undefined, - provider, - ); - } - deps.log( - agentName === "hermes" - ? ` Syncing Hermes model route in sandbox '${sandboxName}'...` - : ` Syncing OpenClaw model identity in sandbox '${sandboxName}'...`, - ); - // In-sandbox config is the last, crash-prone layer (gateway + registry already consistent): - // - don't abort on failure; track whether it synced, never report a false "synced" - // Two degraded states, both fixed by `rebuild` (regenerates openclaw.json + .config-hash from registry): - // - write fails: config left old (old .config-hash still matches it) - // - hash recompute fails: config new but .config-hash stale -> integrity-guard mismatch - let inSandboxConfigSynced = false; - try { - deps.writeSandboxConfig(sandboxName, target, config); + deps.log( + agentName === "hermes" + ? ` Syncing Hermes model route in sandbox '${sandboxName}'...` + : ` Syncing OpenClaw model identity in sandbox '${sandboxName}'...`, + ); + // In-sandbox config is the last, crash-prone layer (gateway + registry already consistent): + // - don't abort on failure; track whether it synced, never report a false "synced" + // Two degraded states, both fixed by `rebuild` (regenerates openclaw.json + .config-hash from registry): + // - write fails: config left old (old .config-hash still matches it) + // - hash recompute fails: config new but .config-hash stale -> integrity-guard mismatch + let inSandboxConfigSynced = false; try { - deps.recomputeSandboxConfigHash(sandboxName, target); - inSandboxConfigSynced = true; - } catch (hashError) { + deps.writeSandboxConfig(sandboxName, target, config); + try { + deps.recomputeSandboxConfigHash(sandboxName, target); + inSandboxConfigSynced = true; + } catch (hashError) { + const detail = + hashError instanceof Error && hashError.message ? hashError.message : String(hashError); + deps.log( + ` Warning: wrote the in-sandbox config for '${sandboxName}' but failed to refresh its ` + + `integrity hash: ${detail}`, + ); + deps.log(` Run '${CLI_NAME} ${sandboxName} rebuild' to resync the in-sandbox config.`); + } + } catch (writeError) { const detail = - hashError instanceof Error && hashError.message ? hashError.message : String(hashError); + writeError instanceof Error && writeError.message ? writeError.message : String(writeError); deps.log( - ` Warning: wrote the in-sandbox config for '${sandboxName}' but failed to refresh its ` + - `integrity hash: ${detail}`, + ` Warning: gateway and registry now use ${provider} / ${model}, but writing the ` + + `in-sandbox config failed: ${detail}`, ); - deps.log(` Run '${CLI_NAME} ${sandboxName} rebuild' to resync the in-sandbox config.`); - } - } catch (writeError) { - const detail = - writeError instanceof Error && writeError.message ? writeError.message : String(writeError); - deps.log( - ` Warning: gateway and registry now use ${provider} / ${model}, but writing the ` + - `in-sandbox config failed: ${detail}`, - ); - deps.log( - ` Run '${CLI_NAME} ${sandboxName} rebuild' to finish applying the model inside the sandbox.`, - ); - } - // Hermes keeps an isolated dashboard-home config that only mirrors the gateway - // config's model routing at sandbox startup. Re-seed it after an in-place - // switch so Dashboard Chat (and /api/model/info) converge on the new model - // instead of silently staying on the previous one (#6893). - // - "converged": dashboard now matches the switch. - // - "absent": Dashboard disabled — nothing to converge, still a success. - // - "failed": warn and fail after the committed mutation is finalized so - // callers cannot accept a partially converged switch. - let dashboardConverged: boolean | undefined; - if (agentName === "hermes" && inSandboxConfigSynced) { - const reseed = deps.seedHermesDashboardConfig(sandboxName, target); - dashboardConverged = reseed !== "failed"; - if (reseed === "failed") { deps.log( - ` Warning: updated the Hermes model route but could not refresh the dashboard ` + - `config for '${sandboxName}'. Restart the sandbox to converge Dashboard Chat.`, + ` Run '${CLI_NAME} ${sandboxName} rebuild' to finish applying the model inside the sandbox.`, ); } - } - const sessionUpdated = updateMatchingOnboardSession( - sandboxName, - provider, - model, - patched.route, - effectiveRegistryMetadata, - deps, - ); + // Hermes keeps an isolated dashboard-home config that only mirrors the gateway + // config's model routing at sandbox startup. Re-seed it after an in-place + // switch so Dashboard Chat (and /api/model/info) converge on the new model + // instead of silently staying on the previous one (#6893). + // - "converged": dashboard now matches the switch. + // - "absent": Dashboard disabled — nothing to converge, still a success. + // - "failed": warn and fail after the committed mutation is finalized so + // callers cannot accept a partially converged switch. + let dashboardConverged: boolean | undefined; + if (agentName === "hermes" && inSandboxConfigSynced) { + const reseed = deps.seedHermesDashboardConfig(sandboxName, target); + dashboardConverged = reseed !== "failed"; + if (reseed === "failed") { + deps.log( + ` Warning: updated the Hermes model route but could not refresh the dashboard ` + + `config for '${sandboxName}'. Restart the sandbox to converge Dashboard Chat.`, + ); + } + } + const sessionUpdated = updateMatchingOnboardSession( + sandboxName, + provider, + model, + patched.route, + effectiveRegistryMetadata, + deps, + ); - return finalizeInferenceMutation( - { - agentName, - configChanged: patched.changed, - nextApi: patched.route.inferenceApi, - previousApi: previousOpenClawInferenceApi, - result: { - sandboxName, - provider, - model, - primaryModelRef: patched.route.primaryModelRef, - providerKey: patched.route.providerKey, + return finalizeInferenceMutation( + { + agentName, configChanged: patched.changed, - sessionUpdated, - inSandboxConfigSynced, - dashboardConverged, + nextApi: patched.route.inferenceApi, + previousApi: previousOpenClawInferenceApi, + result: { + sandboxName, + provider, + model, + primaryModelRef: patched.route.primaryModelRef, + providerKey: patched.route.providerKey, + configChanged: patched.changed, + sessionUpdated, + inSandboxConfigSynced, + dashboardConverged, + }, }, - }, - deps, - ); + deps, + ); + } catch (error) { + if (!httpsPinProviderMutation) throw error; + if (restoredSelectionAfterProviderFailure) throw error; + const detail = error instanceof Error ? error.message : String(error); + const exitCode = error instanceof InferenceSetError ? error.exitCode : 1; + if (!appliedInferenceSelection) { + try { + httpsPinProviderMutation.rollback(); + } catch (rollbackError) { + const rollbackDetail = + rollbackError instanceof Error ? rollbackError.message : String(rollbackError); + throw new InferenceSetError( + `${detail}\n ${rollbackDetail} Re-run onboarding before retrying this switch.`, + exitCode, + ); + } + const unchanged = + httpsPinProviderMutation.action === "create" + ? "The newly created OpenShell provider was removed; the inference selection was not changed." + : "The existing OpenShell provider binding and inference selection were not changed."; + throw new InferenceSetError(`${detail}\n ${unchanged}`, exitCode); + } + const residual = appliedHttpsPinProvider + ? "The OpenShell provider and inference selection remain committed to the safer HTTPS-pinned adapter, but NemoClaw state may not have converged. Retry this command; if convergence still fails, rebuild the sandbox." + : "The inference selection changed, but the HTTPS-pinned provider binding did not converge. Retry this command immediately; if convergence still fails, rebuild the sandbox."; + throw new InferenceSetError(`${detail}\n ${residual}`, exitCode); + } } export async function runInferenceSet( diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 7e3c1694d13..0f34afa94d5 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -55,6 +55,20 @@ describe("destroySandbox flow", () => { expectSuccessfulLiveDestroy(harness, exitSpy); }); + it("revokes the prior HTTPS-pin route only after confirmed deletion and registry removal", async () => { + const routeId = "a".repeat(64); + const harness = createDestroyHarness({ + endpointUrl: `http://host.openshell.internal:11438/route/${routeId}`, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.revokeHttpsPinRuntimeAdapterRouteSpy).toHaveBeenCalledWith(routeId); + expect(harness.removeSandboxSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.revokeHttpsPinRuntimeAdapterRouteSpy.mock.invocationCallOrder[0], + ); + }); + it.each([ ["--yes", "darwin", { yes: true }, "", true], ["NEMOCLAW_NON_INTERACTIVE=1", "darwin", {}, "1", true], @@ -150,6 +164,7 @@ describe("destroySandbox flow", () => { // ...but shared host services are preserved on the unconfirmed delete. expect(harness.stopAllSpy).not.toHaveBeenCalled(); expect(harness.cleanupGatewaySpy).not.toHaveBeenCalled(); + expect(harness.revokeHttpsPinRuntimeAdapterRouteSpy).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/destroy-https-pin-route.test.ts b/src/lib/actions/sandbox/destroy-https-pin-route.test.ts new file mode 100644 index 00000000000..df6ce54173b --- /dev/null +++ b/src/lib/actions/sandbox/destroy-https-pin-route.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; +import { revokeDestroyedSandboxHttpsPinRoute } from "./destroy"; + +const ROUTE_ID = "a".repeat(64); +const ROUTE_URL = `http://host.openshell.internal:11438/route/${ROUTE_ID}`; +const GATEWAY_NAME = "nemoclaw-19080"; + +describe("destroy HTTPS-pin route cleanup (#6141)", () => { + it("revokes an unreferenced route after the owning registry row is gone", async () => { + const revokeRoute = vi.fn(async () => true); + + await revokeDestroyedSandboxHttpsPinRoute(GATEWAY_NAME, ROUTE_ID, { + listSandboxes: () => ({ sandboxes: [], defaultSandbox: null }), + revokeRoute, + }); + + expect(revokeRoute).toHaveBeenCalledWith(ROUTE_ID); + }); + + it("preserves a route that another sandbox still references", async () => { + const revokeRoute = vi.fn(async () => true); + + await revokeDestroyedSandboxHttpsPinRoute(GATEWAY_NAME, ROUTE_ID, { + listSandboxes: () => ({ + sandboxes: [{ name: "peer", endpointUrl: ROUTE_URL }], + defaultSandbox: "peer", + }), + revokeRoute, + }); + + expect(revokeRoute).not.toHaveBeenCalled(); + }); + + it("keeps successful sandbox deletion non-fatal when the registry read fails", async () => { + const warn = vi.fn(); + + await expect( + revokeDestroyedSandboxHttpsPinRoute(GATEWAY_NAME, ROUTE_ID, { + listSandboxes: () => { + throw new Error("registry unavailable"); + }, + revokeRoute: async () => { + throw new Error("delete unavailable"); + }, + warn, + }), + ).resolves.toBeUndefined(); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("could not be revoked")); + }); + + it("keeps successful sandbox deletion non-fatal when adapter route revocation fails", async () => { + const warn = vi.fn(); + + await expect( + revokeDestroyedSandboxHttpsPinRoute(GATEWAY_NAME, ROUTE_ID, { + listSandboxes: () => ({ sandboxes: [], defaultSandbox: null }), + revokeRoute: async () => { + throw new Error("delete unavailable"); + }, + warn, + }), + ).resolves.toBeUndefined(); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining("could not be revoked")); + }); + + it("waits for an in-flight peer route registration before deciding whether to revoke", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-destroy-route-lock-")); + const lockOptions = { stateDir, pollIntervalMs: 1, timeoutMs: 5_000 }; + let releasePeer!: () => void; + const peerReleased = new Promise((resolve) => { + releasePeer = resolve; + }); + let reportPeerEntered!: () => void; + const peerEntered = new Promise((resolve) => { + reportPeerEntered = resolve; + }); + const registryEntries: Array<{ name: string; endpointUrl: string }> = []; + const revokeRoute = vi.fn(async () => true); + const events: string[] = []; + const withTestLock: typeof withGatewayRouteMutationLock = (gatewayName, operation) => + withGatewayRouteMutationLock(gatewayName, operation, lockOptions); + + try { + // Model inference-set's critical section: its route is live after PUT but + // not discoverable by destroy until the registry commit completes. + const peerMutation = withTestLock(GATEWAY_NAME, async () => { + events.push("peer-route-put"); + reportPeerEntered(); + await peerReleased; + registryEntries.push({ name: "peer", endpointUrl: ROUTE_URL }); + events.push("peer-registry-commit"); + }); + await peerEntered; + + const destroyCleanup = revokeDestroyedSandboxHttpsPinRoute(GATEWAY_NAME, ROUTE_ID, { + listSandboxes: () => ({ sandboxes: registryEntries, defaultSandbox: "peer" }), + revokeRoute, + withGatewayRouteMutationLock: withTestLock, + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(events).toEqual(["peer-route-put"]); + expect(revokeRoute).not.toHaveBeenCalled(); + + releasePeer(); + await Promise.all([peerMutation, destroyCleanup]); + + expect(events).toEqual(["peer-route-put", "peer-registry-commit"]); + expect(revokeRoute).not.toHaveBeenCalled(); + } finally { + releasePeer(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 071b8605e87..78a9483f8cc 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -16,6 +16,9 @@ import { resolveDestroyGatewayCleanupDecision, shouldStopHostServicesAfterDestroy, } from "../../domain/sandbox/destroy"; +import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; +import { parseHttpsPinRouteId } from "../../inference/https-pin-runtime"; +import { revokeHttpsPinRuntimeAdapterRoute } from "../../inference/https-pin-runtime-adapter"; import { emitProviderDetachResidualHint, SANDBOX_PROVIDER_SUFFIXES, @@ -29,8 +32,8 @@ import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; import { confirmSandboxDestroy } from "./destroy-confirmation"; import { executeSandboxDestroy } from "./destroy-execution"; -import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gateway-cleanup"; import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; +import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gateway-cleanup"; import { prepareSandboxDestroy } from "./destroy-preflight"; import { type WipeSandboxStateDeps, wipeSandboxState } from "./wipe-state"; @@ -257,6 +260,42 @@ function defaultDestroyWarn(message: string): void { console.warn(` ${YW}⚠${R} ${message}`); } +export async function revokeDestroyedSandboxHttpsPinRoute( + gatewayName: string, + routeId: string, + deps: { + listSandboxes?: typeof registry.listSandboxes; + revokeRoute?: typeof revokeHttpsPinRuntimeAdapterRoute; + warn?: (message: string) => void; + withGatewayRouteMutationLock?: typeof withGatewayRouteMutationLock; + } = {}, +): Promise { + const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; + const revokeRoute = deps.revokeRoute ?? revokeHttpsPinRuntimeAdapterRoute; + const warn = deps.warn ?? defaultDestroyWarn; + const withRouteMutationLock = deps.withGatewayRouteMutationLock ?? withGatewayRouteMutationLock; + try { + await withRouteMutationLock(gatewayName, async () => { + // The peer scan and DELETE are one critical section with inference-set's + // route PUT + registry commit. Otherwise a peer can register the route, + // pause before its registry write, and have destroy revoke its live route. + const stillReferenced = listSandboxes().sandboxes.some( + (entry) => parseHttpsPinRouteId(entry.endpointUrl) === routeId, + ); + if (stillReferenced) return; + const revoked = await revokeRoute(routeId); + if (!revoked) throw new Error("the adapter did not confirm route revocation"); + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + warn( + `Sandbox deletion succeeded, but its superseded HTTPS Pin Runtime route '${routeId}' ` + + `could not be revoked: ${detail}. Uninstall NemoClaw after all sandboxes are removed ` + + `to stop the adapter and purge its in-memory credentials.`, + ); + } +} + export function cleanupShieldsDestroyArtifacts( sandboxName: string, deps: CleanupShieldsDestroyArtifactsDeps = {}, @@ -297,6 +336,7 @@ async function destroySandboxUnlocked( const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } = prepareSandboxDestroy(sandboxName); + const priorHttpsPinRouteId = parseHttpsPinRouteId(sandbox?.endpointUrl); const destructiveResult = await executeSandboxDestroy({ cleanupShieldsArtifacts: cleanupShieldsDestroyArtifacts, force: normalized.force === true, @@ -387,6 +427,9 @@ async function destroySandboxUnlocked( // post-removal lookups return null and would collapse the cleanup target // back to the default gateway. const removed = removeSandboxRegistryEntry(sandboxName); + if (deleteSucceededOrAlreadyGone && removed && priorHttpsPinRouteId) { + await revokeDestroyedSandboxHttpsPinRoute(cleanupGatewayName, priorHttpsPinRouteId); + } const session = onboardSession.loadSession(); if (session && session.sandboxName === sandboxName) { onboardSession.updateSession((s: Session) => { diff --git a/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.test.ts b/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.test.ts index 641a6273d4d..51dacdfafea 100644 --- a/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.test.ts +++ b/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.test.ts @@ -40,6 +40,8 @@ function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps) const OPENROUTER_RUNTIME_ADAPTER_CMDLINE = "/usr/bin/node /home/test/NemoClaw/dist/lib/inference/openrouter-runtime-adapter-entry.js\n"; +const HTTPS_PIN_RUNTIME_ADAPTER_CMDLINE = + "/usr/bin/node /home/test/NemoClaw/dist/lib/inference/https-pin-runtime-adapter.js\n"; type RunStub = (args: readonly string[]) => RunResult | null; @@ -212,3 +214,80 @@ describe("OpenRouter Runtime adapter uninstall cleanup", () => { expect(logs).toContain("No OpenRouter Runtime adapter processes found"); }); }); + +describe("HTTPS Pin Runtime adapter uninstall cleanup", () => { + it("kills the credential-bearing adapter via its verified persisted PID", () => { + const logs: string[] = []; + const killed: number[] = []; + const exited = new Set(); + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-https-pin-")); + const pidFile = path.join(tmpHome, ".nemoclaw", "https-pin-runtime-adapter.pid"); + fs.mkdirSync(path.dirname(pidFile), { recursive: true }); + fs.writeFileSync(pidFile, "44324\n"); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: tmpHome, LOGNAME: "testuser" } as NodeJS.ProcessEnv, + existsSync: (target) => target === pidFile, + isTty: false, + kill: (pid) => { + killed.push(pid); + exited.add(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: runStub({ + ps: psStub("44324", { exited, cmdline: HTTPS_PIN_RUNTIME_ADAPTER_CMDLINE }), + }), + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(killed).toContain(44324); + expect(logs).toContain("Stopped HTTPS Pin Runtime adapter 44324"); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("scans its configured port but never kills a foreign process", () => { + const killed: number[] = []; + const lsofPorts: string[] = []; + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { + HOME: "/tmp/nemoclaw-uninstall-test-https-pin-foreign", + LOGNAME: "testuser", + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT: "12038", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: (pid) => { + killed.push(pid); + return true; + }, + log: vi.fn(), + rmSync: vi.fn(), + run: runStub({ + lsof: lsofPortStub(lsofPorts, new Map([[":12038", ok("99997\n")]])), + ps: psStub("99997", { + exited: new Set(), + cmdline: "/usr/sbin/nginx -g daemon off;\n", + }), + }), + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(lsofPorts).toContain(":12038"); + expect(killed).not.toContain(99997); + }); +}); diff --git a/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.ts b/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.ts index 0c248b73c05..f03ce3fbafd 100644 --- a/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.ts +++ b/src/lib/actions/uninstall/openrouter-runtime-adapter-cleanup.ts @@ -15,7 +15,7 @@ interface RunResult { stderr: string; } -interface OpenRouterRuntimeAdapterCleanupRuntime { +interface RuntimeAdapterCleanupRuntime { commandExists: (command: string) => boolean; env: NodeJS.ProcessEnv; existsSync: (target: string) => boolean; @@ -27,6 +27,16 @@ interface OpenRouterRuntimeAdapterCleanupRuntime { const OPENROUTER_RUNTIME_ADAPTER_CMDLINE_MARK = "openrouter-runtime-adapter"; const DEFAULT_OPENROUTER_RUNTIME_ADAPTER_PORT = 11437; +const HTTPS_PIN_RUNTIME_ADAPTER_CMDLINE_MARK = "https-pin-runtime-adapter"; +const DEFAULT_HTTPS_PIN_RUNTIME_ADAPTER_PORT = 11438; + +type RuntimeAdapterDescriptor = { + cmdlineMark: string; + defaultPort: number; + envPort: string; + label: string; + pidFile: string; +}; function splitNonEmptyLines(output: string): string[] { return output @@ -35,34 +45,36 @@ function splitNonEmptyLines(output: string): string[] { .filter(Boolean); } -function resolveOpenRouterRuntimeAdapterPort( - runtime: OpenRouterRuntimeAdapterCleanupRuntime, +function resolveRuntimeAdapterPort( + runtime: RuntimeAdapterCleanupRuntime, + descriptor: RuntimeAdapterDescriptor, ): number { - const raw = runtime.env.NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT; - if (raw === undefined || raw === "") return DEFAULT_OPENROUTER_RUNTIME_ADAPTER_PORT; + const raw = runtime.env[descriptor.envPort]; + if (raw === undefined || raw === "") return descriptor.defaultPort; const trimmed = String(raw).trim(); - if (!/^\d+$/.test(trimmed)) return DEFAULT_OPENROUTER_RUNTIME_ADAPTER_PORT; + if (!/^\d+$/.test(trimmed)) return descriptor.defaultPort; const parsed = Number(trimmed); - if (parsed < 1024 || parsed > 65535) return DEFAULT_OPENROUTER_RUNTIME_ADAPTER_PORT; + if (parsed < 1024 || parsed > 65535) return descriptor.defaultPort; return parsed; } -function isOpenRouterRuntimeAdapterPid( +function isRuntimeAdapterPid( pid: number, - runtime: OpenRouterRuntimeAdapterCleanupRuntime, + runtime: RuntimeAdapterCleanupRuntime, + descriptor: RuntimeAdapterDescriptor, ): boolean { if (!Number.isInteger(pid) || pid <= 0) return false; const result = runtime.run("ps", ["-p", String(pid), "-o", "args="], { env: runtime.env }); - return result.status === 0 && result.stdout.includes(OPENROUTER_RUNTIME_ADAPTER_CMDLINE_MARK); + return result.status === 0 && result.stdout.includes(descriptor.cmdlineMark); } -function pidExists(pid: number, runtime: OpenRouterRuntimeAdapterCleanupRuntime): boolean { +function pidExists(pid: number, runtime: RuntimeAdapterCleanupRuntime): boolean { return runtime.run("ps", ["-p", String(pid), "-o", "pid="], { env: runtime.env }).status === 0; } function waitForPidExit( pid: number, - runtime: OpenRouterRuntimeAdapterCleanupRuntime, + runtime: RuntimeAdapterCleanupRuntime, timeoutMs: number, ): boolean { const deadline = Date.now() + timeoutMs; @@ -73,48 +85,47 @@ function waitForPidExit( return !pidExists(pid, runtime); } -function pidOwnedByCurrentUser( - pid: number, - runtime: OpenRouterRuntimeAdapterCleanupRuntime, -): boolean { +function pidOwnedByCurrentUser(pid: number, runtime: RuntimeAdapterCleanupRuntime): boolean { const expected = runtime.env.SUDO_USER || runtime.env.LOGNAME || os.userInfo().username; if (!expected) return true; const result = runtime.run("ps", ["-p", String(pid), "-o", "user="], { env: runtime.env }); return result.status === 0 && result.stdout.trim() === expected; } -function tryStopOpenRouterRuntimeAdapterPid( +function tryStopRuntimeAdapterPid( pid: number, - runtime: OpenRouterRuntimeAdapterCleanupRuntime, + runtime: RuntimeAdapterCleanupRuntime, + descriptor: RuntimeAdapterDescriptor, ): boolean { runtime.kill(pid); if (waitForPidExit(pid, runtime, 1000)) { - runtime.log(`Stopped OpenRouter Runtime adapter ${pid}`); + runtime.log(`Stopped ${descriptor.label} ${pid}`); return true; } runtime.kill(pid, "SIGKILL"); if (waitForPidExit(pid, runtime, 1000)) { - runtime.log(`Stopped OpenRouter Runtime adapter ${pid}`); + runtime.log(`Stopped ${descriptor.label} ${pid}`); return true; } - runtime.warn(`Failed to stop OpenRouter Runtime adapter ${pid}`); + runtime.warn(`Failed to stop ${descriptor.label} ${pid}`); return false; } -export function stopOpenRouterRuntimeAdapter( +function stopRuntimeAdapter( paths: Pick, - runtime: OpenRouterRuntimeAdapterCleanupRuntime, + runtime: RuntimeAdapterCleanupRuntime, + descriptor: RuntimeAdapterDescriptor, options: { scanOrphans?: boolean } = {}, ): void { const stopped = new Set(); - const pidFile = path.join(paths.nemoclawStateDir, "openrouter-runtime-adapter.pid"); + const pidFile = path.join(paths.nemoclawStateDir, descriptor.pidFile); if (runtime.existsSync(pidFile)) { try { const raw = fs.readFileSync(pidFile, "utf-8").trim(); const pid = Number.parseInt(raw, 10); - if (Number.isFinite(pid) && pid > 0 && isOpenRouterRuntimeAdapterPid(pid, runtime)) { - if (tryStopOpenRouterRuntimeAdapterPid(pid, runtime)) stopped.add(pid); + if (Number.isFinite(pid) && pid > 0 && isRuntimeAdapterPid(pid, runtime, descriptor)) { + if (tryStopRuntimeAdapterPid(pid, runtime, descriptor)) stopped.add(pid); } } catch { /* ignore - the State step deletes the file shortly anyway */ @@ -122,26 +133,64 @@ export function stopOpenRouterRuntimeAdapter( } if (options.scanOrphans === false) { - if (stopped.size === 0) runtime.log("No selected-gateway OpenRouter Runtime adapter found"); + if (stopped.size === 0) runtime.log(`No selected-gateway ${descriptor.label} found`); return; } if (!runtime.commandExists("lsof")) { if (stopped.size === 0) { - runtime.warn("lsof not found; skipping orphan OpenRouter Runtime adapter scan."); + runtime.warn(`lsof not found; skipping orphan ${descriptor.label} scan.`); } return; } - const adapterPort = resolveOpenRouterRuntimeAdapterPort(runtime); + const adapterPort = resolveRuntimeAdapterPort(runtime, descriptor); const lsof = runtime.run("lsof", ["-ti", `:${adapterPort}`], { env: runtime.env }); const pids = splitNonEmptyLines(lsof.stdout).map(Number).filter(Number.isFinite); for (const pid of pids) { if (stopped.has(pid)) continue; if (!pidOwnedByCurrentUser(pid, runtime)) continue; - if (!isOpenRouterRuntimeAdapterPid(pid, runtime)) continue; - if (tryStopOpenRouterRuntimeAdapterPid(pid, runtime)) stopped.add(pid); + if (!isRuntimeAdapterPid(pid, runtime, descriptor)) continue; + if (tryStopRuntimeAdapterPid(pid, runtime, descriptor)) stopped.add(pid); } - if (stopped.size === 0) runtime.log("No OpenRouter Runtime adapter processes found"); + if (stopped.size === 0) runtime.log(`No ${descriptor.label} processes found`); +} + +export function stopOpenRouterRuntimeAdapter( + paths: Pick, + runtime: RuntimeAdapterCleanupRuntime, + options: { scanOrphans?: boolean } = {}, +): void { + stopRuntimeAdapter( + paths, + runtime, + { + cmdlineMark: OPENROUTER_RUNTIME_ADAPTER_CMDLINE_MARK, + defaultPort: DEFAULT_OPENROUTER_RUNTIME_ADAPTER_PORT, + envPort: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", + label: "OpenRouter Runtime adapter", + pidFile: "openrouter-runtime-adapter.pid", + }, + options, + ); +} + +export function stopHttpsPinRuntimeAdapter( + paths: Pick, + runtime: RuntimeAdapterCleanupRuntime, + options: { scanOrphans?: boolean } = {}, +): void { + stopRuntimeAdapter( + paths, + runtime, + { + cmdlineMark: HTTPS_PIN_RUNTIME_ADAPTER_CMDLINE_MARK, + defaultPort: DEFAULT_HTTPS_PIN_RUNTIME_ADAPTER_PORT, + envPort: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + label: "HTTPS Pin Runtime adapter", + pidFile: "https-pin-runtime-adapter.pid", + }, + options, + ); } diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 41f6c7a0b15..5b517ee54dc 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -287,7 +287,19 @@ describe("uninstall gateway-port segregation (#3053)", () => { path.join(stateDir, "sandboxes.json"), JSON.stringify({ defaultSandbox: null, sandboxes: {} }), ); + const adapterStateEntries = [ + "https-pin-runtime-adapter.pid", + "https-pin-runtime-adapter-token", + "https-pin-runtime-adapter.json", + "https-pin-runtime-adapter.lock", + "https-pin-runtime-adapter.log", + ]; + for (const name of adapterStateEntries) { + fs.writeFileSync(path.join(stateDir, name), name.endsWith(".pid") ? "4242" : "state"); + } const logs: string[] = []; + const kill = vi.fn(() => true); + const run = vi.fn((_command: string, _args: string[]) => ok()); const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, { @@ -299,8 +311,9 @@ describe("uninstall gateway-port segregation (#3053)", () => { } as NodeJS.ProcessEnv, existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), isTty: false, + kill, log: (line) => logs.push(line), - run: vi.fn(() => ok()), + run, runDocker: () => ok(""), }, ); @@ -309,6 +322,17 @@ describe("uninstall gateway-port segregation (#3053)", () => { expect(fs.existsSync(path.join(otherEnv, "sandboxes.json"))).toBe(true); expect(fs.existsSync(path.join(stateDir, "sandboxes.json"))).toBe(false); expect(fs.existsSync(stateDir)).toBe(true); + for (const name of adapterStateEntries) { + expect(fs.existsSync(path.join(stateDir, name))).toBe(true); + } + expect(kill).not.toHaveBeenCalled(); + expect( + run.mock.calls.some( + ([command, args]) => + command === "ps" && JSON.stringify(args).includes("https-pin-runtime-adapter"), + ), + ).toBe(false); + expect(logs).toContain("Sibling gateways remain; kept the shared HTTPS Pin Runtime adapter."); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index 6c7bc0211d2..101f03410c4 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -341,6 +341,7 @@ describe("uninstall run plan", () => { const result = runUninstallPlan( { assumeYes: false, deleteModels: false, keepOpenShell: true }, { + commandExists: () => false, env: { HOME: "/tmp/nemoclaw-uninstall-test" } as NodeJS.ProcessEnv, log: (line) => logs.push(line), readLine: () => "no", @@ -359,6 +360,7 @@ describe("uninstall run plan", () => { const result = runUninstallPlan( { assumeYes: false, deleteModels: false, keepOpenShell: true }, { + commandExists: () => false, env: { HOME: "/tmp/nemoclaw-uninstall-test" } as NodeJS.ProcessEnv, log: (line) => logs.push(line), readLine: () => null, @@ -1014,6 +1016,11 @@ describe("uninstall run plan", () => { fs.writeFileSync(path.join(stateDir, "openrouter-runtime-adapter.json"), "{}"); fs.writeFileSync(path.join(stateDir, "openrouter-runtime-adapter.lock"), "lock"); fs.writeFileSync(path.join(stateDir, "openrouter-runtime-adapter.log"), "{}\n"); + fs.writeFileSync(path.join(stateDir, "https-pin-runtime-adapter.pid"), "1236"); + fs.writeFileSync(path.join(stateDir, "https-pin-runtime-adapter-token"), "secret"); + fs.writeFileSync(path.join(stateDir, "https-pin-runtime-adapter.json"), "{}"); + fs.writeFileSync(path.join(stateDir, "https-pin-runtime-adapter.lock"), "lock"); + fs.writeFileSync(path.join(stateDir, "https-pin-runtime-adapter.log"), "{}\n"); fs.mkdirSync(path.join(stateDir, "source")); return { tmpHome, stateDir }; } @@ -1079,6 +1086,11 @@ describe("uninstall run plan", () => { expect(fs.existsSync(path.join(stateDir, "openrouter-runtime-adapter.json"))).toBe(false); expect(fs.existsSync(path.join(stateDir, "openrouter-runtime-adapter.lock"))).toBe(false); expect(fs.existsSync(path.join(stateDir, "openrouter-runtime-adapter.log"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "https-pin-runtime-adapter.pid"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "https-pin-runtime-adapter-token"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "https-pin-runtime-adapter.json"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "https-pin-runtime-adapter.lock"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "https-pin-runtime-adapter.log"))).toBe(false); expect(fs.existsSync(path.join(stateDir, "source"))).toBe(false); expect(logs).toContain( `Preserving rebuild-backups, backups, sandboxes.json under ${stateDir}.`, diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 9c863c52c6e..4e6d7eb3a6f 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -51,7 +51,10 @@ import { } from "../../state/gateway-registry"; import { GATEWAYS_SUBDIR } from "../../state/state-root"; import { stopHermesForwardWatchers } from "./hermes-forward-watcher-cleanup"; -import { stopOpenRouterRuntimeAdapter } from "./openrouter-runtime-adapter-cleanup"; +import { + stopHttpsPinRuntimeAdapter, + stopOpenRouterRuntimeAdapter, +} from "./openrouter-runtime-adapter-cleanup"; import { classifyShimPath, type FileSystemDeps } from "./plan"; export interface RunResult { @@ -227,6 +230,14 @@ const PRESERVED_USER_DATA_ENTRIES: readonly string[] = [ "sandboxes.json", ]; +const HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES: readonly string[] = [ + "https-pin-runtime-adapter.pid", + "https-pin-runtime-adapter-token", + "https-pin-runtime-adapter.json", + "https-pin-runtime-adapter.lock", + "https-pin-runtime-adapter.log", +]; + // These entries can exist in the shared root without representing a running // default-port environment. Any other shared-root entry is treated // conservatively as default-port state when uninstalling a non-default port. @@ -235,6 +246,7 @@ const SHARED_HOST_STATE_ENTRIES = new Set([ "source", GATEWAYS_SUBDIR, "managed_swap", + ...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES, ]); function removePathExcept( @@ -1559,6 +1571,11 @@ function executePlan( stopOpenRouterRuntimeAdapter(paths, runtime, { scanOrphans: !scopedToSelectedGateway, }); + if (scopedToSelectedGateway) { + runtime.log("Sibling gateways remain; kept the shared HTTPS Pin Runtime adapter."); + } else { + stopHttpsPinRuntimeAdapter(paths, runtime); + } stopModelRouter(paths, runtime, !scopedToSelectedGateway); } else if (step.name === "OpenShell resources") { if ( @@ -1670,6 +1687,7 @@ function executePlan( ? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)] : []), ...(scopedToSelectedGateway && selectedIsDefault ? ["source"] : []), + ...(scopedToSelectedGateway ? HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES : []), ], runtime, ) diff --git a/src/lib/core/ports.test.ts b/src/lib/core/ports.test.ts index 024f4929f5a..e1417040540 100644 --- a/src/lib/core/ports.test.ts +++ b/src/lib/core/ports.test.ts @@ -3,7 +3,12 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; // Import source directly so tests cannot pass against a stale build. -import { parseGatewayPort, parsePort, validateOpenRouterRuntimeAdapterPort } from "./ports"; +import { + parseGatewayPort, + parsePort, + validateHttpsPinRuntimeAdapterPort, + validateOpenRouterRuntimeAdapterPort, +} from "./ports"; const GATEWAY_VALIDATION_OPTIONS = { dashboardPort: 18789, @@ -15,6 +20,7 @@ const GATEWAY_VALIDATION_OPTIONS = { ollamaProxyPort: 11435, bedrockRuntimeAdapterPort: 11436, openrouterRuntimeAdapterPort: 11437, + httpsPinRuntimeAdapterPort: 11438, }; describe("parsePort", () => { @@ -117,6 +123,7 @@ describe("parseGatewayPort", () => { ["11435", "Ollama auth proxy"], ["11436", "Bedrock Runtime adapter"], ["11437", "OpenRouter Runtime adapter"], + ["11438", "HTTPS Pin Runtime adapter"], ])("rejects overlap with default port %s", (port, label) => { process.env[ENV_KEY] = port; expect(() => parseGatewayPort(ENV_KEY, 8080, GATEWAY_VALIDATION_OPTIONS)).toThrow(label); @@ -141,6 +148,16 @@ describe("parseGatewayPort", () => { }), ).toThrow("NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT"); }); + + it("rejects overlap with a configured HTTPS Pin Runtime adapter port", () => { + process.env[ENV_KEY] = "19004"; + expect(() => + parseGatewayPort(ENV_KEY, 8080, { + ...GATEWAY_VALIDATION_OPTIONS, + httpsPinRuntimeAdapterPort: 19004, + }), + ).toThrow("NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT"); + }); }); describe("validateOpenRouterRuntimeAdapterPort", () => { @@ -158,6 +175,7 @@ describe("validateOpenRouterRuntimeAdapterPort", () => { [11434, "Ollama inference"], [11435, "Ollama auth proxy"], [11436, "Bedrock Runtime adapter"], + [11438, "HTTPS Pin Runtime adapter"], [18790, "18789-18799"], ])("rejects OpenRouter adapter overlap with %s", (port, expectedMessage) => { expect(() => @@ -174,3 +192,36 @@ describe("validateOpenRouterRuntimeAdapterPort", () => { ).toThrow("NEMOCLAW_VLLM_PORT"); }); }); + +describe("validateHttpsPinRuntimeAdapterPort", () => { + const ENV_KEY = "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT"; + + it("allows the default HTTPS Pin Runtime adapter port", () => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, 11438, GATEWAY_VALIDATION_OPTIONS), + ).not.toThrow(); + }); + + it.each([ + [8080, "NEMOCLAW_GATEWAY_PORT"], + [8000, "vLLM / NIM inference"], + [11434, "Ollama inference"], + [11435, "Ollama auth proxy"], + [11436, "Bedrock Runtime adapter"], + [11437, "OpenRouter Runtime adapter"], + [18790, "18789-18799"], + ])("rejects HTTPS Pin adapter overlap with %s", (port, expectedMessage) => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, port, GATEWAY_VALIDATION_OPTIONS), + ).toThrow(expectedMessage); + }); + + it("rejects HTTPS Pin adapter overlap with configured service ports", () => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, 19001, { + ...GATEWAY_VALIDATION_OPTIONS, + vllmPort: 19001, + }), + ).toThrow("NEMOCLAW_VLLM_PORT"); + }); +}); diff --git a/src/lib/core/ports.ts b/src/lib/core/ports.ts index 7cd9ff0da50..31ddb4985f9 100644 --- a/src/lib/core/ports.ts +++ b/src/lib/core/ports.ts @@ -33,6 +33,7 @@ export interface GatewayPortValidationOptions { ollamaProxyPort: number; bedrockRuntimeAdapterPort: number; openrouterRuntimeAdapterPort: number; + httpsPinRuntimeAdapterPort: number; } export interface RuntimeAdapterPortValidationOptions extends GatewayPortValidationOptions { @@ -71,6 +72,11 @@ export const OPENROUTER_RUNTIME_ADAPTER_PORT = parsePort( "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", 11437, ); +/** HTTPS DNS-pinning reverse-proxy adapter port (default 11438, override via NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT). */ +export const HTTPS_PIN_RUNTIME_ADAPTER_PORT = parsePort( + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + 11438, +); export function validateGatewayPort( envVar: string, @@ -89,6 +95,7 @@ export function validateGatewayPort( { label: "Ollama auth proxy", port: 11435 }, { label: "Bedrock Runtime adapter", port: 11436 }, { label: "OpenRouter Runtime adapter", port: 11437 }, + { label: "HTTPS Pin Runtime adapter", port: 11438 }, ]; const reservedDefault = reservedDefaults.find((entry) => entry.port === port); if (reservedDefault) { @@ -110,6 +117,10 @@ export function validateGatewayPort( envVar: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", port: options.openrouterRuntimeAdapterPort, }, + { + envVar: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + port: options.httpsPinRuntimeAdapterPort, + }, ]; const conflict = conflicts.find((entry) => entry.port === port); if (conflict) { @@ -145,6 +156,55 @@ export function validateOpenRouterRuntimeAdapterPort( { label: "Ollama inference", port: 11434 }, { label: "Ollama auth proxy", port: 11435 }, { label: "Bedrock Runtime adapter", port: 11436 }, + { label: "HTTPS Pin Runtime adapter", port: 11438 }, + ]; + const reservedDefault = reservedDefaults.find((entry) => entry.port === port); + if (reservedDefault) { + throw new Error( + `Invalid port: ${envVar}="${port}" — must not overlap the ${reservedDefault.label} default port (${reservedDefault.port})`, + ); + } + + const conflicts = [ + { envVar: "NEMOCLAW_GATEWAY_PORT", port: options.gatewayPort }, + { envVar: "NEMOCLAW_DASHBOARD_PORT", port: options.dashboardPort }, + { envVar: "NEMOCLAW_VLLM_PORT", port: options.vllmPort }, + { envVar: "NEMOCLAW_OLLAMA_PORT", port: options.ollamaPort }, + { envVar: "NEMOCLAW_OLLAMA_PROXY_PORT", port: options.ollamaProxyPort }, + { + envVar: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", + port: options.bedrockRuntimeAdapterPort, + }, + { + envVar: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + port: options.httpsPinRuntimeAdapterPort, + }, + ]; + const conflict = conflicts.find((entry) => entry.port === port); + if (conflict) { + throw new Error( + `Invalid port: ${envVar}="${port}" — conflicts with ${conflict.envVar} (${conflict.port})`, + ); + } +} + +export function validateHttpsPinRuntimeAdapterPort( + envVar: string, + port: number, + options: RuntimeAdapterPortValidationOptions, +): void { + if (port >= options.dashboardRangeStart && port <= options.dashboardRangeEnd) { + throw new Error( + `Invalid port: ${envVar}="${port}" — must not overlap the ${options.dashboardRangeStart}-${options.dashboardRangeEnd} dashboard port range`, + ); + } + + const reservedDefaults = [ + { label: "vLLM / NIM inference", port: 8000 }, + { label: "Ollama inference", port: 11434 }, + { label: "Ollama auth proxy", port: 11435 }, + { label: "Bedrock Runtime adapter", port: 11436 }, + { label: "OpenRouter Runtime adapter", port: 11437 }, ]; const reservedDefault = reservedDefaults.find((entry) => entry.port === port); if (reservedDefault) { @@ -163,6 +223,10 @@ export function validateOpenRouterRuntimeAdapterPort( envVar: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", port: options.bedrockRuntimeAdapterPort, }, + { + envVar: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", + port: options.openrouterRuntimeAdapterPort, + }, ]; const conflict = conflicts.find((entry) => entry.port === port); if (conflict) { @@ -184,4 +248,5 @@ export const GATEWAY_PORT = parseGatewayPort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GA ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }); diff --git a/src/lib/inference/https-pin-runtime-adapter-forward.test.ts b/src/lib/inference/https-pin-runtime-adapter-forward.test.ts new file mode 100644 index 00000000000..7ae4faea4cb --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter-forward.test.ts @@ -0,0 +1,449 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import type { AddressInfo } from "node:net"; + +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; + +import { + type CaMaterial, + cleanupCaSetup, + resolveCaSetup, + startTlsServer, +} from "../../../test/helpers/corporate-ca-support"; +import { + describeForwardHttpError, + ForwardHttpError, + forwardHttpsPinnedRequest, + HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES, + type HttpsPinTarget, +} from "./https-pin-runtime-adapter-forward"; + +const servers: http.Server[] = []; +const tlsServers: Array<{ close: () => Promise }> = []; + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); + servers.length = 0; + await Promise.all(tlsServers.map((server) => server.close())); + tlsServers.length = 0; +}); + +function listen(server: http.Server): Promise<{ baseUrl: string; port: number }> { + servers.push(server); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve({ baseUrl: `http://127.0.0.1:${address.port}`, port: address.port }); + }); + }); +} + +const TEST_CREDENTIAL = { name: "x-api-key", value: "secret-upstream-credential" }; + +describe("HTTPS-pin forwarding error responses (#6141)", () => { + it("maps an unrecognized error status to the fixed upstream-failure response", () => { + expect(describeForwardHttpError(new ForwardHttpError(599, "untrusted", "untrusted"))).toEqual({ + status: 502, + code: "untrusted", + message: "untrusted", + }); + }); +}); + +/** A minimal server that forwards every request through `forwardHttpsPinnedRequest` against `target`. */ +function createForwardTestServer( + target: HttpsPinTarget, + options: { upstreamTimeoutMs?: number; bodyTimeoutMs?: number } = {}, +): http.Server { + return http.createServer(async (req, res) => { + const url = new URL(req.url || "/", "http://127.0.0.1"); + await forwardHttpsPinnedRequest({ + req, + res, + forwardPath: url.pathname + url.search, + target, + upstreamTimeoutMs: options.upstreamTimeoutMs, + bodyTimeoutMs: options.bodyTimeoutMs, + }); + }); +} + +function readRequestBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +describe("forwardHttpsPinnedRequest header handling (#6141)", () => { + it("connects to the pinned address while sending the real hostname as Host, and injects the upstream credential", async () => { + const upstreamRequests: Array<{ + headers: http.IncomingHttpHeaders; + body: string; + url: string | undefined; + }> = []; + const upstream = http.createServer(async (req, res) => { + upstreamRequests.push({ + headers: req.headers, + body: await readRequestBody(req), + url: req.url, + }); + res.writeHead(200, { "Content-Type": "application/json", "X-Upstream-Marker": "yes" }); + res.end(JSON.stringify({ ok: true })); + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base/chat?trace=1`, { + method: "POST", + headers: { + Authorization: "Bearer client-supplied-should-be-dropped", + "Content-Type": "application/json", + "X-Trace-Id": "abc123", + }, + body: JSON.stringify({ hello: "world" }), + }); + + expect(response.status).toBe(200); + expect(response.headers.get("x-upstream-marker")).toBe("yes"); + expect(upstreamRequests).toHaveLength(1); + const [seen] = upstreamRequests; + // Host reflects the real target hostname (with its non-default port), + // not the pinned connect address. + expect(seen.headers.host).toBe(`forward-test.example:${upstreamPort}`); + // The adapter's own credential is what reaches upstream... + expect(seen.headers["x-api-key"]).toBe(TEST_CREDENTIAL.value); + // ...and the client-supplied Authorization header never does. + expect(seen.headers.authorization).toBeUndefined(); + expect(seen.headers["x-trace-id"]).toBe("abc123"); + expect(seen.url).toBe("/base/chat?trace=1"); + expect(seen.body).toBe(JSON.stringify({ hello: "world" })); + }); + + it("rejects a request body over the size limit before contacting upstream", async () => { + const upstreamHandler = () => { + throw new Error("upstream must not be contacted for an oversized body"); + }; + const upstream = http.createServer(upstreamHandler); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { + method: "POST", + body: "x".repeat(HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES + 1), + }); + + expect(response.status).toBe(413); + await expect(response.json()).resolves.toMatchObject({ error: { code: "request_too_large" } }); + }); + + it("delivers the 408 timeout body to the client instead of hanging up the shared socket (#6141)", async () => { + const upstreamHandler = () => { + throw new Error("upstream must not be contacted for a stalled request body"); + }; + const upstream = http.createServer(upstreamHandler); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { bodyTimeoutMs: 50 }); + const { port: adapterPort } = await listen(adapter); + + // A raw request that declares a body but never finishes sending it, so + // the adapter's body-read timeout fires instead of the client ever + // completing the write. Destroying `req` before `res` flushes (the bug + // this test guards) would tear down the shared socket and the client + // would see the connection drop instead of a 408 body. + const response = await new Promise<{ status: number | undefined; body: string }>( + (resolve, reject) => { + const req = http.request( + { + host: "127.0.0.1", + port: adapterPort, + path: "/base", + method: "POST", + headers: { "content-length": "100" }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + res.on("end", () => { + resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString("utf8") }); + }); + res.on("error", reject); + }, + ); + req.on("error", reject); + // Fewer bytes than the declared content-length, and `.end()` is + // deliberately never called. + req.write("partial-body"); + }, + ); + + expect(response.status).toBe(408); + expect(JSON.parse(response.body)).toMatchObject({ error: { code: "request_timeout" } }); + }); + + it("cancels the pinned upstream request when the client disconnects before the response finishes (#6141)", async () => { + let upstreamRequestSocket: import("node:net").Socket | undefined; + let resolveUpstreamClosed: () => void; + const upstreamClosed = new Promise((resolve) => { + resolveUpstreamClosed = resolve; + }); + const upstream = http.createServer((req, res) => { + upstreamRequestSocket = req.socket; + req.socket.once("close", () => resolveUpstreamClosed()); + res.writeHead(200, { "Content-Type": "application/json" }); + res.write('{"partial":true'); + // Never call res.end(): the upstream response is left open so the + // only way this promise resolves is via the adapter destroying the + // pinned outbound connection after the client disconnects. + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 30_000 }); + const { port: adapterPort } = await listen(adapter); + + await new Promise((resolve, reject) => { + const clientReq = http.request( + { host: "127.0.0.1", port: adapterPort, path: "/base", method: "POST" }, + (res) => { + res.once("data", () => { + // Simulate the original client abandoning the request once it + // has started receiving a response. + clientReq.destroy(); + resolve(); + }); + res.once("error", () => resolve()); + }, + ); + clientReq.on("error", () => { + /* destroying our own request triggers this; expected. */ + }); + clientReq.end("{}"); + setTimeout(() => reject(new Error("timed out waiting for client response data")), 2000); + }); + + // The pinned outbound connection must be torn down promptly, well under + // the 30s default/configured upstream timeout, rather than lingering + // until the abandoned upstream response finishes on its own. + await expect( + Promise.race([ + upstreamClosed, + new Promise((_, reject) => + setTimeout(() => reject(new Error("upstream connection was not canceled in time")), 2000), + ), + ]), + ).resolves.toBeUndefined(); + expect(upstreamRequestSocket?.destroyed).toBe(true); + }); + + it("times out a stalled upstream response without hanging (#6141)", async () => { + const upstream = http.createServer(async (req) => { + await readRequestBody(req); + // Never responds. + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 50 }); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + expect(response.status).toBe(504); + await expect(response.json()).resolves.toMatchObject({ error: { code: "upstream_timeout" } }); + }); + + it("bounds the upstream deadline before connection establishment (#6141)", async () => { + const upstreamReq = new EventEmitter() as unknown as http.ClientRequest; + const destroy = vi.fn((err?: Error) => { + err ? upstreamReq.emit("error", err) : undefined; + return upstreamReq; + }); + (upstreamReq as unknown as { end: unknown }).end = vi.fn(); + (upstreamReq as unknown as { destroy: unknown }).destroy = destroy; + vi.spyOn(http, "request").mockImplementation(() => upstreamReq); + + const target: HttpsPinTarget = { + targetUrl: new URL("http://forward-test.example/base"), + pinnedAddress: "192.0.2.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 25 }); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + + expect(response.status).toBe(504); + await expect(response.json()).resolves.toMatchObject({ error: { code: "upstream_timeout" } }); + expect(destroy).toHaveBeenCalled(); + }); + + it("discards a late upstream response after the deadline settles (#6141)", async () => { + const upstreamReq = new EventEmitter() as unknown as http.ClientRequest; + (upstreamReq as unknown as { end: unknown }).end = vi.fn(); + (upstreamReq as unknown as { destroy: unknown }).destroy = vi.fn((err?: Error) => { + err ? upstreamReq.emit("error", err) : undefined; + return upstreamReq; + }); + let responseCallback: ((res: http.IncomingMessage) => void) | undefined; + vi.spyOn(http, "request").mockImplementation(((...args: unknown[]) => { + responseCallback = args[args.length - 1] as (res: http.IncomingMessage) => void; + return upstreamReq; + }) as typeof http.request); + + const target: HttpsPinTarget = { + targetUrl: new URL("http://forward-test.example/base"), + pinnedAddress: "192.0.2.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 20 }); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + expect(response.status).toBe(504); + + const lateResponse = new EventEmitter() as unknown as http.IncomingMessage; + (lateResponse as unknown as { statusCode: number }).statusCode = 200; + (lateResponse as unknown as { headers: http.IncomingHttpHeaders }).headers = {}; + const lateDestroy = vi.fn(); + (lateResponse as unknown as { destroy: unknown }).destroy = lateDestroy; + (lateResponse as unknown as { pipe: unknown }).pipe = vi.fn(); + + expect(() => responseCallback?.(lateResponse)).not.toThrow(); + expect(lateDestroy).toHaveBeenCalled(); + }); +}); + +describe("forwardHttpsPinnedRequest redirect fail-closed (#6141)", () => { + it.each([ + 301, 302, 303, 307, 308, + ])("blocks a %i upstream redirect instead of following or relaying it", async (status) => { + const upstream = http.createServer(async (req, res) => { + await readRequestBody(req); + res.writeHead(status, { Location: "http://169.254.169.254/latest/meta-data/" }); + res.end(); + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + expect(response.status).toBe(502); + const body = (await response.json()) as { error: { code: string } }; + expect(body.error.code).toBe("redirect_blocked"); + // The attacker-influenced Location header must never reach the client. + expect(response.headers.get("location")).toBeNull(); + expect(JSON.stringify(body)).not.toContain("169.254.169.254"); + }); +}); + +const sniPinSetup = resolveCaSetup("https-pin-runtime-adapter-forward SNI pinning"); + +afterAll(() => cleanupCaSetup(sniPinSetup)); + +describe.skipIf(!sniPinSetup.ok)("forwardHttpsPinnedRequest TLS SNI pinning (#6141)", () => { + const ca = sniPinSetup as CaMaterial; + + it("validates the certificate against the real target hostname while connecting to the pinned address", async () => { + const tlsServer = await startTlsServer(ca.serverKey, ca.serverCert); + tlsServers.push(tlsServer); + + const trustedAgent = new https.Agent({ ca: fs.readFileSync(ca.corporateCaCert) }); + const originalAgent = https.globalAgent; + https.globalAgent = trustedAgent; + try { + const target: HttpsPinTarget = { + // The leaf cert's SAN covers "localhost"; connecting via the pinned + // loopback address (not a fresh DNS lookup of the hostname) must still + // validate against this real hostname through TLS SNI. + targetUrl: new URL(`https://localhost:${tlsServer.port}/`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/`, { method: "POST", body: "{}" }); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ ok: true }); + } finally { + https.globalAgent = originalAgent; + trustedAgent.destroy(); + } + }); + + it("fails closed when the pinned target hostname is not covered by the upstream certificate", async () => { + const tlsServer = await startTlsServer(ca.serverKey, ca.serverCert); + tlsServers.push(tlsServer); + + const trustedAgent = new https.Agent({ ca: fs.readFileSync(ca.corporateCaCert) }); + const originalAgent = https.globalAgent; + https.globalAgent = trustedAgent; + try { + const target: HttpsPinTarget = { + // Same server/cert as the positive case, but a hostname the leaf + // certificate does not cover: certificate hostname verification must + // still reject this, proving the pin never disables verification. + targetUrl: new URL(`https://not-the-real-host.invalid:${tlsServer.port}/`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/`, { method: "POST", body: "{}" }); + expect(response.status).toBe(502); + } finally { + https.globalAgent = originalAgent; + trustedAgent.destroy(); + } + }); +}); diff --git a/src/lib/inference/https-pin-runtime-adapter-forward.ts b/src/lib/inference/https-pin-runtime-adapter-forward.ts new file mode 100644 index 00000000000..115976d9eeb --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter-forward.ts @@ -0,0 +1,313 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import http from "node:http"; +import https from "node:https"; + +import { compactText } from "../core/url-utils"; +import type { HttpsPinCredentialHeader } from "./https-pin-runtime"; + +export const HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES = 2 * 1024 * 1024; +const HTTPS_PIN_RUNTIME_ADAPTER_BODY_TIMEOUT_MS = 30_000; +const HTTPS_PIN_RUNTIME_ADAPTER_UPSTREAM_TIMEOUT_MS = 30_000; + +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "authorization", + "x-api-key", +]); + +export class ForwardHttpError extends Error { + constructor( + readonly status: number, + message: string, + readonly code: string, + ) { + super(message); + } +} + +export function describeForwardHttpError(err: unknown): { + status: number; + code: string; + message: string; +} { + if (!(err instanceof ForwardHttpError)) { + return { + status: 502, + code: "https_pin_runtime_error", + message: "Upstream request failed.", + }; + } + // Keep the client-facing status on a closed set of literal values. Some + // errors originate in Node's upstream HTTP stack, so an attacker-influenced + // object must never become a dynamic writeHead status/reason argument. + let status: number; + switch (err.status) { + case 400: + status = 400; + break; + case 404: + status = 404; + break; + case 408: + status = 408; + break; + case 413: + status = 413; + break; + case 504: + status = 504; + break; + default: + status = 502; + } + return { status, code: err.code, message: err.message }; +} + +/** + * The pinned outbound peer for one forwarded request: the validated public + * address to connect to, and the real hostname to present as TLS SNI / send + * as the Host header, so certificate validation still targets the real host + * while the TCP connection goes to the address the SSRF preflight validated. + */ +export interface HttpsPinTarget { + targetUrl: URL; + pinnedAddress: string; + credential: HttpsPinCredentialHeader; +} + +export function buildForwardRequestHeaders( + req: http.IncomingMessage, + credential: HttpsPinCredentialHeader, +): http.OutgoingHttpHeaders { + const headers: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (value === undefined || HOP_BY_HOP_HEADERS.has(name.toLowerCase())) continue; + headers[name] = value; + } + headers[credential.name] = credential.value; + return headers; +} + +function buildForwardResponseHeaders(source: http.IncomingHttpHeaders): http.OutgoingHttpHeaders { + const headers: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(source)) { + if (value === undefined || HOP_BY_HOP_HEADERS.has(name.toLowerCase())) continue; + headers[name] = value; + } + return headers; +} + +function readBoundedRequestBody( + req: http.IncomingMessage, + bodyTimeoutMs = HTTPS_PIN_RUNTIME_ADAPTER_BODY_TIMEOUT_MS, +): Promise { + return new Promise((resolve, reject) => { + const contentLength = Number(req.headers["content-length"] || 0); + if ( + Number.isFinite(contentLength) && + contentLength > HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES + ) { + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + // Destroying `req` here (rather than leaving that to the caller) would + // tear down the same underlying socket `res` needs to flush the 408 + // response on -- the client would see a dead connection instead of the + // documented JSON body. The caller destroys `req` itself, after the + // error response finishes writing. + req.removeAllListeners("data"); + reject(new ForwardHttpError(408, "Request body timed out.", "request_timeout")); + }, bodyTimeoutMs); + + req.on("data", (chunk: Buffer) => { + if (settled) return; + size += chunk.length; + if (size > HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES) { + settled = true; + clearTimeout(timer); + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + chunks.push(Buffer.from(chunk)); + }); + req.on("end", () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(Buffer.concat(chunks)); + }); + req.on("error", (err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(err); + }); + }); +} + +export function sendForwardError( + res: http.ServerResponse, + err: unknown, + req?: http.IncomingMessage, +): number { + const { status, code, message } = describeForwardHttpError(err); + // Only the body-read timeout leaves the client still writing indefinitely + // -- every other rejection (e.g. an oversized body) responds without + // needing the rest of the client's upload, and destroying the shared + // socket there would cut off a still-in-flight client write (EPIPE) + // instead of letting it drain normally. + const shouldDestroyRequest = Boolean(req) && code === "request_timeout"; + if (!res.headersSent) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: { message: compactText(message), type: code, code }, + }), + // Only destroy the request socket once the error response has finished + // writing, not before -- `req` and `res` share the same underlying + // socket, so destroying `req` any earlier would take the response down + // with it. + () => { + if (shouldDestroyRequest && !req?.destroyed) req?.destroy(); + }, + ); + } else { + res.destroy(err instanceof Error ? err : undefined); + } + return status; +} + +/** + * Forward one request to a pinned HTTPS peer: connects to `pinnedAddress` + * (the address the SSRF preflight already validated) while sending TLS SNI + * and the Host header for the real target hostname, so certificate + * validation still targets the real host — the Node equivalent of curl + * `--resolve` with strict hostname verification preserved. HTTP targets + * connect directly (no pinning needed; the address itself was already + * validated and substituted upstream of this adapter). + * + * Fails closed on any 3xx upstream response: a redirect is never followed or + * relayed, since a `Location` header is attacker-influenced content that + * could point at an internal address, silently defeating the pin. + */ +export async function forwardHttpsPinnedRequest(options: { + req: http.IncomingMessage; + res: http.ServerResponse; + forwardPath: string; + target: HttpsPinTarget; + upstreamTimeoutMs?: number; + bodyTimeoutMs?: number; +}): Promise { + const { req, res, forwardPath, target } = options; + let body: Buffer; + try { + body = await readBoundedRequestBody(req, options.bodyTimeoutMs); + } catch (err) { + return sendForwardError(res, err, req); + } + + const isHttps = target.targetUrl.protocol === "https:"; + const transport = isHttps ? https : http; + const port = target.targetUrl.port ? Number(target.targetUrl.port) : isHttps ? 443 : 80; + + return new Promise((resolve) => { + let settled = false; + let absoluteDeadline: NodeJS.Timeout | undefined; + const resolveOnce = (status: number) => { + if (settled) return; + settled = true; + if (absoluteDeadline) clearTimeout(absoluteDeadline); + res.off("close", onClientClose); + resolve(status); + }; + const failRequest = (err: unknown) => { + // Once the client-facing response is already finalized (normally, or + // via onClientClose below), res is no longer safe to write to. + if (settled) return; + resolveOnce(sendForwardError(res, err)); + }; + + const headers = buildForwardRequestHeaders(req, target.credential); + // `.host` (not `.hostname`) so a non-default port on the real endpoint is + // preserved in the Host header; TLS SNI below correctly stays bare + // hostname-only since SNI has no port component. + headers.host = target.targetUrl.host; + headers["content-length"] = String(body.length); + + const upstreamReq = transport.request( + { + hostname: target.pinnedAddress, + port, + path: forwardPath, + method: req.method, + headers, + ...(isHttps ? { servername: target.targetUrl.hostname } : {}), + }, + (upstreamRes) => { + if (settled || res.destroyed) { + upstreamRes.destroy(); + return; + } + const status = upstreamRes.statusCode || 502; + if (status >= 300 && status < 400) { + upstreamRes.resume(); + failRequest( + new ForwardHttpError( + 502, + "Upstream redirect blocked: the pinned adapter does not follow or relay redirects.", + "redirect_blocked", + ), + ); + return; + } + res.writeHead(status, buildForwardResponseHeaders(upstreamRes.headers)); + upstreamRes.once("close", () => { + if (!upstreamRes.readableEnded) { + failRequest( + new ForwardHttpError(502, "Upstream response aborted.", "upstream_response_aborted"), + ); + } + }); + upstreamRes.once("error", failRequest); + upstreamRes.pipe(res); + upstreamRes.once("end", () => resolveOnce(status)); + }, + ); + // If the original client disconnects before the response finishes, the + // pinned outbound connection would otherwise keep streaming from the real + // upstream until it finishes on its own or the upstream timeout fires -- + // an abandoned client could hold a pinned connection open indefinitely. + const onClientClose = () => { + if (res.writableEnded) return; + upstreamReq.destroy(); + resolveOnce(0); + }; + res.once("close", onClientClose); + absoluteDeadline = setTimeout(() => { + upstreamReq.destroy( + new ForwardHttpError(504, "Upstream request timed out.", "upstream_timeout"), + ); + }, options.upstreamTimeoutMs ?? HTTPS_PIN_RUNTIME_ADAPTER_UPSTREAM_TIMEOUT_MS); + upstreamReq.on("error", (err) => { + failRequest(err); + }); + upstreamReq.end(body); + }); +} diff --git a/src/lib/inference/https-pin-runtime-adapter.test.ts b/src/lib/inference/https-pin-runtime-adapter.test.ts new file mode 100644 index 00000000000..1cb2dc54e53 --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter.test.ts @@ -0,0 +1,1398 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import http from "node:http"; +import net, { type AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { EndpointDnsLookupFn } from "./endpoint-ssrf-preflight"; +import { + __test, + createHttpsPinRuntimeAdapterServer, + ensureHttpsPinRuntimeAdapter, + revokeHttpsPinRuntimeAdapterRoute, +} from "./https-pin-runtime-adapter"; + +const servers: http.Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); + servers.length = 0; +}); + +function listen(server: http.Server): Promise { + servers.push(server); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${address.port}`); + }); + }); +} + +const TEST_CONTROL_TOKEN = "test-control-plane-token"; +const TEST_ROUTE_GENERATION = "11111111111111111111111111111111"; + +function routeToken(routeId: string, generation = TEST_ROUTE_GENERATION): string { + return __test.deriveRouteToken(TEST_CONTROL_TOKEN, routeId, generation); +} + +function sendRawHttpMethod( + baseUrl: string, + method: string, + requestPath: string, + headers: Record = {}, +): Promise { + const url = new URL(baseUrl); + return new Promise((resolve, reject) => { + const socket = net.createConnection(Number(url.port), url.hostname, () => { + const serializedHeaders = Object.entries({ + Host: url.host, + Connection: "close", + ...headers, + }) + .map(([name, value]) => `${name}: ${value}`) + .join("\r\n"); + socket.write(`${method} ${requestPath} HTTP/1.1\r\n${serializedHeaders}\r\n\r\n`); + }); + let response = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk) => { + response += chunk; + }); + socket.on("end", () => { + const match = response.match(/^HTTP\/1\.1 (\d{3})/u); + match + ? resolve(Number(match[1])) + : reject(new Error(`Invalid raw HTTP response: ${response}`)); + }); + socket.on("error", reject); + }); +} + +describe("createHttpsPinRuntimeAdapterServer health and auth (#6141)", () => { + it("derives stable, distinct route tokens without exposing the control secret", () => { + const routeAFirst = routeToken("route-a"); + const routeASecond = routeToken("route-a"); + const routeB = routeToken("route-b"); + const routeANextGeneration = routeToken("route-a", "22222222222222222222222222222222"); + + expect(routeAFirst).toBe(routeASecond); + expect(routeAFirst).not.toBe(routeB); + expect(routeAFirst).not.toBe(routeANextGeneration); + expect(routeAFirst).not.toBe(TEST_CONTROL_TOKEN); + expect(routeAFirst).toMatch(/^[a-f0-9]{64}$/); + }); + + it("exposes safe health over loopback without leaking the token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/health`); + expect(response.status).toBe(200); + const body = (await response.json()) as { ok: boolean; routeCount: number; tokenHash?: string }; + expect(body).toMatchObject({ ok: true, routeCount: 0 }); + expect(body.tokenHash).toBeUndefined(); + expect(JSON.stringify(body)).not.toContain(TEST_CONTROL_TOKEN); + }); + + it("proves control-plane identity with a fresh challenge without transmitting the token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + const port = Number(new URL(baseUrl).port); + + await expect( + __test.probeAdapterControlHealth({ controlToken: TEST_CONTROL_TOKEN, port }), + ).resolves.toBe(true); + await expect( + __test.probeAdapterControlHealth({ controlToken: "wrong-control-token", port }), + ).resolves.toBe(false); + }); + + it("binds adapter reuse proof to the inspected OpenShell source policy", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + allowedSourceCidrs: ["172.17.0.0/16"], + }); + const baseUrl = await listen(adapter); + const port = Number(new URL(baseUrl).port); + + await expect( + __test.probeAdapterControlHealth({ + controlToken: TEST_CONTROL_TOKEN, + expectedSourceCidrs: ["172.17.0.0/16"], + port, + }), + ).resolves.toBe(true); + await expect( + __test.probeAdapterControlHealth({ + controlToken: TEST_CONTROL_TOKEN, + expectedSourceCidrs: ["192.168.50.0/24"], + port, + }), + ).resolves.toBe(false); + }); + + it.each([ + ["protocol version", { ...__test.CURRENT_ADAPTER_IDENTITY, protocolVersion: "stale-protocol" }], + ["build id", { ...__test.CURRENT_ADAPTER_IDENTITY, buildId: "0".repeat(64) }], + ])("rejects a token-authenticated adapter with a stale %s", async (_label, staleIdentity) => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + adapterIdentity: staleIdentity, + }); + const baseUrl = await listen(adapter); + const port = Number(new URL(baseUrl).port); + + // The stale process can still prove control-token possession for its own + // identity, but the current build must not reuse it. + await expect( + __test.probeAdapterControlHealth({ + controlToken: TEST_CONTROL_TOKEN, + expectedIdentity: staleIdentity, + port, + }), + ).resolves.toBe(true); + await expect( + __test.findReusableAdapterControlToken(TEST_CONTROL_TOKEN, ["127.0.0.1/32"], (options) => + __test.probeAdapterControlHealth({ ...options, port }), + ), + ).resolves.toBeNull(); + }); + + it("does not trust an impostor that replays the former public token hash", async () => { + const seenRequests: Array<{ headers: http.IncomingHttpHeaders; url?: string }> = []; + const oldPublicHash = crypto.createHash("sha256").update(TEST_CONTROL_TOKEN).digest("hex"); + const impostor = http.createServer((req, res) => { + seenRequests.push({ headers: req.headers, url: req.url }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, tokenHash: oldPublicHash })); + }); + const baseUrl = await listen(impostor); + + await expect( + __test.probeAdapterControlHealth({ + controlToken: TEST_CONTROL_TOKEN, + nonce: "a".repeat(64), + port: Number(new URL(baseUrl).port), + }), + ).resolves.toBe(false); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0].url).toBe(`/control/health?nonce=${"a".repeat(64)}`); + expect(seenRequests[0].headers.authorization).toBeUndefined(); + expect(JSON.stringify(seenRequests[0])).not.toContain(TEST_CONTROL_TOKEN); + }); + + it("applies an absolute control-probe deadline even when an impostor drips bytes", async () => { + const impostor = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + const interval = setInterval(() => res.write(" "), 10); + res.once("close", () => clearInterval(interval)); + }); + const baseUrl = await listen(impostor); + const started = Date.now(); + + await expect( + __test.probeAdapterControlHealth({ + controlToken: TEST_CONTROL_TOKEN, + port: Number(new URL(baseUrl).port), + timeoutMs: 60, + }), + ).resolves.toBe(false); + expect(Date.now() - started).toBeLessThan(500); + }); + + it("rejects control-plane and route requests without a valid bearer token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + initialRoutes: { + anything: { + targetBaseUrl: "https://real-upstream.example/v1", + pinnedAddresses: ["93.184.216.34"], + providerType: "openai", + credentialValue: "sk-upstream", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + + const missingAuth = await fetch(`${baseUrl}/route/anything`); + expect(missingAuth.status).toBe(401); + + const wrongAuth = await fetch(`${baseUrl}/route/anything`, { + headers: { Authorization: "Bearer wrong-token" }, + }); + expect(wrongAuth.status).toBe(401); + + const body = (await wrongAuth.json()) as { error: { code: string } }; + expect(body.error.code).toBe("unauthorized"); + }); + + it("returns 404 for an unknown path", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/nonexistent`, { + headers: { Authorization: `Bearer ${TEST_CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(404); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer control plane (#6141)", () => { + it("registers an HTTPS route via the authenticated control plane", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const putResponse = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: "https://real-upstream.example/base", + pinnedAddresses: ["93.184.216.34"], + providerType: "openai", + credentialValue: "sk-upstream-secret", + generation: TEST_ROUTE_GENERATION, + }), + }); + expect(putResponse.status).toBe(200); + await expect(putResponse.json()).resolves.toEqual({ ok: true, routeId: "route-1" }); + + const health = await fetch(`${baseUrl}/health`); + await expect(health.json()).resolves.toMatchObject({ routeCount: 1 }); + }); + + it("uses the anthropic credential header shape for an anthropic route", async () => { + const upstreamRequests: Array<{ headers: http.IncomingHttpHeaders }> = []; + const upstream = http.createServer(async (req, res) => { + upstreamRequests.push({ headers: req.headers }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + initialRoutes: { + "route-anthropic": { + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "anthropic", + credentialValue: "sk-ant-secret", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/route/route-anthropic/v1/messages`, { + method: "POST", + headers: { + "x-api-key": routeToken("route-anthropic"), + "Content-Type": "application/json", + }, + body: "{}", + }); + + expect(response.status).toBe(200); + expect(upstreamRequests[0].headers["x-api-key"]).toBe("sk-ant-secret"); + expect(upstreamRequests[0].headers.authorization).toBeUndefined(); + }); + + it("rejects TRACE, TRACK, and CONNECT without sending injected credentials upstream", async () => { + let upstreamRequests = 0; + const upstream = http.createServer((_req, res) => { + upstreamRequests += 1; + res.writeHead(200); + res.end(); + }); + const upstreamPort = new URL(await listen(upstream)).port; + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + initialRoutes: { + guarded: { + targetBaseUrl: `https://real-upstream.example:${upstreamPort}/v1`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-must-not-be-echoed", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + const headers = { Authorization: `Bearer ${routeToken("guarded")}` }; + + await expect( + sendRawHttpMethod(baseUrl, "TRACE", "/route/guarded/chat/completions", headers), + ).resolves.toBe(405); + const trackStatus = await sendRawHttpMethod( + baseUrl, + "TRACK", + "/route/guarded/chat/completions", + headers, + ); + // Some Node builds reject TRACK in the HTTP parser with 400 before the + // request handler's explicit 405 guard runs. Both outcomes fail closed. + expect([400, 405]).toContain(trackStatus); + await expect( + sendRawHttpMethod(baseUrl, "CONNECT", "/route/guarded/chat/completions", headers), + ).resolves.toBe(405); + expect(upstreamRequests).toBe(0); + }); + + it("invalidates the prior data token when a revoked route id is registered again", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200); + res.end(); + }); + const upstreamPort = new URL(await listen(upstream)).port; + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + const firstGeneration = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const secondGeneration = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + const register = (generation: string) => + fetch(`${baseUrl}/control/routes/reused`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: `https://real-upstream.example:${upstreamPort}/v1`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream", + generation, + }), + }); + + expect((await register(firstGeneration)).status).toBe(200); + const firstToken = routeToken("reused", firstGeneration); + expect( + ( + await fetch(`${baseUrl}/route/reused/models`, { + headers: { Authorization: `Bearer ${firstToken}` }, + }) + ).status, + ).toBe(502); + + const revoked = await fetch(`${baseUrl}/control/routes/reused`, { + method: "DELETE", + headers: { Authorization: `Bearer ${TEST_CONTROL_TOKEN}` }, + }); + expect(revoked.status).toBe(200); + expect((await register(secondGeneration)).status).toBe(200); + + const secondToken = routeToken("reused", secondGeneration); + expect(secondToken).not.toBe(firstToken); + expect( + ( + await fetch(`${baseUrl}/route/reused/models`, { + headers: { Authorization: `Bearer ${firstToken}` }, + }) + ).status, + ).toBe(401); + expect( + ( + await fetch(`${baseUrl}/route/reused/models`, { + headers: { Authorization: `Bearer ${secondToken}` }, + }) + ).status, + ).toBe(502); + }); + + it("binds each sandbox credential to exactly one route and keeps the control token off data paths", async () => { + const routeARequests: http.IncomingHttpHeaders[] = []; + const routeBRequests: http.IncomingHttpHeaders[] = []; + const routeAPaths: string[] = []; + const routeBPaths: string[] = []; + const adapterEvents: unknown[] = []; + const upstreamA = http.createServer((req, res) => { + routeARequests.push(req.headers); + routeAPaths.push(req.url || ""); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ route: "a" })); + }); + const upstreamB = http.createServer((req, res) => { + routeBRequests.push(req.headers); + routeBPaths.push(req.url || ""); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ route: "b" })); + }); + const upstreamAPort = new URL(await listen(upstreamA)).port; + const upstreamBPort = new URL(await listen(upstreamB)).port; + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + logger: (event, fields) => adapterEvents.push({ event, fields }), + initialRoutes: { + "route-a": { + targetBaseUrl: `http://real-upstream.example:${upstreamAPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-route-a", + generation: TEST_ROUTE_GENERATION, + }, + "route-b": { + targetBaseUrl: `http://real-upstream.example:${upstreamBPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "anthropic", + credentialValue: "sk-route-b", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + + const tokenA = routeToken("route-a"); + const tokenB = routeToken("route-b"); + expect(tokenA).not.toBe(tokenB); + expect(tokenA).not.toBe(TEST_CONTROL_TOKEN); + + const crossRouteReplay = await fetch(`${baseUrl}/route/route-b/chat/completions`, { + headers: { "x-api-key": tokenA }, + }); + expect(crossRouteReplay.status).toBe(401); + + const controlTokenReplay = await fetch(`${baseUrl}/route/route-a/chat/completions`, { + headers: { Authorization: `Bearer ${TEST_CONTROL_TOKEN}` }, + }); + expect(controlTokenReplay.status).toBe(401); + const anthropicControlTokenReplay = await fetch(`${baseUrl}/route/route-b/messages`, { + headers: { "x-api-key": TEST_CONTROL_TOKEN }, + }); + expect(anthropicControlTokenReplay.status).toBe(401); + + const openAiCrossHeader = await fetch(`${baseUrl}/route/route-a/chat/completions`, { + headers: { "x-api-key": tokenA }, + }); + expect(openAiCrossHeader.status).toBe(401); + const anthropicCrossHeader = await fetch(`${baseUrl}/route/route-b/messages`, { + headers: { Authorization: `Bearer ${tokenB}` }, + }); + expect(anthropicCrossHeader.status).toBe(401); + + const dataTokenOnControlPlane = await fetch(`${baseUrl}/control/routes/route-a`, { + method: "PUT", + headers: { Authorization: `Bearer ${tokenA}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream.example:${upstreamAPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-replay", + generation: TEST_ROUTE_GENERATION, + }), + }); + expect(dataTokenOnControlPlane.status).toBe(401); + expect(routeARequests).toHaveLength(0); + expect(routeBRequests).toHaveLength(0); + + const validA = await fetch(`${baseUrl}/route/route-a/chat/completions`, { + headers: { Authorization: `Bearer ${tokenA}` }, + }); + const validB = await fetch(`${baseUrl}/route/route-b/messages`, { + headers: { "x-api-key": tokenB }, + }); + expect(validA.status).toBe(200); + expect(validB.status).toBe(200); + expect(routeARequests[0].authorization).toBe("Bearer sk-route-a"); + expect(routeBRequests[0]["x-api-key"]).toBe("sk-route-b"); + expect(routeAPaths[0]).toBe("/base/chat/completions"); + expect(routeBPaths[0]).toBe("/base/messages"); + + const logText = JSON.stringify(adapterEvents); + expect(logText).not.toContain(TEST_CONTROL_TOKEN); + expect(logText).not.toContain(tokenA); + expect(logText).not.toContain(tokenB); + expect(logText).not.toContain("sk-route-a"); + expect(logText).not.toContain("sk-route-b"); + expect(logText).not.toContain("real-upstream.example"); + expect(logText).not.toContain("/base"); + }); + + it.each([ + "/route/scoped/v1/../admin", + "/route/scoped/v1/%2e%2e/admin", + "/route/scoped/v1/%2E%2e/admin", + "/route/scoped/v1%2f..%2fadmin", + "/route/scoped/v1%2F..%2Fadmin", + "/route/scoped/v1/%252e%252e/admin", + "/route/scoped/v1/%252E%252fadmin", + ])("rejects raw or encoded traversal before joining the target base path: %s", (requestPath) => { + expect(() => + __test.buildContainedForwardPath( + { + targetBaseUrl: "https://real-upstream.example/v1", + pinnedAddresses: ["93.184.216.34"], + providerType: "openai", + credentialValue: "not-used", + generation: TEST_ROUTE_GENERATION, + }, + "/admin", + "", + requestPath, + ), + ).toThrow("Route path not found"); + }); + + it.each([ + ["/", "/v1/chat/completions", "/v1/chat/completions"], + ["/v1", "/chat/completions?trace=1", "/v1/chat/completions?trace=1"], + ["/v1", "/v1/chat/completions", "/v1/chat/completions"], + ["/gateway/v1", "/v1/chat/completions", "/gateway/v1/chat/completions"], + ["/v1beta/openai", "/v1/chat/completions", "/v1beta/openai/chat/completions"], + ["/tenant/v1/chat", "/v1/chat/completions", "/tenant/v1/chat/chat/completions"], + ["/tenant/messages", "/messages", "/tenant/messages/messages"], + ["/v1", "/admin", "/v1/admin"], + ["/v1", "/v10/chat/completions", "/v1/v10/chat/completions"], + ])("translates only the OpenAI gateway prefix when joining target base %s with suffix %s", async (targetPath, suffix, expectedPath) => { + const upstreamPaths: string[] = []; + const upstream = http.createServer((req, res) => { + upstreamPaths.push(req.url || ""); + res.writeHead(200); + res.end(); + }); + const upstreamPort = new URL(await listen(upstream)).port; + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + initialRoutes: { + scoped: { + targetBaseUrl: `http://real-upstream.example:${upstreamPort}${targetPath}`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-scoped", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/route/scoped${suffix}`, { + headers: { Authorization: `Bearer ${routeToken("scoped")}` }, + }); + + expect(response.status).toBe(200); + expect(upstreamPaths).toEqual([expectedPath]); + }); + + it("preserves the Anthropic v1 API suffix when joining its target base", () => { + expect( + __test.buildContainedForwardPath( + { + targetBaseUrl: "https://real-upstream.example/base", + pinnedAddresses: ["93.184.216.34"], + providerType: "anthropic", + credentialValue: "not-used", + generation: TEST_ROUTE_GENERATION, + }, + "/v1/messages", + "?trace=1", + "/route/scoped/v1/messages?trace=1", + ), + ).toBe("/base/v1/messages?trace=1"); + }); + + it("seeds routes from initialRoutes at construction, before any PUT", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + initialRoutes: { + "bootstrap-route": { + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-bootstrap", + generation: TEST_ROUTE_GENERATION, + }, + }, + }); + const baseUrl = await listen(adapter); + + const health = await fetch(`${baseUrl}/health`); + await expect(health.json()).resolves.toMatchObject({ routeCount: 1 }); + + const response = await fetch(`${baseUrl}/route/bootstrap-route/`, { + headers: { Authorization: `Bearer ${routeToken("bootstrap-route")}` }, + }); + expect(response.status).toBe(200); + }); + + it("rejects PUT bodies missing required fields with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ targetBaseUrl: "http://example.com/" }), + }); + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "invalid_route" } }); + }); + + it("rejects PUT bodies with an unparseable targetBaseUrl with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: "not-a-url", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-secret", + generation: TEST_ROUTE_GENERATION, + }), + }); + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "invalid_route" } }); + }); + + it("rejects a cleartext HTTP target before storing its upstream credential", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: "http://cleartext.example/v1", + pinnedAddresses: ["93.184.216.34"], + providerType: "openai", + credentialValue: "sk-must-not-cross-cleartext", + generation: TEST_ROUTE_GENERATION, + }), + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "invalid_route" } }); + }); + + it("rejects PUT bodies with an unsupported providerType with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "gemini", + credentialValue: "sk-secret", + generation: TEST_ROUTE_GENERATION, + }), + }); + expect(response.status).toBe(400); + }); + + it("rejects oversized control-plane bodies with 413", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "x".repeat(20 * 1024), + generation: TEST_ROUTE_GENERATION, + }), + }); + expect(response.status).toBe(413); + await expect(response.json()).resolves.toMatchObject({ error: { code: "request_too_large" } }); + }); + + it("returns 404 for a GET on the control-routes path (PUT only)", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + headers: { Authorization: `Bearer ${TEST_CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(404); + }); + + it("returns 404 route_not_found for an unregistered route id", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/route/never-registered`, { + headers: { Authorization: `Bearer ${routeToken("never-registered")}` }, + }); + expect(response.status).toBe(404); + await expect(response.json()).resolves.toMatchObject({ error: { code: "route_not_found" } }); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer orphaned route recovery (#6141)", () => { + it("authenticates an orphan with its stable route token after a same-control-token respawn", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + orphanedRoutes: { + "orphan-1": { providerType: "openai", generation: TEST_ROUTE_GENERATION }, + }, + }); + const baseUrl = await listen(adapter); + + const orphaned = await fetch(`${baseUrl}/route/orphan-1/v1/messages`, { + headers: { Authorization: `Bearer ${routeToken("orphan-1")}` }, + }); + expect(orphaned.status).toBe(503); + await expect(orphaned.json()).resolves.toMatchObject({ + error: { code: "route_needs_recovery" }, + }); + + const neverKnown = await fetch(`${baseUrl}/route/never-known/v1/messages`, { + headers: { Authorization: `Bearer ${routeToken("never-known")}` }, + }); + expect(neverKnown.status).toBe(404); + await expect(neverKnown.json()).resolves.toMatchObject({ error: { code: "route_not_found" } }); + }); + + it("prefers a live route over its own stale orphaned-route id once re-registered", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + orphanedRoutes: { + "healed-route": { providerType: "openai", generation: TEST_ROUTE_GENERATION }, + }, + }); + const baseUrl = await listen(adapter); + + await fetch(`${baseUrl}/control/routes/healed-route`, { + method: "PUT", + headers: { + Authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + targetBaseUrl: `https://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-healed", + generation: TEST_ROUTE_GENERATION, + }), + }); + + const response = await fetch(`${baseUrl}/route/healed-route/`, { + headers: { Authorization: `Bearer ${routeToken("healed-route")}` }, + }); + expect(response.status).toBe(502); + }); +}); + +// Drives the server's request listener directly with a fake req/res instead +// of a real socket, so the simulated `remoteAddress` isn't at the mercy of +// how (or whether) a given host/CI sandbox routes secondary loopback +// addresses like 127.0.0.2 -- only the literal connection identity matters +// to `isLoopbackRemoteAddress`, not real network delivery. +function dispatchFakeRequest( + server: http.Server, + options: { + method: string; + url: string; + remoteAddress: string; + authorization?: string; + body?: unknown; + }, +): Promise<{ status: number; body: unknown }> { + const listener = server.listeners("request")[0] as ( + req: http.IncomingMessage, + res: http.ServerResponse, + ) => unknown; + + const req = new EventEmitter() as unknown as http.IncomingMessage; + Object.assign(req, { + method: options.method, + url: options.url, + headers: options.authorization ? { authorization: options.authorization } : {}, + socket: { remoteAddress: options.remoteAddress }, + }); + + return new Promise((resolve) => { + let status = 0; + const res = { + writeHead(code: number) { + status = code; + }, + end(payload?: string) { + resolve({ status, body: payload ? JSON.parse(payload) : undefined }); + }, + } as unknown as http.ServerResponse; + + void listener(req, res); + queueMicrotask(() => { + const chunks = options.body === undefined ? [] : [Buffer.from(JSON.stringify(options.body))]; + for (const chunk of chunks) (req as unknown as EventEmitter).emit("data", chunk); + (req as unknown as EventEmitter).emit("end"); + }); + }); +} + +describe("createHttpsPinRuntimeAdapterServer control-plane loopback restriction (#6141)", () => { + it("rejects a route registration whose connection did not arrive over loopback", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + + // The container-gateway address the sandbox actually connects from when + // it reaches the adapter through `host.openshell.internal` -- distinct + // from the literal 127.0.0.1 the host process itself always dials from. + const response = await dispatchFakeRequest(adapter, { + method: "PUT", + url: "/control/routes/route-1", + remoteAddress: "172.17.0.2", + authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + body: { + targetBaseUrl: "http://internal.example/base", + pinnedAddresses: ["10.0.0.5"], + providerType: "openai", + credentialValue: "sk-should-not-register", + generation: TEST_ROUTE_GENERATION, + }, + }); + expect(response.status).toBe(404); + + const health = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/health", + remoteAddress: "172.17.0.2", + }); + expect(health.status).toBe(404); + expect(health.body).toMatchObject({ error: { code: "not_found" } }); + }); + + it("still allows route registration over loopback", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + + const response = await dispatchFakeRequest(adapter, { + method: "PUT", + url: "/control/routes/route-1", + remoteAddress: "127.0.0.1", + authorization: `Bearer ${TEST_CONTROL_TOKEN}`, + body: { + targetBaseUrl: "https://real-upstream.example/base", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream-secret", + generation: TEST_ROUTE_GENERATION, + }, + }); + expect(response.status).toBe(200); + expect(response.body).toEqual({ ok: true, routeId: "route-1" }); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer OpenShell bridge source restriction (#6141)", () => { + // These drive the gate itself, so an unregistered route ID is enough: a + // request that passes the private-network gate falls through to the + // "route_not_found" lookup (which never pipes a real upstream response), + // while a request blocked by the gate never reaches that lookup at all and + // instead gets the gate's own "not_found" code. + + it("rejects a route-forward request whose connection arrives from a public address", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + + // A peer that reached this 0.0.0.0-bound port from outside the intended + // Docker-bridge sandbox boundary -- an address the adapter should never + // trust a replayed bearer token from. 203.0.113.0/24 is the reserved + // TEST-NET-3 documentation range (RFC 5737), never a real bridge subnet. + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "203.0.113.5", + authorization: `Bearer ${routeToken("never-registered")}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "not_found" } }); + }); + + it("still passes a route-forward request from the Docker-bridge sandbox address through to route lookup", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + allowedSourceCidrs: ["172.17.0.0/16"], + }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "172.17.0.2", + authorization: `Bearer ${routeToken("never-registered")}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "route_not_found" } }); + }); + + it("rejects a private peer outside the inspected OpenShell bridge subnet", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + allowedSourceCidrs: ["172.17.0.0/16"], + }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "192.168.50.8", + authorization: `Bearer ${routeToken("never-registered")}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "not_found" } }); + }); + + it("hides health metadata from a private peer outside the inspected bridge subnet", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: TEST_CONTROL_TOKEN, + allowedSourceCidrs: ["172.17.0.0/16"], + }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/health", + remoteAddress: "192.168.50.8", + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "not_found" } }); + expect(response.body).not.toHaveProperty("routeCount"); + }); + + it("still passes a route-forward request over loopback through to route lookup", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: TEST_CONTROL_TOKEN }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "127.0.0.1", + authorization: `Bearer ${routeToken("never-registered")}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "route_not_found" } }); + }); +}); + +describe("discoverOpenShellBridgeSourceCidrs (#6141)", () => { + it("accepts only validated subnets from the inspected OpenShell Docker network", () => { + const capture = vi.fn(() => + JSON.stringify([ + { Subnet: "172.17.0.0/16", Gateway: "172.17.0.1" }, + { Subnet: "fd00:1234::/64", Gateway: "fd00:1234::1" }, + { Subnet: "not-a-cidr" }, + ]), + ) as unknown as NonNullable[0]>; + + expect(__test.discoverOpenShellBridgeSourceCidrs(capture)).toEqual([ + "172.17.0.0/16", + "fd00:1234::/64", + ]); + expect(capture).toHaveBeenCalledWith( + ["docker", "network", "inspect", "openshell-docker", "--format", "{{json .IPAM.Config}}"], + { ignoreError: true }, + ); + }); + + it("fails closed when the OpenShell bridge has no valid source subnet", () => { + const capture = vi.fn(() => "[]") as unknown as NonNullable< + Parameters[0] + >; + + expect(() => __test.discoverOpenShellBridgeSourceCidrs(capture)).toThrow( + /refusing to expose the credential-bearing HTTPS Pin Runtime adapter/, + ); + }); +}); + +describe("adapter recovery lock (#6141)", () => { + // The statically-imported `__test.LOCK_PATH` above is derived from this + // machine's real os.homedir() at module-evaluation time, same as a real, + // possibly-concurrently-running adapter's lock. Acquiring/deleting it here + // could steal or wedge that live adapter's lock. Give each test its own + // HOME (and therefore its own LOCK_PATH under a fresh temp `.nemoclaw`) via + // vi.resetModules() plus a fresh dynamic import, since STATE_DIR is only + // ever read once, at import time. + let tempHome: string; + let lockModule: typeof import("./https-pin-runtime-adapter"); + + beforeEach(async () => { + tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-adapter-lock-test-")); + vi.stubEnv("HOME", tempHome); + vi.resetModules(); + lockModule = await import("./https-pin-runtime-adapter"); + }); + + afterEach(() => { + try { + fs.unlinkSync(lockModule.__test.LOCK_PATH); + } catch { + /* nothing to clean up */ + } + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("blocks a second acquire while the first holder has not released", () => { + const release = lockModule.__test.tryAcquireAdapterLock(); + expect(release).not.toBeNull(); + expect(lockModule.__test.tryAcquireAdapterLock()).toBeNull(); + release?.(); + expect(lockModule.__test.tryAcquireAdapterLock()).not.toBeNull(); + }); + + it("serializes concurrent withAdapterLock operations instead of interleaving them", async () => { + const order: string[] = []; + const slow = lockModule.__test.withAdapterLock(async () => { + order.push("slow:start"); + await new Promise((resolve) => setTimeout(resolve, 50)); + order.push("slow:end"); + }); + // Give `slow` a head start so it wins the lock first. + await new Promise((resolve) => setTimeout(resolve, 5)); + const fast = lockModule.__test.withAdapterLock(async () => { + order.push("fast:start"); + order.push("fast:end"); + }); + + await Promise.all([slow, fast]); + + expect(order).toEqual(["slow:start", "slow:end", "fast:start", "fast:end"]); + }); + + it("reuses a token-authenticated live adapter without requiring PID metadata", async () => { + const probeHealth = vi.fn(async () => true); + + await expect( + lockModule.__test.findReusableAdapterControlToken( + "persisted-control-token", + ["127.0.0.1/32"], + probeHealth, + ), + ).resolves.toBe("persisted-control-token"); + expect(probeHealth).toHaveBeenCalledWith({ + controlToken: "persisted-control-token", + expectedIdentity: __test.CURRENT_ADAPTER_IDENTITY, + expectedSourceCidrs: ["127.0.0.1/32"], + }); + }); + + it("deletes a route from an authenticated live adapter even when PID metadata is missing", async () => { + const deleteRoute = vi.fn(async () => {}); + const removeRouteState = vi.fn(); + + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => null, + readControlToken: () => "persisted-control-token", + probeHealth: async () => true, + deleteRoute, + isAdapterProcess: () => false, + removeRouteState, + }), + ).resolves.toBe(true); + expect(deleteRoute).toHaveBeenCalledWith("persisted-control-token", "a".repeat(64)); + expect(removeRouteState).toHaveBeenCalledWith("a".repeat(64)); + expect(deleteRoute.mock.invocationCallOrder[0]).toBeLessThan( + removeRouteState.mock.invocationCallOrder[0], + ); + }); + + it("preserves persisted state when the authenticated route DELETE fails", async () => { + const removeRouteState = vi.fn(); + + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => null, + readControlToken: () => "persisted-control-token", + probeHealth: async () => true, + deleteRoute: async () => { + throw new Error("delete failed"); + }, + isAdapterProcess: () => false, + removeRouteState, + }), + ).rejects.toThrow("delete failed"); + expect(removeRouteState).not.toHaveBeenCalled(); + }); + + it("preserves both route metadata updates when registration transactions overlap", async () => { + let persistedRoutes: Record = {}; + const register = (routeId: string, providerType: "openai" | "anthropic", delayMs: number) => + lockModule.__test.withAdapterLock(async () => { + const snapshot = { ...persistedRoutes }; + await new Promise((resolve) => setTimeout(resolve, delayMs)); + persistedRoutes = { ...snapshot, [routeId]: { providerType } }; + }); + + await Promise.all([register("route-a", "openai", 30), register("route-b", "anthropic", 0)]); + + expect(persistedRoutes).toEqual({ + "route-a": { providerType: "openai" }, + "route-b": { providerType: "anthropic" }, + }); + }); + + it("persists only opaque recovery metadata without source URL, pins, or credentials", () => { + const controlToken = "host-only-control-secret"; + const dataToken = lockModule.__test.deriveRouteToken( + controlToken, + "route-a", + TEST_ROUTE_GENERATION, + ); + const upstreamCredential = "real-upstream-secret"; + lockModule.__test.persistRouteState("route-a", { + providerType: "openai", + generation: TEST_ROUTE_GENERATION, + registeredAt: "2026-07-18T00:00:00.000Z", + }); + + const stateText = fs.readFileSync(lockModule.__test.STATE_PATH, "utf8"); + expect(stateText).not.toContain(controlToken); + expect(stateText).not.toContain(dataToken); + expect(stateText).not.toContain(upstreamCredential); + expect(stateText).not.toContain("public.example.test"); + expect(stateText).not.toContain("/v1"); + expect(stateText).not.toContain("93.184.216.34"); + expect(JSON.parse(stateText)).toMatchObject({ + routes: { + "route-a": { providerType: "openai", generation: TEST_ROUTE_GENERATION }, + }, + }); + }); + + it("waits for a terminated adapter PID to exit before allowing replacement spawn", async () => { + const observations = [true, true, false]; + const isRunning = vi.fn(() => observations.shift() ?? false); + const sleep = vi.fn(async () => {}); + + await expect( + lockModule.__test.waitForAdapterProcessExit(12345, { + isRunning, + sleep, + attempts: 4, + intervalMs: 25, + }), + ).resolves.toBe(true); + expect(isRunning).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledWith(25); + }); + + it("refuses replacement when the old adapter never exits within the bounded wait", async () => { + const sleep = vi.fn(async () => {}); + + await expect( + lockModule.__test.waitForAdapterProcessExit(12345, { + isRunning: () => true, + sleep, + attempts: 3, + intervalMs: 25, + }), + ).resolves.toBe(false); + expect(sleep).toHaveBeenCalledTimes(2); + }); +}); + +describe("revokeHttpsPinRuntimeAdapterRoute input validation (#6141)", () => { + it("rejects malformed route ids before touching lifecycle state", async () => { + await expect(revokeHttpsPinRuntimeAdapterRoute("../control/routes/other")).rejects.toThrow( + "invalid HTTPS Pin Runtime route id", + ); + }); +}); + +describe("computeRespawnState orphaned-route bookkeeping (#6141)", () => { + it("records the source limitation and removal condition for orphan recovery", () => { + expect(__test.ORPHANED_ROUTE_RECOVERY_BOUNDARY).toEqual({ + whyNotSourceFix: + "Durable recovery metadata intentionally omits the upstream URL, pinned addresses, and credential; only the owning inference set caller can supply all three.", + removalCondition: + "Retire orphaning/manual re-registration only when a reviewed secure recovery source or capability can rehydrate every registered route after respawn without persisting plaintext credentials, exposing them to OpenShell or a sandbox, or weakening per-route token and pinned-address isolation.", + }); + }); + + it("marks every persisted route except the one being bootstrapped as orphaned", () => { + const priorRoutes = { + a: { + providerType: "openai", + generation: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + registeredAt: "2026-07-18T00:00:00.000Z", + targetBaseUrl: "http://a.example/secret-path", + pinnedAddresses: ["10.0.0.1"], + }, + b: { + providerType: "openai", + generation: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + registeredAt: "2026-07-18T00:00:00.000Z", + }, + c: { + providerType: "anthropic", + generation: "cccccccccccccccccccccccccccccccc", + registeredAt: "2026-07-18T00:00:00.000Z", + }, + }; + + const { orphanedRoutes, persistedRoutes } = __test.computeRespawnState(priorRoutes, "b"); + + expect(orphanedRoutes).toEqual({ + a: { providerType: "openai", generation: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + c: { providerType: "anthropic", generation: "cccccccccccccccccccccccccccccccc" }, + }); + expect(Object.keys(persistedRoutes).sort()).toEqual(["a", "c"]); + expect(persistedRoutes.a).toMatchObject({ providerType: "openai" }); + expect(JSON.stringify(persistedRoutes)).not.toContain("a.example"); + expect(JSON.stringify(persistedRoutes)).not.toContain("10.0.0.1"); + expect(typeof persistedRoutes.a.orphanedAt).toBe("string"); + expect(persistedRoutes.c).toMatchObject({ providerType: "anthropic" }); + expect(typeof persistedRoutes.c.orphanedAt).toBe("string"); + expect(persistedRoutes.b).toBeUndefined(); + }); + + it("orphans nothing when there is no prior state to recover from", () => { + const { orphanedRoutes, persistedRoutes } = __test.computeRespawnState({}, "bootstrap-only"); + + expect(orphanedRoutes).toEqual({}); + expect(persistedRoutes).toEqual({}); + }); +}); + +describe("HTTPS Pin Runtime adapter child environment (#6141)", () => { + it("keeps upstream route credentials out of the long-lived child environment", () => { + const env = __test.buildAdapterChildEnv(TEST_CONTROL_TOKEN, ["172.17.0.0/16"], { + orphaned: { + providerType: "openai", + generation: TEST_ROUTE_GENERATION, + }, + }); + + expect(env).not.toHaveProperty("NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_BOOTSTRAP_ROUTE"); + expect(env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ALLOWED_SOURCE_CIDRS).toBe('["172.17.0.0/16"]'); + expect(JSON.stringify(env)).not.toContain("sk-upstream-secret"); + expect(JSON.stringify(env)).not.toContain("real-upstream.example"); + }); + + it("accepts only a validated JSON array of bridge source CIDRs", () => { + expect(__test.parseAllowedSourceCidrs('["172.17.0.0/16","fd00::/64"]')).toEqual([ + "172.17.0.0/16", + "fd00::/64", + ]); + expect(__test.parseAllowedSourceCidrs('["not-a-cidr"]')).toEqual([]); + expect(__test.parseAllowedSourceCidrs('{"cidr":"172.17.0.0/16"}')).toEqual([]); + }); +}); + +describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#6141)", () => { + const privateLookup: EndpointDnsLookupFn = async () => [{ address: "10.48.203.205", family: 4 }]; + const publicLookup: EndpointDnsLookupFn = async () => [{ address: "93.184.216.34", family: 4 }]; + + it("rejects a cleartext HTTP endpoint at the exported lifecycle boundary", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "http://public.example.test/v1", + providerType: "openai", + credentialValue: "sk-secret", + lookup: publicLookup, + }), + ).rejects.toThrow("requires an HTTPS endpoint URL"); + }); + + it("rejects a DNS-private endpoint before ever considering the credential", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://internal.example.test/v1", + providerType: "openai", + // Deliberately empty: if the credential check ran first, the error + // message would mention "credential" instead of the SSRF reason. + credentialValue: "", + lookup: privateLookup, + }), + ).rejects.toThrow(/resolves to private\/internal address/); + }); + + it("rejects an empty credential only after the endpoint already resolved publicly", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://public.example.test/v1", + providerType: "openai", + credentialValue: " ", + lookup: publicLookup, + }), + ).rejects.toThrow(/requires a non-empty credential value/); + }); + + it("rejects a loopback endpoint (no pinnable address) before the credential check", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://localhost/v1", + providerType: "openai", + credentialValue: "", + lookup: publicLookup, + }), + ).rejects.toThrow(/requires a DNS-resolved public address/); + }); + + it("surfaces the underlying resolver failure when DNS lookup itself errors", async () => { + const failingLookup: EndpointDnsLookupFn = async () => { + throw new Error("ENOTFOUND"); + }; + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://does-not-resolve.example.test/v1", + providerType: "openai", + credentialValue: "sk-secret", + lookup: failingLookup, + }), + ).rejects.toThrow(/cannot resolve endpoint host/); + }); +}); diff --git a/src/lib/inference/https-pin-runtime-adapter.ts b/src/lib/inference/https-pin-runtime-adapter.ts new file mode 100644 index 00000000000..00e531bfdfe --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter.ts @@ -0,0 +1,1576 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTPS DNS-pinning runtime adapter server and lifecycle. + * + * Unlike the Bedrock/OpenRouter adapters (one process per singleton external + * endpoint), a host can have multiple DNS-backed HTTPS custom endpoints + * configured concurrently, so this adapter is one shared process serving many + * routes. Routes are registered on an already-running adapter through an + * authenticated control-plane `PUT /control/routes/:routeId` call instead of + * a full respawn, because respawning would lose every other route's + * credential value — those values enter the process only through the + * authenticated loopback control plane and are never written to disk or + * placed in the child process environment. Persisted recovery bookkeeping + * contains only opaque route ids, provider type, non-secret token generation + * values, and timestamps. + * + * If the adapter process dies, only the next route whose owning command calls + * `ensureHttpsPinRuntimeAdapter` recovers automatically; other previously + * registered routes stay unreachable until their owning command re-runs. This + * is an accepted consequence of never persisting plaintext credentials -- but + * the freshly spawned process is still told which route ids those are (never + * their credentials), so it can answer them with an actionable + * `route_needs_recovery` response instead of a 404 indistinguishable from a + * route that never existed (#6141). + * + * Recovery boundary: + * - whyNotSourceFix: the durable row intentionally omits the upstream URL, + * pinned addresses, and credential. Only the owning `inference set` caller + * holds all three, so a respawn cannot reconstruct another route safely. + * - removalCondition: retire orphaning/manual re-registration only when a + * reviewed secure recovery source or capability can rehydrate every + * registered route after respawn without persisting plaintext credentials, + * exposing them to OpenShell or a sandbox, or weakening per-route token and + * pinned-address isolation. + */ + +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import { BlockList, isIP } from "node:net"; +import path from "node:path"; + +import { + BEDROCK_RUNTIME_ADAPTER_PORT, + DASHBOARD_PORT, + DASHBOARD_PORT_RANGE_END, + DASHBOARD_PORT_RANGE_START, + GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, + OLLAMA_PORT, + OLLAMA_PROXY_PORT, + OPENROUTER_RUNTIME_ADAPTER_PORT, + VLLM_PORT, + validateHttpsPinRuntimeAdapterPort, +} from "../core/ports"; +import { compactText } from "../core/url-utils"; +import { getVersion } from "../core/version"; +import { ROOT, run, runCapture } from "../runner"; +import { buildMinimalCredentialAdapterEnv } from "../subprocess-env"; +import { assertEndpointResolvesPublic, type EndpointDnsLookupFn } from "./endpoint-ssrf-preflight"; +import { + buildHttpsPinRouteBaseUrl, + buildHttpsPinRouteLoopbackBaseUrl, + computeHttpsPinRouteId, + HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, + HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN, + HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + type HttpsPinCredentialProviderType, + resolveHttpsPinCredentialHeader, +} from "./https-pin-runtime"; +import { + describeForwardHttpError, + ForwardHttpError, + forwardHttpsPinnedRequest, + type HttpsPinTarget, + sendForwardError, +} from "./https-pin-runtime-adapter-forward"; +import { + appendLocalAdapterJsonLine, + DEFAULT_LOCAL_ADAPTER_STATE_DIR, + ensureLocalAdapterStateDir, + isLocalAdapterProcess, + type JsonObject, + killLocalAdapterPid, + loadLocalAdapterPid, + persistLocalAdapterPid, + readLocalAdapterJsonFile, + readLocalAdapterTextFile, + removeLocalAdapterFile, + spawnDetachedNodeAdapter, + waitForLocalAdapterHealth, + writeLocalAdapterJsonFile, + writeLocalAdapterSecretFile, +} from "./local-adapter-lifecycle"; + +const STATE_DIR = DEFAULT_LOCAL_ADAPTER_STATE_DIR; +const TOKEN_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter-token"); +const PID_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.pid"); +const STATE_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.json"); +const LOCK_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.lock"); +export const LOG_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.log"); +const PROCESS_NEEDLE = "https-pin-runtime-adapter.js"; +const MAX_CONTROL_BODY_BYTES = 16 * 1024; +// Matches the sibling OpenRouter adapter's lock retry budget +// (openrouter-runtime-adapter-lifecycle.ts): long enough to outlast a normal +// spawn-and-health-check cycle, short enough to fail loudly on a truly stuck +// lock rather than hang the CLI command indefinitely. +const LOCK_RETRY_ATTEMPTS = 100; +const LOCK_RETRY_MS = 100; +const STALE_LOCK_MS = 30_000; +const PROCESS_EXIT_WAIT_ATTEMPTS = 30; +const PROCESS_EXIT_WAIT_MS = 100; +const ADAPTER_PROTOCOL_VERSION = "3"; +const OPEN_SHELL_DOCKER_NETWORK = "openshell-docker"; + +interface AdapterIdentity { + protocolVersion: string; + buildId: string; +} + +/** + * Captured once per process so an adapter that survives an upgrade keeps + * proving the build it actually started from, not the files currently on + * disk. The opaque digest avoids exposing local version or source metadata. + */ +const CURRENT_ADAPTER_IDENTITY: Readonly = Object.freeze({ + protocolVersion: ADAPTER_PROTOCOL_VERSION, + buildId: crypto + .createHash("sha256") + .update(`nemoclaw:https-pin-adapter-build:v1\0${getVersion()}`) + .digest("hex"), +}); + +interface RouteRuntime { + targetBaseUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + generation: string; +} + +interface RoutePersistedMeta { + providerType: HttpsPinCredentialProviderType; + generation: string; + registeredAt: string; +} + +type OrphanedRouteMeta = Pick; + +/** Executable architecture contract mirrored by the orphan-recovery tests. */ +const ORPHANED_ROUTE_RECOVERY_BOUNDARY = { + whyNotSourceFix: + "Durable recovery metadata intentionally omits the upstream URL, pinned addresses, and credential; only the owning inference set caller can supply all three.", + removalCondition: + "Retire orphaning/manual re-registration only when a reviewed secure recovery source or capability can rehydrate every registered route after respawn without persisting plaintext credentials, exposing them to OpenShell or a sandbox, or weakening per-route token and pinned-address isolation.", +} as const; + +type AdapterLogFields = Record; +type AdapterLogger = (event: string, fields?: AdapterLogFields) => void; + +function normalizeLogField( + value: string | number | boolean | null | undefined, +): string | number | boolean | null { + if (value === undefined) return null; + if (typeof value === "string") return compactText(value).slice(0, 180); + return value; +} + +function defaultAdapterLogger(event: string, fields: AdapterLogFields = {}): void { + try { + const payload: Record = { + ts: new Date().toISOString(), + event: normalizeLogField(event) as string, + }; + for (const [key, value] of Object.entries(fields)) { + payload[key] = normalizeLogField(value); + } + appendLocalAdapterJsonLine(LOG_PATH, payload); + } catch { + /* best-effort diagnostics only */ + } +} + +function logAdapterEvent( + logger: AdapterLogger, + event: string, + fields: AdapterLogFields = {}, +): void { + try { + logger(event, fields); + } catch { + /* best-effort diagnostics only */ + } +} + +function authMatches(actual: string | string[] | undefined, token: string): boolean { + const header = Array.isArray(actual) ? actual[0] : actual; + if (!header) return false; + const expected = Buffer.from(`Bearer ${token}`); + const received = Buffer.from(header); + return received.length === expected.length && crypto.timingSafeEqual(received, expected); +} + +function rawTokenMatches(actual: string | string[] | undefined, token: string): boolean { + const header = Array.isArray(actual) ? actual[0] : actual; + if (!header) return false; + const expected = Buffer.from(token); + const received = Buffer.from(header); + return received.length === expected.length && crypto.timingSafeEqual(received, expected); +} + +function routeAuthMatches( + req: http.IncomingMessage, + token: string, + providerType: HttpsPinCredentialProviderType, +): boolean { + return providerType === "anthropic" + ? rawTokenMatches(req.headers["x-api-key"], token) + : authMatches(req.headers.authorization, token); +} + +function isLoopbackRemoteAddress(remoteAddress: string | undefined): boolean { + if (!remoteAddress) return false; + const normalized = remoteAddress.replace(/^::ffff:/, ""); + return normalized === "127.0.0.1" || normalized === "::1"; +} + +/** Parse one exact Docker IPAM subnet before it becomes a route-source capability. */ +function normalizeAllowedSourceCidr(value: string): string | null { + const candidate = value.trim(); + const slash = candidate.lastIndexOf("/"); + if (slash <= 0) return null; + const address = candidate.slice(0, slash); + const family = isIP(address); + const prefix = Number(candidate.slice(slash + 1)); + const maxPrefix = family === 4 ? 32 : family === 6 ? 128 : 0; + if (!maxPrefix || !Number.isInteger(prefix) || prefix < 1 || prefix > maxPrefix) return null; + return `${address}/${prefix}`; +} + +function buildAllowedRouteSourceMatcher(allowedSourceCidrs: readonly string[]): { + cidrs: string[]; + matches: (remoteAddress: string | undefined) => boolean; +} { + const cidrs = [ + ...new Set( + allowedSourceCidrs + .map(normalizeAllowedSourceCidr) + .filter((entry): entry is string => Boolean(entry)), + ), + ]; + if (cidrs.length === 0) { + throw new Error("HTTPS Pin Runtime adapter requires an OpenShell bridge source CIDR."); + } + const blockList = new BlockList(); + for (const cidr of cidrs) { + const [address, prefixText] = cidr.split("/"); + const family = isIP(address) === 4 ? "ipv4" : "ipv6"; + blockList.addSubnet(address, Number(prefixText), family); + } + return { + cidrs, + matches(remoteAddress) { + if (!remoteAddress) return false; + if (isLoopbackRemoteAddress(remoteAddress)) return true; + const normalized = remoteAddress.replace(/^::ffff:/, ""); + const family = isIP(normalized); + return family === 4 + ? blockList.check(normalized, "ipv4") + : family === 6 + ? blockList.check(normalized, "ipv6") + : false; + }, + }; +} + +function routeSourcePolicyDigest(cidrs: readonly string[]): string { + return crypto + .createHash("sha256") + .update(`nemoclaw:https-pin-route-sources:v1\0${[...cidrs].sort().join("\0")}`) + .digest("hex"); +} + +function discoverOpenShellBridgeSourceCidrs(capture: typeof runCapture = runCapture): string[] { + let raw = ""; + try { + raw = capture( + [ + "docker", + "network", + "inspect", + OPEN_SHELL_DOCKER_NETWORK, + "--format", + "{{json .IPAM.Config}}", + ], + { ignoreError: true }, + ); + } catch { + raw = ""; + } + try { + const parsed = JSON.parse(raw.trim()) as unknown; + if (!Array.isArray(parsed)) throw new Error("expected Docker IPAM array"); + const cidrs = parsed + .map((entry) => + entry && typeof entry === "object" && typeof (entry as JsonObject).Subnet === "string" + ? String((entry as JsonObject).Subnet) + : "", + ) + .map(normalizeAllowedSourceCidr) + .filter((entry): entry is string => Boolean(entry)); + if (cidrs.length > 0) return [...new Set(cidrs)]; + } catch { + // Fall through to the fail-closed error below. + } + throw new Error( + `Cannot determine the ${OPEN_SHELL_DOCKER_NETWORK} bridge source CIDR; refusing to expose the credential-bearing HTTPS Pin Runtime adapter.`, + ); +} + +function controlChallengeProof( + controlToken: string, + nonce: string, + identity: Readonly, + sourcePolicyDigest: string, +): string { + return crypto + .createHmac("sha256", controlToken) + .update( + `nemoclaw:https-pin-control-challenge:v3\0${identity.protocolVersion}\0${identity.buildId}\0${sourcePolicyDigest}\0${nonce}`, + ) + .digest("hex"); +} + +/** + * Derives the credential for exactly one sandbox-facing route from the + * persisted host-only control secret. The explicit domain separator prevents + * the derived value from being confused with any other HMAC use, while the + * route id binding means a credential issued for route A cannot authorize + * route B. The non-secret generation keeps the token stable across ordinary + * adapter restarts while ensuring DELETE plus re-registration cannot + * resurrect a previously issued token. + */ +function deriveRouteToken(controlToken: string, routeId: string, generation: string): string { + return crypto + .createHmac("sha256", controlToken) + .update(`nemoclaw:https-pin-route:v2\0${routeId}\0${generation}`) + .digest("hex"); +} + +function sendJson(res: http.ServerResponse, status: number, body: unknown): void { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); +} + +function buildContainedForwardPath( + route: RouteRuntime, + normalizedSuffix: string, + search: string, + rawRequestTarget: string, +): string { + const rawPath = rawRequestTarget.split("?", 1)[0]; + // Reject encoded path delimiters/dot segments (including a first layer of + // double encoding) before a downstream framework can decode them into a + // different path than the adapter authorized. Literal dot segments are + // already normalized by URL parsing and fail the base-prefix check below. + if ( + /%(?:2e|2f|5c|25)/i.test(rawPath) || + /(?:^|\/)\.{1,2}(?:\/|$)/u.test(rawPath) || + rawPath.includes("\\") || + rawPath.includes("\0") + ) { + throw new ForwardHttpError(404, "Route path not found.", "route_path_not_found"); + } + + const targetPath = new URL(route.targetBaseUrl).pathname.replace(/\/+$/, "") || "/"; + const suffix = normalizedSuffix === "/" ? "" : normalizedSuffix; + // OpenShell exposes OpenAI routes at the canonical `/v1` gateway surface, + // while targetBaseUrl is already the provider base validated by NemoClaw. + // Translate only that structural gateway segment; inferring arbitrary path + // overlap could collapse legitimate resource names such as `messages`. + const translatedSuffix = + targetPath !== "/" && + route.providerType === "openai" && + (suffix === "/v1" || suffix.startsWith("/v1/")) + ? suffix.slice(3) + : suffix; + const joined = targetPath === "/" ? translatedSuffix || "/" : `${targetPath}${translatedSuffix}`; + const canonical = new URL(joined, "http://adapter.invalid").pathname; + const contained = + canonical === joined && + (targetPath === "/" || canonical === targetPath || canonical.startsWith(`${targetPath}/`)); + if (!contained) { + throw new ForwardHttpError(404, "Route path not found.", "route_path_not_found"); + } + return `${canonical}${search}`; +} + +function readControlRequestJson(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + req.on("data", (chunk: Buffer) => { + if (settled) return; + size += chunk.length; + if (size > MAX_CONTROL_BODY_BYTES) { + // Reject without destroying the socket: destroying `req` mid-stream + // tears down the underlying connection before the 413 response can + // flush, so the caller sees a raw connection reset instead of a + // clean error. Draining the remainder of a small control-plane body + // (16 KB cap) to let `res.end()` reach the client is cheap. + settled = true; + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + chunks.push(Buffer.from(chunk)); + }); + req.on("end", () => { + if (settled) return; + settled = true; + try { + const raw = Buffer.concat(chunks).toString("utf8"); + const parsed = raw ? JSON.parse(raw) : {}; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("expected a JSON object"); + } + resolve(parsed as JsonObject); + } catch { + reject(new ForwardHttpError(400, "Request body must be valid JSON.", "invalid_json")); + } + }); + req.on("error", (err) => { + if (settled) return; + settled = true; + reject(err); + }); + }); +} + +function parseRoutePutBody(raw: JsonObject): RouteRuntime { + const targetBaseUrl = typeof raw.targetBaseUrl === "string" ? raw.targetBaseUrl.trim() : ""; + const providerType = + raw.providerType === "anthropic" || raw.providerType === "openai" ? raw.providerType : null; + const credentialValue = typeof raw.credentialValue === "string" ? raw.credentialValue : ""; + const generation = typeof raw.generation === "string" ? raw.generation : ""; + const pinnedAddresses = Array.isArray(raw.pinnedAddresses) + ? raw.pinnedAddresses.filter( + (entry): entry is string => typeof entry === "string" && entry.length > 0, + ) + : []; + if ( + !targetBaseUrl || + !providerType || + !credentialValue || + pinnedAddresses.length === 0 || + !/^[0-9a-f]{32}$/u.test(generation) + ) { + throw new ForwardHttpError( + 400, + "targetBaseUrl, providerType, credentialValue, pinnedAddresses, and a valid generation are required.", + "invalid_route", + ); + } + try { + const target = new URL(targetBaseUrl); + if ( + target.protocol !== "https:" || + target.username || + target.password || + target.search || + target.hash + ) { + throw new Error("credential-bearing URL components are not supported"); + } + } catch { + throw new ForwardHttpError( + 400, + "targetBaseUrl must be a valid HTTPS URL without userinfo, query, or fragment components.", + "invalid_route", + ); + } + return { targetBaseUrl, pinnedAddresses, providerType, credentialValue, generation }; +} + +/** + * Builds the shared adapter server. Routes live only in memory (`routes`), + * seeded from `initialRoutes` at startup and otherwise populated by + * authenticated `PUT /control/routes/:routeId` calls from + * `ensureHttpsPinRuntimeAdapter`. + */ +export function createHttpsPinRuntimeAdapterServer(options: { + controlToken: string; + /** + * Exact OpenShell Docker IPAM subnets. Direct unit callers may omit this + * and get loopback-only behavior; the spawned production adapter always + * receives inspected bridge CIDRs in its authenticated bootstrap. + */ + allowedSourceCidrs?: readonly string[]; + initialRoutes?: Record; + orphanedRoutes?: Record; + logger?: AdapterLogger; + adapterIdentity?: Readonly; +}): http.Server { + const logger = options.logger || defaultAdapterLogger; + const adapterIdentity = options.adapterIdentity || CURRENT_ADAPTER_IDENTITY; + const allowedRouteSources = buildAllowedRouteSourceMatcher( + options.allowedSourceCidrs ?? ["127.0.0.1/32"], + ); + const sourcePolicyDigest = routeSourcePolicyDigest(allowedRouteSources.cidrs); + const routes = new Map(Object.entries(options.initialRoutes || {})); + const orphanedRoutes = new Map( + Object.entries(options.orphanedRoutes || {}), + ); + + const server = http.createServer(async (req, res) => { + const started = Date.now(); + let routeId = "unknown"; + try { + const url = new URL(req.url || "/", "http://127.0.0.1"); + + if (req.method === "GET" && url.pathname === "/health") { + if (!allowedRouteSources.matches(req.socket.remoteAddress)) { + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + return; + } + sendJson(res, 200, { + ok: true, + routeCount: routes.size, + }); + return; + } + + if (req.method === "GET" && url.pathname === "/control/health") { + if (!isLoopbackRemoteAddress(req.socket.remoteAddress)) { + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + return; + } + const nonce = url.searchParams.get("nonce") || ""; + if (!/^[0-9a-f]{64}$/u.test(nonce)) { + sendJson(res, 400, { + error: { + message: "Invalid control challenge", + type: "invalid_request", + code: "invalid_control_challenge", + }, + }); + return; + } + sendJson(res, 200, { + ok: true, + protocolVersion: adapterIdentity.protocolVersion, + buildId: adapterIdentity.buildId, + sourcePolicyDigest, + proof: controlChallengeProof( + options.controlToken, + nonce, + adapterIdentity, + sourcePolicyDigest, + ), + }); + return; + } + + if (req.method === "TRACE" || req.method === "TRACK") { + sendJson(res, 405, { + error: { + message: "Method not allowed", + type: "method_not_allowed", + code: "method_not_allowed", + }, + }); + logAdapterEvent(logger, "request_rejected", { + method: req.method, + status: 405, + reason: "method_not_allowed", + durationMs: Date.now() - started, + }); + return; + } + + const controlMatch = url.pathname.match(/^\/control\/routes\/([^/]+)$/); + if (controlMatch) { + routeId = controlMatch[1]; + if (!isLoopbackRemoteAddress(req.socket.remoteAddress)) { + // Route registration accepts a caller-supplied targetBaseUrl and + // pinnedAddresses with no SSRF re-validation here -- that only + // happens host-side in ensureHttpsPinRuntimeAdapter before it + // calls this endpoint over loopback. The sandbox never receives + // the control token, but the source check remains a second boundary + // against any accidental credential exposure or host routing drift. + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "control_plane_non_loopback", + durationMs: Date.now() - started, + }); + return; + } + if (!authMatches(req.headers.authorization, options.controlToken)) { + sendJson(res, 401, { + error: { message: "Unauthorized", type: "unauthorized", code: "unauthorized" }, + }); + logAdapterEvent(logger, "request_rejected", { + method: req.method || "unknown", + status: 401, + reason: "control_plane_unauthorized", + durationMs: Date.now() - started, + }); + return; + } + if (req.method === "DELETE") { + routes.delete(routeId); + orphanedRoutes.delete(routeId); + sendJson(res, 200, { ok: true, routeId }); + logAdapterEvent(logger, "route_revoked", { + routeId, + routeCount: routes.size, + durationMs: Date.now() - started, + }); + return; + } + if (req.method !== "PUT") { + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + return; + } + const body = await readControlRequestJson(req); + const route = parseRoutePutBody(body); + routes.set(routeId, route); + sendJson(res, 200, { ok: true, routeId }); + logAdapterEvent(logger, "route_registered", { + routeId, + providerType: route.providerType, + routeCount: routes.size, + durationMs: Date.now() - started, + }); + return; + } + + const routeMatch = url.pathname.match(/^\/route\/([^/]+)(\/.*)?$/); + if (routeMatch) { + routeId = routeMatch[1]; + if (!allowedRouteSources.matches(req.socket.remoteAddress)) { + // Route tokens are scoped to one route, but a peer that reaches this + // host port from outside the intended sandbox-to-host boundary must + // not be able to exercise even its own route credential. + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "route_non_openshell_network", + durationMs: Date.now() - started, + }); + return; + } + const route = routes.get(routeId); + const orphanedRoute = orphanedRoutes.get(routeId); + if (!route && !orphanedRoute) { + sendJson(res, 404, { + error: { message: "Unknown route", type: "not_found", code: "route_not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "route_not_found", + durationMs: Date.now() - started, + }); + return; + } + const providerType = route?.providerType ?? orphanedRoute?.providerType; + const generation = route?.generation ?? orphanedRoute?.generation; + const routeToken = generation + ? deriveRouteToken(options.controlToken, routeId, generation) + : null; + const authenticated = providerType + ? Boolean(routeToken && routeAuthMatches(req, routeToken, providerType)) + : false; + if (!authenticated) { + sendJson(res, 401, { + error: { message: "Unauthorized", type: "unauthorized", code: "unauthorized" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 401, + reason: "route_unauthorized", + durationMs: Date.now() - started, + }); + return; + } + if (!route) { + if (orphanedRoute) { + // Known before the adapter's last restart but not recovered by + // it -- distinct from a route that never existed, so the caller + // gets an actionable signal instead of an indistinguishable 404. + sendJson(res, 503, { + error: { + message: + "This route was registered before the adapter's last restart and was not recovered. Re-run the original `inference set --endpoint-url` command for this endpoint.", + type: "unavailable", + code: "route_needs_recovery", + }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 503, + reason: "route_needs_recovery", + durationMs: Date.now() - started, + }); + return; + } + // The no-route/no-orphan case returns above. This branch exists only + // to make the type narrowing explicit after authenticated orphan + // handling. + throw new ForwardHttpError(404, "Unknown route", "route_not_found"); + } + const forwardPath = buildContainedForwardPath( + route, + routeMatch[2] || "/", + url.search, + req.url || "/", + ); + const target: HttpsPinTarget = { + targetUrl: new URL(route.targetBaseUrl), + pinnedAddress: route.pinnedAddresses[0], + credential: resolveHttpsPinCredentialHeader(route.providerType, route.credentialValue), + }; + const status = await forwardHttpsPinnedRequest({ req, res, forwardPath, target }); + logAdapterEvent(logger, "request_forwarded", { + routeId, + status, + durationMs: Date.now() - started, + }); + return; + } + + sendJson(res, 404, { error: { message: "Not found", type: "not_found", code: "not_found" } }); + } catch (err) { + const { status, code } = describeForwardHttpError(err); + logAdapterEvent(logger, "request_failed", { + routeId, + status, + code, + durationMs: Date.now() - started, + }); + sendForwardError(res, err); + } + }); + + // CONNECT bypasses Node's normal request callback. Reject it explicitly so + // this credential-injecting proxy cannot be repurposed as a generic tunnel. + server.on("connect", (_req, socket) => { + socket.end("HTTP/1.1 405 Method Not Allowed\r\nConnection: close\r\nContent-Length: 0\r\n\r\n"); + logAdapterEvent(logger, "request_rejected", { + method: "CONNECT", + status: 405, + reason: "method_not_allowed", + }); + }); + return server; +} + +function parseAllowedSourceCidrs(raw: string | undefined): string[] { + if (!raw) return []; + try { + const parsed = JSON.parse(raw) as unknown; + const allowedSourceCidrs = Array.isArray(parsed) + ? parsed.filter( + (entry): entry is string => typeof entry === "string" && Boolean(entry.trim()), + ) + : []; + return buildAllowedRouteSourceMatcher(allowedSourceCidrs).cidrs; + } catch { + return []; + } +} + +function parseOrphanedRoutes(raw: string | undefined): Record { + if (!raw) return {}; + try { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {}; + const routes: Record = {}; + for (const [id, value] of Object.entries(parsed)) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const meta = value as JsonObject; + const providerType = meta.providerType; + const generation = meta.generation; + if ( + id && + (providerType === "openai" || providerType === "anthropic") && + typeof generation === "string" && + /^[0-9a-f]{32}$/u.test(generation) + ) { + routes[id] = { providerType, generation }; + } + } + return routes; + } catch { + return {}; + } +} + +export function startHttpsPinRuntimeAdapterFromEnv(): http.Server { + // Keep this read explicit so the env-var documentation gate can prove the + // internal host-only secret is accounted for in its allowlist. + const controlToken = process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN; + const port = Number( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT || HTTPS_PIN_RUNTIME_ADAPTER_PORT, + ); + + if (!controlToken) throw new Error(`${HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV} is required`); + if (!Number.isInteger(port) || port <= 0) { + throw new Error("NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT must be a valid port"); + } + + const allowedSourceCidrs = parseAllowedSourceCidrs( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ALLOWED_SOURCE_CIDRS, + ); + const orphanedRoutes = parseOrphanedRoutes( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTES, + ); + + const server = createHttpsPinRuntimeAdapterServer({ + controlToken, + allowedSourceCidrs, + orphanedRoutes, + }); + server.listen(port, HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, () => { + defaultAdapterLogger("adapter_ready", { + bindHost: HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, + port, + routeCount: 0, + orphanedRouteCount: Object.keys(orphanedRoutes).length, + allowedSourceCidrs: allowedSourceCidrs.join(","), + logPath: LOG_PATH, + }); + console.log( + `HTTPS Pin Runtime adapter listening on ${HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST}:${port}; log ${LOG_PATH}`, + ); + }); + return server; +} + +function loadPersistedPid(): number | null { + return loadLocalAdapterPid(PID_PATH); +} + +function isAdapterProcess(pid: number | null | undefined): boolean { + return isLocalAdapterProcess(pid, PROCESS_NEEDLE, runCapture); +} + +async function waitForAdapterProcessExit( + pid: number, + options: { + isRunning?: (candidatePid: number) => boolean; + sleep?: (ms: number) => Promise; + attempts?: number; + intervalMs?: number; + } = {}, +): Promise { + const isRunning = options.isRunning || ((candidatePid: number) => isAdapterProcess(candidatePid)); + const sleep = options.sleep || sleepMs; + const attempts = options.attempts || PROCESS_EXIT_WAIT_ATTEMPTS; + const intervalMs = options.intervalMs || PROCESS_EXIT_WAIT_MS; + for (let attempt = 0; attempt < attempts; attempt++) { + if (!isRunning(pid)) return true; + if (attempt + 1 < attempts) await sleep(intervalMs); + } + return false; +} + +async function killStaleAdapter(): Promise { + const persistedPid = loadPersistedPid(); + const wasAdapterProcess = isAdapterProcess(persistedPid); + killLocalAdapterPid({ pidPath: PID_PATH, processMatcher: PROCESS_NEEDLE, run, runCapture }); + if (wasAdapterProcess && persistedPid && !(await waitForAdapterProcessExit(persistedPid))) { + throw new Error( + `HTTPS Pin Runtime adapter process ${persistedPid} did not exit after SIGTERM; refusing to start a competing listener.`, + ); + } +} + +/** + * Unlike the Bedrock/OpenRouter adapters' hand-maintained `scripts/*.js` + * wrappers, this adapter is spawned directly from its own compiled output so + * the entrypoint stays TypeScript-only (see the `require.main` guard below). + */ +function getAdapterScriptPath(): string { + return path.join(ROOT, "dist", "lib", "inference", "https-pin-runtime-adapter.js"); +} + +function probeAdapterControlHealth(options: { + controlToken: string; + port?: number; + nonce?: string; + timeoutMs?: number; + expectedIdentity?: Readonly; + expectedSourceCidrs?: readonly string[]; +}): Promise { + const nonce = options.nonce || crypto.randomBytes(32).toString("hex"); + const expectedIdentity = options.expectedIdentity || CURRENT_ADAPTER_IDENTITY; + const expectedSources = buildAllowedRouteSourceMatcher( + options.expectedSourceCidrs ?? ["127.0.0.1/32"], + ); + const expectedSourcePolicyDigest = routeSourcePolicyDigest(expectedSources.cidrs); + const expectedProof = controlChallengeProof( + options.controlToken, + nonce, + expectedIdentity, + expectedSourcePolicyDigest, + ); + return new Promise((resolve) => { + let settled = false; + let absoluteDeadline: NodeJS.Timeout | null = null; + const settle = (value: boolean) => { + if (settled) return; + settled = true; + if (absoluteDeadline) clearTimeout(absoluteDeadline); + resolve(value); + }; + const timeoutMs = options.timeoutMs || 1000; + const req = http.request( + { + hostname: HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + port: options.port || HTTPS_PIN_RUNTIME_ADAPTER_PORT, + path: `/control/health?nonce=${nonce}`, + method: "GET", + timeout: timeoutMs, + }, + (res) => { + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (chunk: Buffer) => { + if (settled) return; + size += chunk.length; + if (size > 1024) { + res.destroy(); + settle(false); + return; + } + chunks.push(Buffer.from(chunk)); + }); + res.on("end", () => { + if (settled || res.statusCode !== 200) { + settle(false); + return; + } + try { + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as JsonObject; + const protocolVersion = + typeof body.protocolVersion === "string" ? body.protocolVersion : ""; + const buildId = typeof body.buildId === "string" ? body.buildId : ""; + const sourcePolicyDigest = + typeof body.sourcePolicyDigest === "string" ? body.sourcePolicyDigest : ""; + if ( + protocolVersion !== expectedIdentity.protocolVersion || + buildId !== expectedIdentity.buildId || + sourcePolicyDigest !== expectedSourcePolicyDigest + ) { + settle(false); + return; + } + const proof = typeof body.proof === "string" ? body.proof : ""; + const expected = Buffer.from(expectedProof); + const received = Buffer.from(proof); + settle( + received.length === expected.length && crypto.timingSafeEqual(received, expected), + ); + } catch { + settle(false); + } + }); + }, + ); + req.on("timeout", () => { + req.destroy(); + settle(false); + }); + req.on("error", () => settle(false)); + absoluteDeadline = setTimeout(() => { + req.destroy(); + settle(false); + }, timeoutMs); + req.end(); + }); +} + +async function waitForAdapterHealth( + token: string, + expectedSourceCidrs: readonly string[], + port = HTTPS_PIN_RUNTIME_ADAPTER_PORT, +): Promise { + return waitForLocalAdapterHealth( + () => + probeAdapterControlHealth({ + port, + controlToken: token, + expectedSourceCidrs, + }), + { + attempts: 20, + intervalMs: 100, + }, + ); +} + +function putRoute(options: { + controlToken: string; + routeId: string; + targetBaseUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + generation: string; +}): Promise { + return new Promise((resolve, reject) => { + const payload = JSON.stringify({ + targetBaseUrl: options.targetBaseUrl, + pinnedAddresses: options.pinnedAddresses, + providerType: options.providerType, + credentialValue: options.credentialValue, + generation: options.generation, + }); + // The file-backed value here is a purpose-specific 0600 control token, + // intentionally sent only to the fixed loopback adapter after its HMAC + // health challenge proved that the expected process owns this port. The + // destination and request path never derive from file data. + const req = http.request( + { + hostname: HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + port: HTTPS_PIN_RUNTIME_ADAPTER_PORT, + path: `/control/routes/${options.routeId}`, + method: "PUT", + headers: { + Authorization: `Bearer ${options.controlToken}`, + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + }, + timeout: 3000, + }, + (res) => { + res.on("data", () => {}); + res.on("end", () => { + if (res.statusCode === 200) { + resolve(); + } else { + reject( + new Error( + `HTTPS Pin Runtime adapter rejected route registration (status ${res.statusCode}).`, + ), + ); + } + }); + }, + ); + req.on("timeout", () => { + req.destroy(); + reject(new Error("HTTPS Pin Runtime adapter route registration timed out.")); + }); + req.on("error", reject); + req.end(payload); + }); +} + +function deleteRoute(controlToken: string, routeId: string): Promise { + return new Promise((resolve, reject) => { + // This is the same fixed-loopback authenticated control boundary as PUT; + // callers prove adapter identity with the HMAC health challenge before + // transmitting the purpose-specific token read from its 0600 state file. + const req = http.request( + { + hostname: HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + port: HTTPS_PIN_RUNTIME_ADAPTER_PORT, + path: `/control/routes/${routeId}`, + method: "DELETE", + headers: { Authorization: `Bearer ${controlToken}` }, + timeout: 3000, + }, + (res) => { + res.on("data", () => {}); + res.on("end", () => { + if (res.statusCode === 200) resolve(); + else { + reject( + new Error( + `HTTPS Pin Runtime adapter rejected route revocation (status ${res.statusCode}).`, + ), + ); + } + }); + }, + ); + req.on("timeout", () => { + req.destroy(); + reject(new Error("HTTPS Pin Runtime adapter route revocation timed out.")); + }); + req.on("error", reject); + req.end(); + }); +} + +function extractPersistedRoutes(prior: JsonObject | null): Record { + if (!prior?.routes || typeof prior.routes !== "object" || Array.isArray(prior.routes)) return {}; + const sanitized: Record = {}; + for (const [id, value] of Object.entries(prior.routes)) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const meta = value as JsonObject; + if (meta.providerType !== "openai" && meta.providerType !== "anthropic") continue; + if (typeof meta.generation !== "string" || !/^[0-9a-f]{32}$/u.test(meta.generation)) { + continue; + } + sanitized[id] = { + providerType: meta.providerType, + generation: meta.generation, + registeredAt: + typeof meta.registeredAt === "string" ? meta.registeredAt : new Date(0).toISOString(), + ...(typeof meta.orphanedAt === "string" ? { orphanedAt: meta.orphanedAt } : {}), + }; + } + return sanitized; +} + +function persistRouteState(routeId: string, meta: RoutePersistedMeta): void { + const prior = readLocalAdapterJsonFile(STATE_PATH); + const priorRoutes = extractPersistedRoutes(prior); + writeLocalAdapterJsonFile(STATE_PATH, { + pid: (prior?.pid as number | null | undefined) ?? loadPersistedPid(), + updatedAt: new Date().toISOString(), + // Re-registering a route (fresh `meta`, no `orphanedAt`) always + // supersedes any prior orphaned entry for the same id -- this is how a + // route heals after its owner re-runs `inference set` post-recovery. + routes: { ...priorRoutes, [routeId]: meta }, + }); +} + +function removeRouteState(routeId: string): void { + const prior = readLocalAdapterJsonFile(STATE_PATH); + const routes = extractPersistedRoutes(prior); + delete routes[routeId]; + writeLocalAdapterJsonFile(STATE_PATH, { + pid: (prior?.pid as number | null | undefined) ?? loadPersistedPid(), + updatedAt: new Date().toISOString(), + routes, + }); +} + +/** + * Computes which previously-registered routes a fresh adapter respawn will + * NOT recover (every one except the route currently being bootstrapped), + * since credentials are only ever seeded into the process at spawn/PUT time + * and are never persisted to disk (see the module doc comment). Returns + * their ids -- so the freshly spawned process can tell "this route was + * orphaned by a restart" apart from "this route never existed" and respond + * accordingly instead of a bare 404 either way -- plus the persisted-state + * shape that keeps them recorded (still without credentials) until their + * owner re-runs `inference set` and `persistRouteState` supersedes them. + * + * The module-level `whyNotSourceFix` and `removalCondition` define the exit + * criterion for this deliberately degraded state: this function must keep + * orphaning non-bootstrap routes until a reviewed recovery source can + * rehydrate every route while preserving the same credential and route + * isolation boundaries. + */ +function computeRespawnState( + priorRoutes: Record, + bootstrapRouteId: string, +): { + orphanedRoutes: Record; + persistedRoutes: Record; +} { + const orphanedRoutes: Record = {}; + const persistedRoutes: Record = {}; + const orphanedAt = new Date().toISOString(); + for (const [id, meta] of Object.entries(priorRoutes)) { + if (id === bootstrapRouteId) continue; + if (meta.providerType !== "openai" && meta.providerType !== "anthropic") continue; + if (typeof meta.generation !== "string" || !/^[0-9a-f]{32}$/u.test(meta.generation)) continue; + orphanedRoutes[id] = { providerType: meta.providerType, generation: meta.generation }; + persistedRoutes[id] = { + providerType: meta.providerType, + generation: meta.generation, + registeredAt: + typeof meta.registeredAt === "string" ? meta.registeredAt : new Date(0).toISOString(), + orphanedAt, + }; + } + return { orphanedRoutes, persistedRoutes }; +} + +/** + * Ensures the shared adapter process is running and holds a current, + * pin-validated route for `(gatewayName, provider, endpointUrl)`, then + * returns the sandbox-facing base URL OpenShell should be registered with. + * + * Re-runs the SSRF preflight on every call so the pinned address is never + * older than this call — the address that gets registered is the one that + * gets connected to, closing the TOCTOU window between validation and the + * OpenShell gateway's own (would-be) resolution. + */ +export async function ensureHttpsPinRuntimeAdapter(options: { + gatewayName: string; + provider: string; + endpointUrl: string; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + lookup?: EndpointDnsLookupFn; + discoverAllowedSourceCidrs?: () => string[]; +}): Promise<{ + baseUrl: string; + localBaseUrl: string; + logPath: string; + credentialEnv: string; + token: string; + routeId: string; + pinnedAddresses: string[]; +}> { + let sourceUrl: URL; + try { + sourceUrl = new URL(options.endpointUrl); + } catch { + throw new Error("HTTPS Pin Runtime adapter requires a valid endpoint URL."); + } + if (sourceUrl.protocol !== "https:") { + throw new Error("HTTPS Pin Runtime adapter requires an HTTPS endpoint URL."); + } + if (sourceUrl.username || sourceUrl.password || sourceUrl.search || sourceUrl.hash) { + throw new Error( + "HTTPS Pin Runtime adapter endpoint URLs cannot contain userinfo, query, or fragment components.", + ); + } + const preflight = await assertEndpointResolvesPublic(options.endpointUrl, options.lookup); + if (!preflight.ok) { + throw new Error( + `HTTPS Pin Runtime adapter cannot validate "${options.endpointUrl}": ${preflight.reason}`, + ); + } + const pinnedAddresses = + preflight.addresses && preflight.addresses.length > 0 ? preflight.addresses : []; + if (pinnedAddresses.length === 0) { + throw new Error( + `HTTPS Pin Runtime adapter requires a DNS-resolved public address for "${options.endpointUrl}".`, + ); + } + // Checked only after the endpoint itself is proven safe to pin: an + // unreachable/private endpoint must fail on that ground, not report a + // confusing credential error for a URL that was never going to be allowed. + if (!options.credentialValue || !options.credentialValue.trim()) { + throw new Error( + `HTTPS Pin Runtime adapter requires a non-empty credential value for "${options.endpointUrl}".`, + ); + } + + const routeId = computeHttpsPinRouteId( + options.gatewayName, + options.provider, + options.endpointUrl, + ); + const allowedSourceCidrs = buildAllowedRouteSourceMatcher( + options.discoverAllowedSourceCidrs?.() ?? discoverOpenShellBridgeSourceCidrs(), + ).cidrs; + // Keep the lifecycle lock through the whole adapter-registration + // transaction. In particular, persistRouteState is a read/modify/write of + // the shared state file; releasing after spawn/reuse would let concurrent + // CLI processes register both live routes but race their metadata writes + // and silently drop one route from restart recovery. + const token = await withAdapterLock(async () => { + const priorRoute = extractPersistedRoutes(readLocalAdapterJsonFile(STATE_PATH))[routeId]; + const generation = + typeof priorRoute?.generation === "string" + ? priorRoute.generation + : crypto.randomBytes(16).toString("hex"); + const controlToken = await ensureAdapterProcessLocked({ + routeId, + allowedSourceCidrs, + }); + + await putRoute({ + controlToken, + routeId, + targetBaseUrl: options.endpointUrl, + pinnedAddresses, + providerType: options.providerType, + credentialValue: options.credentialValue, + generation, + }); + persistRouteState(routeId, { + providerType: options.providerType, + generation, + registeredAt: new Date().toISOString(), + }); + + // Only this route-scoped value leaves the host lifecycle boundary. The + // control token stays in its 0600 host state file and the adapter process + // environment; it is never staged into OpenShell. + return deriveRouteToken(controlToken, routeId, generation); + }); + + return { + baseUrl: buildHttpsPinRouteBaseUrl(routeId), + localBaseUrl: buildHttpsPinRouteLoopbackBaseUrl(routeId), + logPath: LOG_PATH, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token, + routeId, + pinnedAddresses, + }; +} + +/** + * Revokes one no-longer-referenced route after its provider, selection, and + * registry transaction has committed. The lifecycle lock serializes this + * control-plane delete with route registration and adapter respawn. + */ +export async function revokeHttpsPinRuntimeAdapterRoute(routeId: string): Promise { + if (!/^[0-9a-f]{64}$/u.test(routeId)) { + throw new Error("Refusing to revoke an invalid HTTPS Pin Runtime route id."); + } + return withAdapterLock(() => revokeRouteLocked(routeId)); +} + +async function revokeRouteLocked( + routeId: string, + deps: { + loadPid: () => number | null; + readControlToken: () => string | null; + probeHealth: (options: { controlToken: string }) => Promise; + deleteRoute: (controlToken: string, candidateRouteId: string) => Promise; + isAdapterProcess: (pid: number | null) => boolean; + removeRouteState: (candidateRouteId: string) => void; + } = { + loadPid: loadPersistedPid, + readControlToken: () => readLocalAdapterTextFile(TOKEN_PATH), + probeHealth: (options) => probeAdapterControlHealth(options), + deleteRoute, + isAdapterProcess, + removeRouteState, + }, +): Promise { + const pid = deps.loadPid(); + const controlToken = deps.readControlToken(); + const authenticatedLiveAdapter = Boolean( + controlToken && (await deps.probeHealth({ controlToken: controlToken as string })), + ); + if (authenticatedLiveAdapter && controlToken) { + await deps.deleteRoute(controlToken, routeId); + } else if (deps.isAdapterProcess(pid)) { + throw new Error("Cannot authenticate the live HTTPS Pin Runtime adapter for revocation."); + } + deps.removeRouteState(routeId); + return true; +} + +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function removeStaleLock(): void { + try { + const ageMs = Date.now() - fs.statSync(LOCK_PATH).mtimeMs; + if (ageMs > STALE_LOCK_MS) fs.unlinkSync(LOCK_PATH); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } +} + +function tryAcquireAdapterLock(): (() => void) | null { + ensureLocalAdapterStateDir(STATE_DIR); + removeStaleLock(); + try { + const fd = fs.openSync(LOCK_PATH, "wx", 0o600); + fs.writeFileSync(fd, `${process.pid}\n${new Date().toISOString()}\n`); + fs.closeSync(fd); + return () => { + try { + fs.unlinkSync(LOCK_PATH); + } catch { + /* best-effort lock cleanup */ + } + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "EEXIST") return null; + throw err; + } +} + +/** + * Serializes the read-check-kill-spawn recovery decision in + * `ensureAdapterProcess` across concurrent `inference set` invocations. + * Without this, two callers can both see no healthy prior process, both kill + * and respawn, and race to bind the same port and overwrite + * PID_PATH/TOKEN_PATH/STATE_PATH -- leaking a process and potentially + * leaving the persisted token out of sync with whichever process actually + * won the port. + */ +async function withAdapterLock(operation: () => Promise): Promise { + for (let attempt = 0; attempt < LOCK_RETRY_ATTEMPTS; attempt++) { + const release = tryAcquireAdapterLock(); + if (release) { + try { + return await operation(); + } finally { + release(); + } + } + await sleepMs(LOCK_RETRY_MS); + } + throw new Error("HTTPS Pin Runtime adapter startup is already in progress"); +} + +function validateAdapterPortConfiguration(): void { + validateHttpsPinRuntimeAdapterPort( + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + HTTPS_PIN_RUNTIME_ADAPTER_PORT, + { + dashboardPort: DASHBOARD_PORT, + dashboardRangeStart: DASHBOARD_PORT_RANGE_START, + dashboardRangeEnd: DASHBOARD_PORT_RANGE_END, + gatewayPort: GATEWAY_PORT, + vllmPort: VLLM_PORT, + ollamaPort: OLLAMA_PORT, + ollamaProxyPort: OLLAMA_PROXY_PORT, + bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, + openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, + }, + ); +} + +async function findReusableAdapterControlToken( + priorToken: string | null, + allowedSourceCidrs: readonly string[] = ["127.0.0.1/32"], + probeHealth: (options: { + controlToken: string; + expectedIdentity?: Readonly; + expectedSourceCidrs?: readonly string[]; + }) => Promise = probeAdapterControlHealth, +): Promise { + if (!priorToken) return null; + return (await probeHealth({ + controlToken: priorToken, + expectedIdentity: CURRENT_ADAPTER_IDENTITY, + expectedSourceCidrs: allowedSourceCidrs, + })) + ? priorToken + : null; +} + +function buildAdapterChildEnv( + controlToken: string, + allowedSourceCidrs: readonly string[], + orphanedRoutes: Record, +): Record { + return { + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT: String(HTTPS_PIN_RUNTIME_ADAPTER_PORT), + [HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV]: controlToken, + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ALLOWED_SOURCE_CIDRS: JSON.stringify(allowedSourceCidrs), + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTES: JSON.stringify(orphanedRoutes), + }; +} + +/** Returns the host-only control token, reusing the running process when possible or spawning fresh. */ +async function ensureAdapterProcessLocked(options: { + routeId: string; + allowedSourceCidrs: string[]; +}): Promise { + validateAdapterPortConfiguration(); + const priorToken = readLocalAdapterTextFile(TOKEN_PATH); + // The authenticated, build-bound health response is stronger identity + // evidence than a PID file. Reuse the live adapter even if its PID metadata + // is absent or stale, but replace it when the protocol or build differs so + // an upgrade cannot keep older forwarding security behavior alive. + const reusableToken = await findReusableAdapterControlToken( + priorToken, + options.allowedSourceCidrs, + ); + if (reusableToken) return reusableToken; + + await killStaleAdapter(); + // Reusing a still-valid persisted token (rather than always minting a new + // one) keeps previously registered OpenShell provider credentials working + // across an adapter respawn whenever possible. + const token = priorToken || crypto.randomBytes(24).toString("hex"); + // A fresh process starts with an empty in-memory route map -- every route + // other than the one being bootstrapped now is unrecoverable this restart, + // since credentials are never persisted to disk (see module doc comment). + // Tell the freshly spawned process which route ids those are so it can + // answer them with an actionable "needs recovery" response instead of a + // bare 404 indistinguishable from a route that never existed (#6141). + const priorState = readLocalAdapterJsonFile(STATE_PATH); + const { orphanedRoutes, persistedRoutes } = computeRespawnState( + extractPersistedRoutes(priorState), + options.routeId, + ); + const child = spawnDetachedNodeAdapter({ + scriptPath: getAdapterScriptPath(), + env: buildAdapterChildEnv(token, options.allowedSourceCidrs, orphanedRoutes), + // This is a long-lived, credential-bearing process, so it gets a + // purpose-built minimal environment rather than the general subprocess + // allowlist -- it must not inherit DOCKER_HOST/KUBECONFIG/SSH_AUTH_SOCK/ + // proxy capabilities that an ordinary short-lived CLI subprocess might + // legitimately need. See #6141. + buildEnv: buildMinimalCredentialAdapterEnv, + }); + try { + persistLocalAdapterPid(PID_PATH, child.pid); + if (!(await waitForAdapterHealth(token, options.allowedSourceCidrs))) { + throw new Error( + `HTTPS Pin Runtime adapter did not become healthy on ${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}`, + ); + } + writeLocalAdapterSecretFile(TOKEN_PATH, token); + // Keep the orphaned routes recorded (still without credentials) instead + // of dropping them: `persistRouteState` supersedes an entry here the + // moment its owner re-runs `inference set`, which is how a route heals. + writeLocalAdapterJsonFile(STATE_PATH, { + pid: child.pid ?? null, + updatedAt: new Date().toISOString(), + routes: persistedRoutes, + }); + } catch (err) { + await killStaleAdapter(); + removeLocalAdapterFile(STATE_PATH); + throw err; + } + return token; +} + +export const __test = { + ORPHANED_ROUTE_RECOVERY_BOUNDARY, + CURRENT_ADAPTER_IDENTITY, + deriveRouteToken, + buildAdapterChildEnv, + buildContainedForwardPath, + parseAllowedSourceCidrs, + waitForAdapterProcessExit, + persistRouteState, + getAdapterScriptPath, + probeAdapterControlHealth, + tryAcquireAdapterLock, + withAdapterLock, + computeRespawnState, + buildAllowedRouteSourceMatcher, + discoverOpenShellBridgeSourceCidrs, + findReusableAdapterControlToken, + revokeRouteLocked, + LOCK_PATH, + STATE_PATH, +}; + +// Detached-process entrypoint: `spawnDetachedNodeAdapter` runs this compiled +// file directly with plain `node` (see `getAdapterScriptPath`), so this guard +// is the only thing that distinguishes that invocation from the normal +// `require()` used by the rest of the CLI. +if (require.main === module) { + try { + startHttpsPinRuntimeAdapterFromEnv(); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} diff --git a/src/lib/inference/https-pin-runtime.test.ts b/src/lib/inference/https-pin-runtime.test.ts new file mode 100644 index 00000000000..735222419bf --- /dev/null +++ b/src/lib/inference/https-pin-runtime.test.ts @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildHttpsPinRouteBaseUrl, + buildHttpsPinRouteLoopbackBaseUrl, + computeHttpsPinRouteId, + HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN, + HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST, + isHttpsPinRuntimeEligible, + parseHttpsPinRouteId, + resolveHttpsPinCredentialHeader, +} from "./https-pin-runtime"; + +describe("isHttpsPinRuntimeEligible (#6141)", () => { + it("is eligible for a DNS-backed HTTPS hostname", () => { + expect(isHttpsPinRuntimeEligible("https://api.example.com/v1")).toBe(true); + }); + + it("is not eligible for HTTP, even with a DNS-backed hostname", () => { + expect(isHttpsPinRuntimeEligible("http://api.example.com/v1")).toBe(false); + }); + + it("is not eligible for an HTTPS IPv4 literal", () => { + expect(isHttpsPinRuntimeEligible("https://93.184.216.34/v1")).toBe(false); + }); + + it("is not eligible for an HTTPS IPv6 literal", () => { + expect(isHttpsPinRuntimeEligible("https://[2001:db8::1]/v1")).toBe(false); + }); + + it("is not eligible for NemoClaw's own OpenShell-managed host alias", () => { + expect(isHttpsPinRuntimeEligible("https://host.openshell.internal/v1")).toBe(false); + }); + + it("is not eligible for other OpenShell-managed aliases (inference.local, host.docker.internal)", () => { + expect(isHttpsPinRuntimeEligible("https://inference.local/v1")).toBe(false); + expect(isHttpsPinRuntimeEligible("https://host.docker.internal/v1")).toBe(false); + }); + + it("is not eligible for an unparseable URL", () => { + expect(isHttpsPinRuntimeEligible("not-a-url")).toBe(false); + }); + + it("is not eligible for null/undefined/empty input", () => { + expect(isHttpsPinRuntimeEligible(null)).toBe(false); + expect(isHttpsPinRuntimeEligible(undefined)).toBe(false); + expect(isHttpsPinRuntimeEligible("")).toBe(false); + }); + + it("accepts a URL instance the same as an equivalent string", () => { + expect(isHttpsPinRuntimeEligible(new URL("https://api.example.com/v1"))).toBe(true); + }); +}); + +describe("resolveHttpsPinCredentialHeader (#6141)", () => { + it("uses x-api-key for the anthropic provider type", () => { + expect(resolveHttpsPinCredentialHeader("anthropic", "sk-ant-secret")).toEqual({ + name: "x-api-key", + value: "sk-ant-secret", + }); + }); + + it("uses a Bearer authorization header for the openai provider type", () => { + expect(resolveHttpsPinCredentialHeader("openai", "sk-secret")).toEqual({ + name: "authorization", + value: "Bearer sk-secret", + }); + }); +}); + +describe("computeHttpsPinRouteId (#6141)", () => { + it("is deterministic for the same (gateway, provider, endpoint) triple", () => { + const a = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + const b = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + expect(a).toBe(b); + }); + + it("differs when any input differs", () => { + const base = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + expect( + computeHttpsPinRouteId("other-gw", "compatible-endpoint", "https://api.example.com/v1"), + ).not.toBe(base); + expect( + computeHttpsPinRouteId("gw", "compatible-anthropic-endpoint", "https://api.example.com/v1"), + ).not.toBe(base); + expect( + computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.other.com/v1"), + ).not.toBe(base); + }); + + it("keeps component boundaries unambiguous", () => { + expect(computeHttpsPinRouteId("a b", "c", "https://example.com/v1")).not.toBe( + computeHttpsPinRouteId("a", "b c", "https://example.com/v1"), + ); + }); + + it("never contains the endpoint hostname or path (safe to persist and log)", () => { + const id = computeHttpsPinRouteId( + "gw", + "compatible-endpoint", + "https://api.example.com/some/secret/path", + ); + expect(id).not.toContain("api.example.com"); + expect(id).not.toContain("secret"); + expect(id).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +describe("buildHttpsPinRouteBaseUrl / buildHttpsPinRouteLoopbackBaseUrl (#6141)", () => { + it("builds an opaque sandbox-facing route with no source URL material", () => { + const url = buildHttpsPinRouteBaseUrl("routeid1234567890abc"); + expect(url).toBe(`${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/routeid1234567890abc`); + }); + + it("builds the host-side loopback equivalent for the same route", () => { + const url = buildHttpsPinRouteLoopbackBaseUrl("routeid1234567890abc"); + expect(url).toBe(`${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}/route/routeid1234567890abc`); + }); + + it("parses only the exact opaque route base", () => { + const id = "a".repeat(64); + expect(parseHttpsPinRouteId(buildHttpsPinRouteBaseUrl(id))).toBe(id); + expect( + parseHttpsPinRouteId(`http://${HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST}:22438/route/${id}`), + ).toBe(id); + expect(parseHttpsPinRouteId(`${buildHttpsPinRouteBaseUrl(id)}/v1`)).toBeNull(); + expect(parseHttpsPinRouteId(`${buildHttpsPinRouteBaseUrl(id)}?secret=1`)).toBeNull(); + expect(parseHttpsPinRouteId(`https://example.test/route/${id}`)).toBeNull(); + }); +}); diff --git a/src/lib/inference/https-pin-runtime.ts b/src/lib/inference/https-pin-runtime.ts new file mode 100644 index 00000000000..9051025ae73 --- /dev/null +++ b/src/lib/inference/https-pin-runtime.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTPS DNS-pinning runtime adapter: classification and hidden adapter + * constants for arbitrary DNS-backed HTTPS custom inference endpoints + * (`compatible-endpoint` / `compatible-anthropic-endpoint`). + * + * A DNS-backed HTTPS custom endpoint cannot be registered with OpenShell + * directly: OpenShell's gateway re-resolves the hostname when it makes its + * own outbound connection, which can race the SSRF preflight's resolution + * (TOCTOU/DNS rebinding) and exposes the real hostname to that runtime + * boundary. This module classifies which endpoints need the adapter; the + * adapter itself (`https-pin-runtime-adapter.ts`) terminates the pinned + * outbound HTTPS connection on the host, immediately after the addresses it + * connects to were validated, and registers only its own loopback-adjacent + * `host.openshell.internal` route with OpenShell. + */ + +import crypto from "node:crypto"; +import { isIP } from "node:net"; + +import { HTTPS_PIN_RUNTIME_ADAPTER_PORT } from "../core/ports"; +import { isOpenShellManagedHost } from "./endpoint-ssrf-preflight"; + +/** Route-scoped bearer token OpenShell uses to reach this adapter. Never the real upstream credential. */ +export const HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV = + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN"; +/** Host-only control-plane secret used to manage the shared adapter process. Never registered with OpenShell. */ +export const HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV = + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN"; +export const HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST = "0.0.0.0"; +export const HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST = "127.0.0.1"; +export const HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST = "host.openshell.internal"; +export const HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN = `http://${HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST}:${HTTPS_PIN_RUNTIME_ADAPTER_PORT}`; +export const HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN = `http://${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST}:${HTTPS_PIN_RUNTIME_ADAPTER_PORT}`; + +export type HttpsPinCredentialProviderType = "openai" | "anthropic"; + +export interface HttpsPinCredentialHeader { + name: string; + value: string; +} + +/** Upstream credential header for the real endpoint, matching each provider type's existing convention. */ +export function resolveHttpsPinCredentialHeader( + providerType: HttpsPinCredentialProviderType, + credentialValue: string, +): HttpsPinCredentialHeader { + if (providerType === "anthropic") { + return { name: "x-api-key", value: credentialValue }; + } + return { name: "authorization", value: `Bearer ${credentialValue}` }; +} + +function parseUrl(value: string | URL | null | undefined): URL | null { + const raw = value instanceof URL ? value.href : String(value || "").trim(); + if (!raw) return null; + try { + return new URL(raw); + } catch { + return null; + } +} + +/** + * True when `endpointUrl` is exactly the shape that the DNS-pinning adapter + * exists for: HTTPS, a DNS-backed hostname (not an IP literal), and not one + * of NemoClaw's own trusted OpenShell-managed aliases. HTTP endpoints are + * already handled by direct IP substitution; HTTPS IP-literal endpoints + * already connect to an address the caller can see up front; OpenShell + * aliases are already exempt loopback-equivalent routes. + */ +export function isHttpsPinRuntimeEligible(endpointUrl: string | URL | null | undefined): boolean { + const url = parseUrl(endpointUrl); + if (!url || url.protocol !== "https:") return false; + const hostname = url.hostname; + const bare = + hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + if (isIP(bare)) return false; + if (isOpenShellManagedHost(hostname)) return false; + return true; +} + +/** Deterministic, stable identifier for one (gateway, provider, endpoint) route. Safe to persist and log. */ +export function computeHttpsPinRouteId( + gatewayName: string, + provider: string, + endpointUrl: string, +): string { + return crypto + .createHash("sha256") + .update(`nemoclaw:https-pin-route-id:v1\0${gatewayName}\0${provider}\0${endpointUrl}`) + .digest("hex"); +} + +/** Opaque sandbox-facing base URL for one route; never includes source URL material. */ +export function buildHttpsPinRouteBaseUrl(routeId: string): string { + return `${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/${routeId}`; +} + +/** Host-side (loopback) equivalent of {@link buildHttpsPinRouteBaseUrl}, for health checks and the control plane. */ +export function buildHttpsPinRouteLoopbackBaseUrl(routeId: string): string { + return `${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}/route/${routeId}`; +} + +/** + * Parse only the opaque adapter route shape persisted by NemoClaw. + * + * The port is intentionally not compared with the current process setting: + * cleanup must still recognize routes persisted before an explicit adapter + * port change. + */ +export function parseHttpsPinRouteId(baseUrl: string | null | undefined): string | null { + const url = parseUrl(baseUrl); + if ( + !url || + url.protocol !== "http:" || + url.hostname !== HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST || + url.search || + url.hash || + url.username || + url.password + ) { + return null; + } + return url.pathname.match(/^\/route\/([0-9a-f]{64})$/u)?.[1] ?? null; +} diff --git a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts index c901ca2b4ed..36894e3990f 100644 --- a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts +++ b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts @@ -11,6 +11,7 @@ import { DASHBOARD_PORT_RANGE_END, DASHBOARD_PORT_RANGE_START, GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT, OPENROUTER_RUNTIME_ADAPTER_PORT, @@ -222,6 +223,7 @@ function validateAdapterPortConfiguration(): void { ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }, ); } diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index 84b8d94b876..335073d7c1b 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -12,6 +12,7 @@ import { DASHBOARD_PORT_RANGE_END, DASHBOARD_PORT_RANGE_START, GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT, OPENROUTER_RUNTIME_ADAPTER_PORT, @@ -112,6 +113,7 @@ function resolveGatewayRecoveryTarget(options: StartGatewayForRecoveryOptions = ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }); return { gatewayName, gatewayPort }; } diff --git a/src/lib/onboard/inference-providers/hermes.ts b/src/lib/onboard/inference-providers/hermes.ts index 3f5fb848e09..f1527d53330 100644 --- a/src/lib/onboard/inference-providers/hermes.ts +++ b/src/lib/onboard/inference-providers/hermes.ts @@ -53,9 +53,10 @@ export async function setupHermesProviderInference( } // DNS-resolving + pinning validation closes the DNS-rebinding gap a // string-only hostname check leaves open. For HTTP this returns the - // pinned-IP URL. DNS-backed HTTPS fails closed until NemoClaw has a - // runtime-aware transport that can preserve TLS SNI/Host while pinning the - // resolved peer IP across the downstream OpenShell boundary. + // pinned-IP URL. DNS-backed HTTPS fails closed here: onboarding does not + // wire the HTTPS Pin Runtime adapter (see inference-set-route-containment.ts), + // so a DNS-backed HTTPS endpoint must be set after onboarding via + // `inference set --endpoint-url`. try { const validated = await rewriteConfigUrlsWithDnsPinning(endpointUrl, deps.lookup); resolvedEndpointUrl = typeof validated === "string" ? validated : endpointUrl; diff --git a/src/lib/sandbox/config.ts b/src/lib/sandbox/config.ts index a7b8d906ea8..7e30d3ebd4f 100644 --- a/src/lib/sandbox/config.ts +++ b/src/lib/sandbox/config.ts @@ -153,10 +153,11 @@ function buildConfigSetRestartGuidance(sandboxName: string, agentName: string): ]; } -class ConfigUrlValidationError extends Error { +export class ConfigUrlValidationError extends Error { constructor( readonly urlValue: string, message: string, + readonly reason: "dns_backed_https_unsupported" | "invalid" = "invalid", ) { super(message); this.name = "ConfigUrlValidationError"; @@ -950,16 +951,22 @@ async function rewriteConfigUrlsWithDnsPinning( // HTTPS endpoints fail closed for generic persisted config because the // downstream consumer would otherwise perform a second DNS lookup while // NemoClaw cannot pin the peer IP and preserve TLS SNI/Host across the - // OpenShell runtime boundary. + // OpenShell runtime boundary. This validator handles arbitrary persisted + // config values, not just inference endpoints, so the message stays + // generic; callers that know the field is an inference endpoint add + // their own guidance by checking `reason` on the thrown error. if (validated.protocol === "https:" && validated.pinnedUrl !== validated.originalUrl) { - throw new Error( - "DNS-backed HTTPS URLs are not supported for persisted sandbox config yet. " + - "Use an HTTPS IP-literal endpoint, an HTTP endpoint that can be DNS-pinned, " + - "or wait for the runtime-aware HTTPS pinning transport.", + throw new ConfigUrlValidationError( + trimmed, + "DNS-backed HTTPS URLs are not supported for arbitrary persisted sandbox config " + + "values. Use an HTTPS IP-literal endpoint or an HTTP endpoint that can be " + + "DNS-pinned.", + "dns_backed_https_unsupported", ); } return validated.protocol === "http:" ? validated.pinnedUrl : validated.originalUrl; } catch (err: unknown) { + if (err instanceof ConfigUrlValidationError) throw err; const message = err instanceof Error ? err.message : String(err); throw new ConfigUrlValidationError(trimmed, message); } diff --git a/src/lib/subprocess-env.test.ts b/src/lib/subprocess-env.test.ts index 97fd9022e76..2e417b539d7 100644 --- a/src/lib/subprocess-env.test.ts +++ b/src/lib/subprocess-env.test.ts @@ -196,3 +196,75 @@ describe("buildSubprocessEnv NO_PROXY injection", () => { expect(env.NO_PROXY).toBe(LOCAL_NO_PROXY); }); }); + +describe("buildMinimalCredentialAdapterEnv", () => { + const originalEnv = process.env; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("excludes toolchain, proxy, and unrelated secret-like variables even when ambient", async () => { + process.env.DOCKER_HOST = "unix:///var/run/docker.sock"; + process.env.KUBECONFIG = "/home/user/.kube/config"; + process.env.SSH_AUTH_SOCK = "/tmp/ssh-agent.sock"; + process.env.HTTP_PROXY = "http://proxy.example.com:8888"; + process.env.HTTPS_PROXY = "http://proxy.example.com:8888"; + process.env.NO_PROXY = "corp.internal"; + process.env.NVIDIA_INFERENCE_API_KEY = "should-not-leak"; + process.env.GITHUB_TOKEN = "should-not-leak"; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.DOCKER_HOST).toBeUndefined(); + expect(env.KUBECONFIG).toBeUndefined(); + expect(env.SSH_AUTH_SOCK).toBeUndefined(); + expect(env.HTTP_PROXY).toBeUndefined(); + expect(env.HTTPS_PROXY).toBeUndefined(); + expect(env.NO_PROXY).toBeUndefined(); + expect(env.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + }); + + it("includes only the documented runtime and TLS names", async () => { + process.env.HOME = "/home/user"; + process.env.PATH = "/usr/bin:/bin"; + process.env.NODE_ENV = "production"; + process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/corp-ca.pem"; + process.env.SSL_CERT_FILE = "/etc/ssl/cert.pem"; + process.env.USER = "someone-else-is-not-included"; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.HOME).toBe("/home/user"); + expect(env.PATH).toBe("/usr/bin:/bin"); + expect(env.NODE_ENV).toBe("production"); + expect(env.NODE_EXTRA_CA_CERTS).toBe("/etc/ssl/corp-ca.pem"); + expect(env.SSL_CERT_FILE).toBe("/etc/ssl/cert.pem"); + expect(env.USER).toBeUndefined(); + }); + + it("merges explicit adapter bootstrap fields via extra", async () => { + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv({ + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT: "9999", + }); + expect(env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT).toBe("9999"); + }); + + it("does not inject NO_PROXY the way buildSubprocessEnv does, since PROXY vars are never forwarded", async () => { + process.env.HTTP_PROXY = "http://proxy.example.com:8888"; + delete process.env.NO_PROXY; + delete process.env.no_proxy; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.NO_PROXY).toBeUndefined(); + expect(env.no_proxy).toBeUndefined(); + }); +}); diff --git a/src/lib/subprocess-env.ts b/src/lib/subprocess-env.ts index 54067365810..f334e79ef6a 100644 --- a/src/lib/subprocess-env.ts +++ b/src/lib/subprocess-env.ts @@ -132,3 +132,34 @@ export function buildSubprocessEnv(extra?: Record): Record, +): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value === undefined) continue; + if (ADAPTER_RUNTIME_NAMES.includes(key) || TLS.includes(key)) { + env[key] = value; + } + } + if (extra) { + Object.assign(env, extra); + } + return env; +} diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts index fae00f17944..6af84a4e29b 100644 --- a/test/cloudflared-update-check-workflow.test.ts +++ b/test/cloudflared-update-check-workflow.test.ts @@ -40,7 +40,7 @@ function pinValues(source: string, name: string): string[] { function writePinFixture(file: string, version: string, sha256: string): void { fs.writeFileSync( file, - ["one", "two", "three"] + ["one", "two", "three", "four"] .map( (job) => ` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`, @@ -149,11 +149,11 @@ describe("cloudflared update-check workflow contract", () => { expect(checkout?.with?.["persist-credentials"]).toBe(false); }); - it("extracts exactly three identical reviewed version and SHA256 pins", () => { + it("extracts exactly four identical reviewed version and SHA256 pins", () => { const versions = pinValues(e2e, "CLOUDFLARED_VERSION"); const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256"); - expect(versions).toHaveLength(3); - expect(hashes).toHaveLength(3); + expect(versions).toHaveLength(4); + expect(hashes).toHaveLength(4); expect(new Set(versions).size).toBe(1); expect(new Set(hashes).size).toBe(1); expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u); @@ -203,7 +203,7 @@ describe("cloudflared update-check workflow contract", () => { ); expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:"); expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:"); - expect(fixture.result.stderr).toContain("Set all three version/SHA256 pairs"); + expect(fixture.result.stderr).toContain("Set all four version/SHA256 pairs"); } finally { fs.rmSync(fixture.tempDir, { recursive: true, force: true }); } diff --git a/test/e2e/live/cloudflared-prerequisite.ts b/test/e2e/live/cloudflared-prerequisite.ts new file mode 100644 index 00000000000..e6a5662e668 --- /dev/null +++ b/test/e2e/live/cloudflared-prerequisite.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import YAML from "yaml"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; + +const CLOUDLFARED_STEP_NAME = "Install and verify cloudflared prerequisite"; + +interface CloudflaredPin { + version: string; + debSha256: string; +} + +function requirePin(value: unknown, label: string, pattern: RegExp): string { + if (typeof value !== "string" || !pattern.test(value)) { + throw new Error(`inference-routing ${label} is missing or invalid`); + } + return value; +} + +export function readInferenceRoutingCloudflaredPin( + workflowPath = path.join(REPO_ROOT, ".github", "workflows", "e2e.yaml"), +): CloudflaredPin { + const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as { + jobs?: Record }> }>; + }; + const step = workflow.jobs?.["inference-routing"]?.steps?.find( + (candidate) => candidate.name === CLOUDLFARED_STEP_NAME, + ); + return { + version: requirePin( + step?.env?.CLOUDFLARED_VERSION, + "cloudflared version pin", + /^\d+\.\d+\.\d+$/, + ), + debSha256: requirePin( + step?.env?.CLOUDFLARED_DEB_SHA256, + "cloudflared SHA256 pin", + /^[0-9a-f]{64}$/, + ), + }; +} + +async function commandOutput( + host: HostCliClient, + command: string, + args: string[], + artifactName: string, +): Promise { + const result = await host.command(command, args, { + artifactName: `inference-routing-cloudflared-${artifactName}`, + captureLimitBytes: 4 * 1024 * 1024, + cwd: REPO_ROOT, + timeoutMs: 120_000, + }); + if (result.exitCode !== 0) { + throw new Error( + `${command} failed while preparing cloudflared: ${result.stderr || result.stdout}`, + ); + } + return result.stdout; +} + +export async function resolveVerifiedCloudflaredBinary( + cleanup: Pick, + host: HostCliClient, + runtime: Pick = process, +): Promise { + if (runtime.platform !== "linux" || runtime.arch !== "x64") { + throw new Error("cloudflared is required for the DNS-backed HTTPS routing proof"); + } + + const pin = readInferenceRoutingCloudflaredPin(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-")); + cleanup.add(`remove verified cloudflared prerequisite ${root}`, () => { + fs.rmSync(root, { recursive: true, force: true }); + }); + const deb = path.join(root, `cloudflared-${pin.version}-linux-amd64.deb`); + const url = + `https://github.com/cloudflare/cloudflared/releases/download/${pin.version}/` + + "cloudflared-linux-amd64.deb"; + await commandOutput( + host, + "curl", + ["--fail", "--location", "--proto", "=https", "--proto-redir", "=https", url, "--output", deb], + "download", + ); + + const actualSha256 = createHash("sha256").update(fs.readFileSync(deb)).digest("hex"); + if (actualSha256 !== pin.debSha256) { + throw new Error(`cloudflared package SHA256 mismatch: expected ${pin.debSha256}`); + } + const packageName = ( + await commandOutput(host, "dpkg-deb", ["-f", deb, "Package"], "package-name") + ).trim(); + const version = ( + await commandOutput(host, "dpkg-deb", ["-f", deb, "Version"], "package-version") + ).trim(); + const architecture = ( + await commandOutput(host, "dpkg-deb", ["-f", deb, "Architecture"], "package-architecture") + ).trim(); + if (packageName !== "cloudflared" || version !== pin.version || architecture !== "amd64") { + throw new Error( + `unexpected cloudflared package metadata: package=${packageName} version=${version} architecture=${architecture}`, + ); + } + + const extracted = path.join(root, "extracted"); + await commandOutput(host, "dpkg-deb", ["-x", deb, extracted], "extract"); + const binary = path.join(extracted, "usr", "bin", "cloudflared"); + fs.accessSync(binary, fs.constants.X_OK); + const reportedVersion = await commandOutput(host, binary, ["--version"], "verify-version"); + if (!reportedVersion.includes(`cloudflared version ${pin.version}`)) { + throw new Error(`unexpected cloudflared version output: ${reportedVersion.trim()}`); + } + return binary; +} diff --git a/test/e2e/live/https-pin-compatible-server.ts b/test/e2e/live/https-pin-compatible-server.ts new file mode 100644 index 00000000000..97a5d783d4a --- /dev/null +++ b/test/e2e/live/https-pin-compatible-server.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; + +import { + closeServer, + writeJsonResponse as jsonResponse, + listenServer as listenOnRandomPort, + readRequestBody, +} from "../fixtures/http-protocol.ts"; +import type { StartedHttpServer } from "./mcp-bridge-servers.ts"; + +export interface FakeHttpsCompatibleRequest { + readonly method: string; + readonly path: string; + readonly hostHeader?: string; + readonly auth: "ok" | "missing" | "invalid"; + readonly body: string; +} + +export interface FakeHttpsCompatibleServer extends StartedHttpServer { + requests(): readonly FakeHttpsCompatibleRequest[]; + setChatRedirect(location: string | null): void; +} + +function requireTcpPort(server: https.Server): number { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("fake HTTPS compatible endpoint did not bind to a TCP port"); + } + return address.port; +} + +function generateEphemeralTlsMaterial(): { dir: string; cert: Buffer; key: Buffer } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-https-pin-tls-")); + const keyPath = path.join(dir, "server.key"); + const certPath = path.join(dir, "server.crt"); + // Only the loopback hop from cloudflared to this process consumes this + // certificate, and the quick tunnel is launched with --no-tls-verify for + // that local origin (test/e2e/setup-mcp-test-tls.sh documents the identical + // rationale for the MCP HTTPS fixture). A self-signed leaf with no separate + // CA is sufficient; the sandbox only ever sees the public tunnel origin and + // its real, publicly trusted trycloudflare.com certificate. + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-sha256", + "-nodes", + "-days", + "1", + "-subj", + "/CN=nemoclaw-https-pin-e2e", + "-keyout", + keyPath, + "-out", + certPath, + ], + { killSignal: "SIGKILL", stdio: "ignore", timeout: 15_000 }, + ); + return { dir, cert: fs.readFileSync(certPath), key: fs.readFileSync(keyPath) }; +} + +/** + * A minimal OpenAI-compatible HTTPS endpoint for proving the HTTPS-pin + * runtime adapter's live routing (#6141): authenticated `/v1/models` and + * `/v1/chat/completions`, plus a request ledger so the test can assert on + * exactly what the pinned adapter forwarded (Host header, credential, + * method/path) once this server sits behind a real public tunnel. + */ +export async function startFakeHttpsCompatibleServer(options: { + apiKey: string; + model: string; + chatContent?: string; +}): Promise { + const tls = generateEphemeralTlsMaterial(); + const requests: FakeHttpsCompatibleRequest[] = []; + const chatContent = options.chatContent ?? "ok"; + let chatRedirectLocation: string | null = null; + + const server = https.createServer({ cert: tls.cert, key: tls.key }, async (req, res) => { + const requestPath = new URL(req.url ?? "/", "https://https-pin.local").pathname; + const body = req.method === "HEAD" ? "" : await readRequestBody(req); + const authHeader = req.headers.authorization ?? ""; + const auth: FakeHttpsCompatibleRequest["auth"] = + authHeader === `Bearer ${options.apiKey}` ? "ok" : authHeader ? "invalid" : "missing"; + + // The public quick-tunnel readiness probe issues an unauthenticated HEAD + // request. Keep it out of the request ledger so offset-based assertions + // in the test only ever measure real chat-completion traffic. + if (req.method !== "HEAD") { + requests.push({ + method: req.method ?? "", + path: requestPath, + hostHeader: req.headers.host, + auth, + body, + }); + } + + if (auth !== "ok") { + jsonResponse(res, 401, { error: { message: "missing bearer credential" } }); + return; + } + if ( + ["GET", "HEAD"].includes(req.method ?? "") && + ["/models", "/v1/models"].includes(requestPath) + ) { + jsonResponse(res, 200, { object: "list", data: [{ id: options.model, object: "model" }] }); + return; + } + if ( + req.method === "POST" && + ["/chat/completions", "/v1/chat/completions"].includes(requestPath) + ) { + if (chatRedirectLocation) { + res.writeHead(302, { Location: chatRedirectLocation }); + res.end(); + return; + } + jsonResponse(res, 200, { + id: "chatcmpl-https-pin", + object: "chat.completion", + created: 0, + model: options.model, + choices: [ + { index: 0, message: { role: "assistant", content: chatContent }, finish_reason: "stop" }, + ], + }); + return; + } + jsonResponse(res, 404, { error: { message: "not found" } }); + }); + + await listenOnRandomPort(server); + return { + port: requireTcpPort(server), + requests: () => requests, + setChatRedirect: (location) => { + chatRedirectLocation = location; + }, + close: async () => { + await closeServer(server); + fs.rmSync(tls.dir, { recursive: true, force: true }); + }, + }; +} diff --git a/test/e2e/live/inference-routing.test.ts b/test/e2e/live/inference-routing.test.ts index f8a2259b465..62268ac6c8b 100644 --- a/test/e2e/live/inference-routing.test.ts +++ b/test/e2e/live/inference-routing.test.ts @@ -5,11 +5,20 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN } from "../../../src/lib/inference/https-pin-runtime.ts"; +import { REGISTRY_FILE, type SandboxEntry } from "../../../src/lib/state/registry.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import { resolveVerifiedCloudflaredBinary } from "./cloudflared-prerequisite.ts"; +import { + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./dns-rebinding-hosts-fixture.ts"; +import { startFakeHttpsCompatibleServer } from "./https-pin-compatible-server.ts"; import { CREDENTIAL_CLASSIFICATION_PATTERN, cleanupSandbox, @@ -27,6 +36,7 @@ import { TRANSPORT_CLASSIFICATION_PATTERN, writeFakeOpenShellForBlueprintFailClosed, } from "./inference-routing-helpers.ts"; +import { startPublicMcpHttpsTunnel } from "./mcp-bridge-servers.ts"; // This is the PR-required inference-routing lane. Credential-backed provider // smokes live in inference-routing-provider-smoke.test.ts and are never selected @@ -374,3 +384,354 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere }), ); }); + +test("TC-INF-11 DNS-backed HTTPS custom endpoint routes through the local pinning adapter (#6141)", { + timeout: 20 * 60_000, + meta: { + e2ePhases: [ + "confirm live inference prerequisites", + "clear the HTTPS pin sandbox", + "start the public HTTPS compatible endpoint", + "onboard with the placeholder endpoint", + "reject credential-bearing endpoint state", + "switch to the DNS-backed HTTPS endpoint", + "verify pinned route isolation and DNS rebinding", + "verify private redirect rejection", + ], + }, +}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + progress.phase("confirm live inference prerequisites"); + await requireLivePrerequisites(host, skip); + const model = "nemoclaw-e2e-https-pin"; + const apiKey = "sk-https-pin-TEST-NOT-A-REAL-VALUE"; + const sandboxName = inferenceSandboxName("e2e-https-pin"); + cleanup.add(`best-effort inference-routing https-pin cleanup for ${sandboxName}`, () => + cleanupSandbox(host, sandbox, sandboxName), + ); + progress.phase("clear the HTTPS pin sandbox"); + await cleanupSandbox(host, sandbox, sandboxName); + + progress.phase("start the public HTTPS compatible endpoint"); + const fake = await startFakeHttpsCompatibleServer({ apiKey, chatContent: "PONG", model }); + cleanup.add("close https-pin fake HTTPS compatible server", async () => { + try { + await artifacts.writeJson("tc-inf-11-https-pin-endpoint-requests.json", fake.requests()); + } finally { + await fake.close(); + } + }); + + // A genuinely public, DNS-resolvable, publicly-trusted-certificate origin + // is required: the adapter's SSRF preflight rejects loopback/private + // addresses, and only a real TLS trust chain exercises its SNI-pinned + // certificate validation. This reuses the same trycloudflare.com quick + // tunnel mechanism as the MCP-bridge DNS-rebinding coverage. + const cloudflaredBin = await resolveVerifiedCloudflaredBinary(cleanup, host); + const tunnel = await startPublicMcpHttpsTunnel({ + cloudflaredBin, + cleanup, + label: "https-pin inference routing", + progress, + readinessPath: "/v1/models", + readinessStatus: 401, + server: fake, + }); + const endpointUrl = `${tunnel.origin}/v1`; + const endpointHostname = new URL(tunnel.origin).hostname; + + await artifacts.target.declare({ + id: "https-pin-runtime-adapter-dns-backed-endpoint", + issue: 6141, + contract: [ + "inference set routes a DNS-backed HTTPS endpoint through the local pinning adapter", + "the real upstream hostname is never persisted to the NemoClaw sandbox registry", + "credential-bearing query and userinfo endpoints are rejected without changing host state", + "OpenShell's own policy view never references the real upstream hostname", + "a real chat completion round-trips through the pinned TLS connection to the public endpoint", + "a DNS rebind of the upstream hostname after inference set does not redirect adapter traffic", + "an upstream redirect to a private target is rejected without relaying Location or reaching the target", + ], + endpointUrl, + model, + }); + + // Onboarding's own SSRF preflight (assertEndpointResolvesPublic) only + // rejects private/internal addresses; it does not fail closed on + // DNS-backed HTTPS the way the HTTPS Pin Runtime adapter's call site does, + // and onboarding never wires that adapter itself (only + // inference-set-route-containment.ts's normalizeCustomEndpointUrl does, on + // the `inference set --endpoint-url` path). Onboard with a disposable + // plain-HTTP placeholder endpoint first -- the same shape TC-INF-09 already + // onboards successfully with -- then switch to the DNS-backed HTTPS + // endpoint through `inference set --endpoint-url`, the actual #6141 call + // site this test exercises. + // Advertise localhost so onboarding exercises its host-bridge rewrite, but + // listen beyond host loopback so the resulting sandbox route can reach it. + const placeholder = await startFakeOpenAiCompatibleServer({ + apiKey, + chatContent: "placeholder", + host: "0.0.0.0", + model, + port: 8000, + progress, + publicHost: "localhost", + requireAuth: true, + requireAuthModels: true, + }); + cleanup.add("close https-pin onboarding placeholder endpoint", () => placeholder.close()); + + progress.phase("onboard with the placeholder endpoint"); + const onboard = await onboardSandbox( + artifacts, + sandboxName, + { + COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_ENDPOINT_URL: placeholder.baseUrl, + NEMOCLAW_MODEL: model, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }, + [apiKey], + "tc-inf-11-onboard-https-pin-placeholder", + progress, + 15 * 60_000, + ); + expectOnboardSuccess(onboard, "TC-INF-11 https-pin-endpoint placeholder onboard"); + cleanup.add(`strict inference-routing https-pin cleanup for ${sandboxName}`, () => + cleanupSandbox(host, sandbox, sandboxName, { strict: true }), + ); + + progress.phase("reject credential-bearing endpoint state"); + const userinfoEndpoint = new URL(endpointUrl); + userinfoEndpoint.username = "e2e-user"; + userinfoEndpoint.password = apiKey; + for (const [shape, credentialEndpoint] of [ + ["userinfo", userinfoEndpoint.toString()], + ["query", `${endpointUrl}?api_key=${encodeURIComponent(apiKey)}`], + ] as const) { + const rejected = await runNemoclawCli( + [ + "inference", + "set", + "--provider", + "compatible-endpoint", + "--model", + model, + "--sandbox", + sandboxName, + "--endpoint-url", + credentialEndpoint, + "--credential-env", + "COMPATIBLE_API_KEY", + "--inference-api", + "openai-completions", + ], + { + artifactName: `tc-inf-11-reject-${shape}-endpoint`, + artifacts, + env: { ...buildAvailabilityProbeEnv(), COMPATIBLE_API_KEY: apiKey }, + progress, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + const rejectedText = redactedResultText(rejected); + expect(rejected.exitCode, rejectedText).not.toBe(0); + expect(rejectedText).toContain("without userinfo, query, or fragment components"); + const unchangedRegistry = fs.readFileSync(REGISTRY_FILE, "utf8"); + expect(unchangedRegistry).not.toContain(apiKey); + expect(unchangedRegistry).not.toContain(endpointHostname); + } + + progress.phase("switch to the DNS-backed HTTPS endpoint"); + const inferenceSet = await runNemoclawCli( + [ + "inference", + "set", + "--provider", + "compatible-endpoint", + "--model", + model, + "--sandbox", + sandboxName, + "--endpoint-url", + endpointUrl, + "--credential-env", + "COMPATIBLE_API_KEY", + "--inference-api", + "openai-completions", + ], + { + artifactName: "tc-inf-11-inference-set-https-pin-endpoint", + artifacts, + env: { ...buildAvailabilityProbeEnv(), COMPATIBLE_API_KEY: apiKey }, + progress, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect( + inferenceSet.exitCode, + `TC-INF-11 inference set https-pin endpoint failed\n${redactedResultText(inferenceSet)}`, + ).toBe(0); + + // The real hostname must never reach the NemoClaw sandbox registry on + // disk: only the local adapter's host.openshell.internal route is + // persisted (#6141 requirement: hostname hidden from the runtime + // boundary; credential-bearing URL state is never persisted in plaintext). + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const registryEntry = registry.sandboxes?.[sandboxName]; + expect(registryEntry?.endpointUrl ?? "").toContain( + `${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/`, + ); + expect(registryEntry?.endpointUrl ?? "").not.toContain(endpointHostname); + + const provider = await sandbox.openshell( + ["provider", "get", "-g", "nemoclaw", "compatible-endpoint"], + { + artifactName: "tc-inf-11-provider-get-compatible-endpoint", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + const providerText = resultText(provider).replace(/\u001b\[[0-9;]*m/g, ""); + expect(provider.exitCode, providerText).toBe(0); + expect(providerText).toContain("Type: openai"); + expect(providerText).toContain("Credential keys: COMPATIBLE_API_KEY"); + expect(providerText).toContain("Config keys: OPENAI_BASE_URL"); + + progress.phase("verify pinned route isolation and DNS rebinding"); + // OpenShell's own network-policy view is a second, independent witness: + // it must never learn the real upstream hostname either, only the local + // adapter's host.openshell.internal boundary that everything else here + // already resolves through. + const policy = await sandbox.openshell(["policy", "get", "--full", sandboxName], { + artifactName: "tc-inf-11-policy-get-https-pin", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const policyText = resultText(policy).replace(/\u001b\[[0-9;]*m/g, ""); + expect(policy.exitCode, policyText).toBe(0); + expect(policyText).not.toContain(endpointHostname); + + const sandboxRequestOffset = fake.requests().length; + // OpenShell 0.0.85 refreshes the sandbox-side inference bundle every five + // seconds. Because this switch intentionally keeps the same provider/model + // identity while replacing only its endpoint binding, an immediate request + // can still use the placeholder route cached before `inference set`. Poll + // through two refresh intervals, but accept success only after the real + // pinned upstream records the authenticated request. + let routeProbeAttempt = 0; + await expect + .poll( + async () => { + routeProbeAttempt += 1; + await expectOpenAiChatThroughSandbox( + sandbox, + sandboxName, + model, + [apiKey], + `https-pin-endpoint-inference-local-chat-${routeProbeAttempt}`, + ); + return fake + .requests() + .slice(sandboxRequestOffset) + .some( + (request) => + request.auth === "ok" && + request.method === "POST" && + request.path === "/v1/chat/completions", + ); + }, + { interval: 5_000, timeout: 11_000 }, + ) + .toBe(true); + expect(fake.requests().slice(sandboxRequestOffset)).toContainEqual( + expect.objectContaining({ + auth: "ok", + method: "POST", + path: "/v1/chat/completions", + }), + ); + + // The assertions above only prove the *initial* `inference set` reached + // the real target. They do not prove the adapter is resistant to a DNS + // record changing after the route is already pinned -- the exact + // SSRF/DNS-rebinding vulnerability the pinning mechanism exists to close. + // Rebind the tunnel hostname to a reserved, unreachable documentation + // address (RFC 5737 TEST-NET-1) now that the route is registered: if the + // adapter re-resolved DNS per request instead of using the addresses it + // already pinned, this chat call would fail to connect instead of + // succeeding. + const hostsFixture = await setupDnsRebindingHostsFixture(host, sandboxName, endpointHostname); + cleanup.add(`restore https-pin DNS rebinding hosts fixture for ${sandboxName}`, () => + restoreDnsRebindingHostsFixture(host, sandboxName, hostsFixture), + ); + await remapDnsRebindingHostname( + host, + sandboxName, + hostsFixture, + "192.0.2.1", + "tc-inf-11-dns-rebind-after-inference-set", + ); + + const rebindRequestOffset = fake.requests().length; + await expectOpenAiChatThroughSandbox( + sandbox, + sandboxName, + model, + [apiKey], + "https-pin-endpoint-dns-rebinding-chat", + ); + expect(fake.requests().slice(rebindRequestOffset)).toContainEqual( + expect.objectContaining({ + auth: "ok", + method: "POST", + path: "/v1/chat/completions", + }), + ); + + await restoreDnsRebindingHostsFixture(host, sandboxName, hostsFixture); + + progress.phase("verify private redirect rejection"); + const privateTargetRequestOffset = placeholder.requests().length; + const redirectTarget = new URL("chat/completions", `${placeholder.baseUrl}/`).toString(); + fake.setChatRedirect(redirectTarget); + const redirectPayload = JSON.stringify({ + model, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + max_tokens: 50, + }); + const redirect = await sandbox.exec( + sandboxName, + [ + "curl", + "-sS", + "--include", + "--location", + "--max-redirs", + "3", + "--max-time", + "60", + "https://inference.local/v1/chat/completions", + "-H", + "Content-Type: application/json", + "--data-raw", + redirectPayload, + ], + { + artifactName: "tc-inf-11-private-redirect-rejection", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 90_000, + }, + ); + const redirectText = resultText(redirect); + expect(redirect.exitCode, redirectText).toBe(0); + expect(redirectText).toMatch(/HTTP\/1\.[01] 502/u); + expect(redirectText).toContain("redirect_blocked"); + expect(redirectText.toLowerCase()).not.toContain("location:"); + expect(placeholder.requests()).toHaveLength(privateTargetRequestOffset); +}); diff --git a/test/e2e/live/mcp-bridge-servers.ts b/test/e2e/live/mcp-bridge-servers.ts index c643a257e98..900c517eda3 100644 --- a/test/e2e/live/mcp-bridge-servers.ts +++ b/test/e2e/live/mcp-bridge-servers.ts @@ -192,38 +192,54 @@ export function buildCloudflaredQuickTunnelArgs(port: number): string[] { ]; } -async function probePublicTunnel(origin: string): Promise<{ +async function probePublicTunnel( + origin: string, + readinessPath: string, + readinessStatus: number, +): Promise<{ ready: boolean; diagnostic: string; }> { try { - const response = await fetch(`${origin}/mcp`, { + const response = await fetch(`${origin}${readinessPath}`, { method: "HEAD", redirect: "manual", signal: AbortSignal.timeout(5_000), }); await response.body?.cancel(); return { - ready: response.status === 405, - diagnostic: `public HEAD /mcp returned HTTP ${response.status}`, + ready: response.status === readinessStatus, + diagnostic: `public HEAD ${readinessPath} returned HTTP ${response.status}`, }; } catch (error) { return { ready: false, // Avoid reflecting request URLs or child output here. The error class is // enough to distinguish DNS/transport failure without risking headers. - diagnostic: `public HEAD /mcp failed (${error instanceof Error ? error.name : "unknown error"})`, + diagnostic: `public HEAD ${readinessPath} failed (${error instanceof Error ? error.name : "unknown error"})`, }; } } +/** + * Publishes a local HTTPS origin behind a real `trycloudflare.com` quick + * tunnel: a genuinely public, DNS-resolvable, publicly-trusted-certificate + * endpoint. Named for its original MCP-bridge fixture caller; reused as-is + * (via the optional readiness override below) for the HTTPS-pin runtime + * adapter's live coverage, since both need the identical real-tunnel proof + * and only differ in which local path/status means "ready". + */ export async function startPublicMcpHttpsTunnel(options: { cleanup: TunnelCleanupRegistry; label: string; progress: Pick & TestProgressCapability; server: StartedHttpServer; cloudflaredBin?: string; + readinessPath?: string; + readinessStatus?: number; }): Promise { + const readinessPath = options.readinessPath ?? "/mcp"; + const readinessStatus = options.readinessStatus ?? 405; const args = buildCloudflaredQuickTunnelArgs(options.server.port); let lastFailure = "cloudflared did not publish a quick-tunnel URL"; @@ -289,7 +305,7 @@ export async function startPublicMcpHttpsTunnel(options: { break; } if (origin) { - const probe = await probePublicTunnel(origin); + const probe = await probePublicTunnel(origin, readinessPath, readinessStatus); if (probe.ready) { consecutiveReadyProbes += 1; if (consecutiveReadyProbes >= QUICK_TUNNEL_CONSECUTIVE_READY_PROBES) { diff --git a/test/e2e/support/cloudflared-prerequisite.test.ts b/test/e2e/support/cloudflared-prerequisite.test.ts new file mode 100644 index 00000000000..55d9ccd12e2 --- /dev/null +++ b/test/e2e/support/cloudflared-prerequisite.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + readInferenceRoutingCloudflaredPin, + resolveVerifiedCloudflaredBinary, +} from "../live/cloudflared-prerequisite.ts"; + +describe("inference-routing cloudflared prerequisite (#6141)", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("reads the reviewed version and digest from the exact workflow", () => { + expect(readInferenceRoutingCloudflaredPin()).toEqual({ + version: "2026.6.1", + debSha256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + }); + }); + + it("rejects a workflow without an immutable digest", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-pin-")); + const workflow = path.join(root, "e2e.yaml"); + fs.writeFileSync( + workflow, + [ + "jobs:", + " inference-routing:", + " steps:", + " - name: Install and verify cloudflared prerequisite", + " env:", + ' CLOUDFLARED_VERSION: "2026.6.1"', + ' CLOUDFLARED_DEB_SHA256: "mutable"', + "", + ].join("\n"), + ); + try { + expect(() => readInferenceRoutingCloudflaredPin(workflow)).toThrow( + "inference-routing cloudflared SHA256 pin is missing or invalid", + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("ignores a PATH-injected binary and starts the verified package flow", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-path-")); + const injected = path.join(root, "cloudflared"); + fs.writeFileSync(injected, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + vi.stubEnv("PATH", `${root}${path.delimiter}${process.env.PATH ?? ""}`); + const command = vi.fn(async (name: string, args: string[]) => ({ + command: [name, ...args], + exitCode: 1, + signal: null, + timedOut: false, + stdout: "", + stderr: "download blocked by test", + artifacts: { stdout: "", stderr: "", result: "" }, + })); + const cleanup = new CleanupRegistry(); + + try { + await expect( + resolveVerifiedCloudflaredBinary(cleanup, { command } as unknown as HostCliClient, { + platform: "linux", + arch: "x64", + }), + ).rejects.toThrow("curl failed while preparing cloudflared"); + expect(command).toHaveBeenCalledTimes(1); + expect(command.mock.calls[0]?.[0]).toBe("curl"); + expect(command.mock.calls.flat().join(" ")).not.toContain(injected); + } finally { + await cleanup.runAll(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index af290259c9a..df6936a5aaa 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -348,27 +348,50 @@ describe("e2e workflow boundary", () => { } }); - // source-shape-contract: security -- Mutates the shipped workflow to prove PR-safe routing rejects credential-backed smokes + // source-shape-contract: security -- Mutates the shipped workflow to prove PR-safe routing rejects credential-backed smokes and mutable tunnel tooling it("rejects credential-backed provider smokes in the PR-safe inference-routing job", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inference-routing-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); const workflow = readWorkflow() as { - jobs: Record }>; + jobs: Record< + string, + { steps?: Array<{ name?: string; run?: string; env?: Record }> } + >; }; const run = workflow.jobs["inference-routing"]?.steps?.find( (step) => step.name === "Run inference routing live test", ); expect(run).toBeDefined(); run!.run = "npx vitest run --project e2e-live inference-routing-provider-smoke.test.ts"; + const prerequisite = workflow.jobs["inference-routing"]?.steps?.find( + (step) => step.name === "Install and verify cloudflared prerequisite", + ); + expect(prerequisite?.env).toBeDefined(); + prerequisite!.env!.CLOUDFLARED_VERSION = "latest"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + const digestWorkflowPath = path.join(tmp, "digest-workflow.yaml"); + const digestWorkflow = readWorkflow() as { + jobs: Record }> }>; + }; + const digestPrerequisite = digestWorkflow.jobs["inference-routing"]?.steps?.find( + (step) => step.name === "Install and verify cloudflared prerequisite", + ); + expect(digestPrerequisite?.env).toBeDefined(); + digestPrerequisite!.env!.CLOUDFLARED_DEB_SHA256 = "mutable"; + fs.writeFileSync(digestWorkflowPath, YAML.stringify(digestWorkflow)); + try { expect(validateE2eWorkflowBoundary(workflowPath)).toEqual( expect.arrayContaining([ "step 'Run inference routing live test' run script must include test/e2e/live/inference-routing.test.ts", "step 'Run inference routing live test' run script must not include inference-routing-provider-smoke.test.ts", + "inference-routing cloudflared prerequisite step must pin CLOUDFLARED_VERSION=2026.6.1", ]), ); + expect(validateE2eWorkflowBoundary(digestWorkflowPath)).toContain( + "inference-routing cloudflared prerequisite step must pin CLOUDFLARED_DEB_SHA256=ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + ); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index d8a1c43c4ae..1d16cc11b02 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -29,6 +29,7 @@ export type DestroyHarness = { prepareMcpBridgesForDestroySpy: MockInstance; promptSpy: MockInstance; removeSandboxSpy: MockInstance; + revokeHttpsPinRuntimeAdapterRouteSpy: MockInstance; restoreMcpBridgesAfterDestroyAbortSpy: MockInstance; runOpenshellSpy: MockInstance; selectGatewaySpy: MockInstance; @@ -44,6 +45,7 @@ type DestroyHarnessOptions = { deleteOutput?: string; deleteStatus?: number; dockerPsOutput?: string; + endpointUrl?: string; finalizeMcpError?: string; liveListOutput?: string; mcpAddState?: "prepared"; @@ -113,6 +115,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr const sandboxProviderCleanup = requireDist("../../onboard/sandbox-provider-cleanup.js"); const nim = requireDist("../../inference/nim.js"); const ollamaProxy = requireDist("../../inference/ollama/proxy.js"); + const httpsPinRuntimeAdapter = requireDist("../../inference/https-pin-runtime-adapter.js"); const tunnelServices = requireDist("../../tunnel/services.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); @@ -134,6 +137,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr vi.spyOn(registry, "getSandbox").mockReturnValue({ ...sandboxEntry, agent: options.agent ?? sandboxEntry.agent, + ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), ...(options.mcpServers?.length ? { mcp: { @@ -160,6 +164,9 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr registeredSandboxCount = Math.max(0, registeredSandboxCount - 1); return true; }); + const revokeHttpsPinRuntimeAdapterRouteSpy = vi + .spyOn(httpsPinRuntimeAdapter, "revokeHttpsPinRuntimeAdapterRoute") + .mockResolvedValue(true); vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", }); @@ -319,6 +326,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr prepareMcpBridgesForDestroySpy, promptSpy, removeSandboxSpy, + revokeHttpsPinRuntimeAdapterRouteSpy, restoreMcpBridgesAfterDestroyAbortSpy, runOpenshellSpy, selectGatewaySpy, diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index c383a0de2a5..e0066b41f3a 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1017,6 +1017,34 @@ function validateGatewayGuardRecoveryJob(errors: string[], jobs: WorkflowRecord) function validateInferenceRoutingJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "inference-routing"; const steps = asSteps(asRecord(jobs[jobName]).steps); + const cloudflaredPrereq = requireJobStep( + errors, + jobName, + steps, + "Install and verify cloudflared prerequisite", + ); + const cloudflaredPrereqEnv = asRecord(cloudflaredPrereq?.env); + if (cloudflaredPrereqEnv.CLOUDFLARED_VERSION !== REVIEWED_CLOUDFLARED_VERSION) { + errors.push( + `inference-routing cloudflared prerequisite step must pin CLOUDFLARED_VERSION=${REVIEWED_CLOUDFLARED_VERSION}`, + ); + } + if (cloudflaredPrereqEnv.CLOUDFLARED_DEB_SHA256 !== REVIEWED_CLOUDFLARED_DEB_SHA256) { + errors.push( + `inference-routing cloudflared prerequisite step must pin CLOUDFLARED_DEB_SHA256=${REVIEWED_CLOUDFLARED_DEB_SHA256}`, + ); + } + requireRunContains( + errors, + cloudflaredPrereq, + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb", + ); + requireRunContains(errors, cloudflaredPrereq, "sha256sum -c -"); + requireRunContains(errors, cloudflaredPrereq, "dpkg-deb -f"); + requireRunContains(errors, cloudflaredPrereq, "sudo dpkg -i"); + requireRunContains(errors, cloudflaredPrereq, "cloudflared version ${CLOUDFLARED_VERSION}"); + requireRunDoesNotContain(errors, cloudflaredPrereq, "pkg.cloudflare.com"); + requireRunDoesNotContain(errors, cloudflaredPrereq, "apt-get install"); const run = requireJobStep(errors, jobName, steps, "Run inference routing live test"); requireRunContains(errors, run, "test/e2e/live/inference-routing.test.ts"); requireRunDoesNotContain(errors, run, "inference-routing-provider-smoke.test.ts"); @@ -3130,8 +3158,8 @@ function runContainsCloudflaredAptInstall(run: string): boolean { ); } -const TUNNEL_LIFECYCLE_CLOUDFLARED_VERSION = "2026.6.1"; -const TUNNEL_LIFECYCLE_CLOUDFLARED_DEB_SHA256 = +const REVIEWED_CLOUDFLARED_VERSION = "2026.6.1"; +const REVIEWED_CLOUDFLARED_DEB_SHA256 = "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"; function validateTunnelLifecycleJob(errors: string[], jobs: WorkflowRecord): void { @@ -3220,14 +3248,14 @@ function validateTunnelLifecycleJob(errors: string[], jobs: WorkflowRecord): voi "NVIDIA_INFERENCE_API_KEY", ); requireRunContains(errors, cloudflaredPrereq, "cloudflared --version"); - if (cloudflaredPrereqEnv.CLOUDFLARED_VERSION !== TUNNEL_LIFECYCLE_CLOUDFLARED_VERSION) { + if (cloudflaredPrereqEnv.CLOUDFLARED_VERSION !== REVIEWED_CLOUDFLARED_VERSION) { errors.push( - `tunnel-lifecycle cloudflared prerequisite step must pin CLOUDFLARED_VERSION=${TUNNEL_LIFECYCLE_CLOUDFLARED_VERSION}`, + `tunnel-lifecycle cloudflared prerequisite step must pin CLOUDFLARED_VERSION=${REVIEWED_CLOUDFLARED_VERSION}`, ); } - if (cloudflaredPrereqEnv.CLOUDFLARED_DEB_SHA256 !== TUNNEL_LIFECYCLE_CLOUDFLARED_DEB_SHA256) { + if (cloudflaredPrereqEnv.CLOUDFLARED_DEB_SHA256 !== REVIEWED_CLOUDFLARED_DEB_SHA256) { errors.push( - `tunnel-lifecycle cloudflared prerequisite step must pin CLOUDFLARED_DEB_SHA256=${TUNNEL_LIFECYCLE_CLOUDFLARED_DEB_SHA256}`, + `tunnel-lifecycle cloudflared prerequisite step must pin CLOUDFLARED_DEB_SHA256=${REVIEWED_CLOUDFLARED_DEB_SHA256}`, ); } requireRunContains(