diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index fdd062b1c22..7eabc71140c 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -27,6 +27,18 @@ "name": "NEMOCLAW_BEDROCK_RUNTIME_REGION", "reason": "Internal child-process setting used only to pass the resolved Bedrock Runtime region to the hidden local adapter. Users should rely on the endpoint URL or standard AWS region environment variables." }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + "reason": "Internal child-process setting used only when launching the hidden HTTPS Pin Runtime adapter. The port is a fixed internal constant, not a public user-facing configuration knob." + }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_BOOTSTRAP_ROUTE", + "reason": "Internal child-process setting carrying a JSON-encoded route (including a credential value) used only to seed the hidden HTTPS Pin Runtime adapter at startup. Never user-set." + }, + { + "name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTE_IDS", + "reason": "Internal child-process setting carrying a JSON-encoded array of route ids (no credentials) that a fresh HTTPS Pin Runtime adapter respawn could not recover, so it can respond to them distinctly from a route that never existed. Never user-set." + }, { "name": "NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE", "reason": "Internal installer sentinel exported only during OpenShell gateway replacement so onboard restores the pre-upgrade sandbox backup. Not user-facing." diff --git a/docs/inference/custom-endpoint-security.mdx b/docs/inference/custom-endpoint-security.mdx index e8519400c17..59b95a62b27 100644 --- a/docs/inference/custom-endpoint-security.mdx +++ b/docs/inference/custom-endpoint-security.mdx @@ -9,6 +9,7 @@ keywords: ["nemoclaw endpoint security", "inference endpoint ssrf", "custom endp content: type: "concept" --- +import { AgentOnly } from "../_components/AgentGuide"; NemoClaw keeps provider credentials on the host and validates explicit custom endpoint URLs before saving them through security-sensitive configuration paths. @@ -44,9 +45,22 @@ Legacy entries without a source, endpoints recorded by `inference set`, and diff For a public HTTP URL, NemoClaw stores the validated IP address so the downstream runtime cannot resolve the hostname again and reach another address. -NemoClaw rejects DNS-backed HTTPS URLs in these paths because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary. -Use an HTTPS IP-literal endpoint with a certificate valid for that address. -Use a public HTTP endpoint only when your deployment permits unencrypted traffic. +NemoClaw rejects DNS-backed HTTPS URLs for Hermes Provider setup, host-side `config set`, and a direct blueprint run because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary. +For those paths, use an HTTPS IP-literal endpoint with a certificate valid for that address, or use a public HTTP endpoint if your deployment permits unencrypted traffic. + + + +## Use a DNS-Backed HTTPS Endpoint with `inference set` + +`$$nemoclaw inference set --endpoint-url ` on an already-onboarded sandbox supports a DNS-backed HTTPS custom endpoint through the HTTPS Pin Runtime adapter. +After SSRF validation passes, NemoClaw starts a local reverse-proxy adapter on the host that terminates a pinned, SNI-correct outbound TLS connection to the real upstream hostname, re-validating that the resolved peer IP is still public. +The sandbox, its OpenShell provider configuration and network policy, and the persisted sandbox registry only ever see a local route through `http://host.openshell.internal:/route//...`. +The real upstream hostname never reaches the sandbox or the persisted registry. + +This support is specific to `inference set` on an already-onboarded sandbox. +Hermes Provider setup, host-side `config set`, and a direct blueprint run still reject DNS-backed HTTPS URLs as described above. + + ## Use the Sandbox Host Alias diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 83bbe24e024..d6b2e094ac0 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2968,7 +2968,8 @@ NemoClaw rejects loopback, link-local, private, and internal endpoint addresses, For a same-provider model change, pass `--endpoint-url` with the exact canonical endpoint URL that the target sandbox registry identifies as onboarding-established. Missing or `inference set` provenance and every different URL remain subject to the full address validation above. For public HTTP URLs, NemoClaw stores the validated IP address to prevent DNS rebinding. -DNS-backed HTTPS URLs are rejected because NemoClaw cannot pin the downstream peer address while preserving TLS SNI and host validation across the OpenShell runtime boundary; HTTPS IP-literal URLs remain supported. +For a DNS-backed HTTPS URL, NemoClaw routes the endpoint through a local HTTPS Pin Runtime adapter that terminates a pinned, SNI-correct outbound connection to the real upstream hostname; the sandbox and the persisted registry only ever see a local `host.openshell.internal` route, never the real hostname. +HTTPS IP-literal URLs remain supported and do not need the adapter. NemoClaw accepts `http://host.openshell.internal:` only with an explicit port from `1024` through `65535`; this narrow exception supports NemoClaw's sandbox-to-host inference routes and is not a general private-endpoint bypass. `--credential-env` may also be supplied for compatible provider metadata; supported `--inference-api` values are `openai-completions`, `anthropic-messages`, and `openai-responses`. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 2062a70edff..5ad2c75afaf 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1922,7 +1922,8 @@ After the upgrade, recreate the sandbox with `$$nemoclaw onboard`. NemoClaw rejects an explicit custom endpoint when it resolves a public HTTPS hostname but cannot pin the same peer address across the downstream OpenShell runtime boundary while preserving TLS SNI and host validation. This can appear during a direct blueprint run, custom-endpoint onboarding, or a host-side `config set` write. -It can also appear during a runtime `$$nemoclaw inference set` switch. +It does not appear during a runtime `$$nemoclaw inference set` switch on an already-onboarded sandbox; that command routes a DNS-backed HTTPS endpoint through a local HTTPS Pin Runtime adapter instead of rejecting it. +Refer to [Commands](commands) for details. Use an HTTPS IP-literal endpoint whose certificate is valid for that address. diff --git a/nemoclaw/src/blueprint/ssrf.ts b/nemoclaw/src/blueprint/ssrf.ts index 4e86936d65b..791f4b89f45 100644 --- a/nemoclaw/src/blueprint/ssrf.ts +++ b/nemoclaw/src/blueprint/ssrf.ts @@ -112,11 +112,12 @@ export async function validateEndpointUrl(url: string): Promise { }, }, }); + // The DNS-backed HTTPS endpoint is pinned via the HTTPS-pin runtime + // adapter, so the persisted endpointUrl is the adapter's local route base + // URL, not the raw operator-supplied hostname — mirroring the existing + // HTTP precedent of persisting the validated/pinned address. The + // persisted credentialEnv is the adapter's own bearer-token env var, not + // the real upstream secret: the sandbox only ever authenticates to the + // local adapter, which injects the real credential on the outbound leg + // (PRA-1, #6141). expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ "alpha", expect.objectContaining({ provider: "compatible-endpoint", model: "mock-responses-model", - endpointUrl: "https://compatible.example/v1", - credentialEnv: "COMPATIBLE_API_KEY", + endpointUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, preferredInferenceApi: "openai-responses", }), ]); expect(deps.getSession()).toMatchObject({ provider: "compatible-endpoint", model: "mock-responses-model", - endpointUrl: "https://compatible.example/v1", - credentialEnv: "COMPATIBLE_API_KEY", + endpointUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, preferredInferenceApi: "openai-responses", }); expect(deps.calls.restartSandboxGateway).toHaveBeenCalledWith("alpha"); @@ -317,6 +327,12 @@ describe("runInferenceSet compatible providers", () => { }, rewriteConfigUrlsWithDnsPinning: (value) => actualConfig.rewriteConfigUrlsWithDnsPinning(value, lookup), + // DNS-backed HTTPS endpoints (the "DNS-private" case below) route + // through the HTTPS-pin runtime adapter instead of + // rewriteConfigUrlsWithDnsPinning, so its real SSRF preflight is + // exercised here too, with the same injected DNS lookup. + ensureHttpsPinRuntimeAdapter: (adapterOptions) => + realEnsureHttpsPinRuntimeAdapter({ ...adapterOptions, lookup }), }); await expect( diff --git a/src/lib/actions/inference-set-endpoint-security.test.ts b/src/lib/actions/inference-set-endpoint-security.test.ts index 586481cae2a..dc3c2f30ecd 100644 --- a/src/lib/actions/inference-set-endpoint-security.test.ts +++ b/src/lib/actions/inference-set-endpoint-security.test.ts @@ -57,4 +57,23 @@ describe("custom inference endpoint DNS pinning", () => { ), ).rejects.toThrow(/DNS-backed HTTPS URLs are not supported/); }); + + it("adds the HTTPS Pin Runtime adapter hint only at the inference-set call site, not in the generic config validator's own message (#6141)", async () => { + const lookup = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]); + + // The generic validator (also used by plain `config set` for arbitrary + // fields) must not mention inference set or the adapter -- it has no way + // to know the field it's validating is an inference endpoint. + await expect( + rewriteConfigUrlsWithDnsPinning("https://public-endpoint.example/v1/", lookup), + ).rejects.toThrow(/^(?!.*(?:inference set|HTTPS Pin Runtime adapter)).*$/is); + + // normalizeCustomEndpointUrl is only ever called for `inference set + // --endpoint-url`, so it appends the adapter-specific hint itself. + await expect( + normalizeCustomEndpointUrl("https://public-endpoint.example/v1/", (value) => + rewriteConfigUrlsWithDnsPinning(value, lookup), + ), + ).rejects.toThrow(/HTTPS Pin Runtime adapter/); + }); }); diff --git a/src/lib/actions/inference-set-gateway-route-containment.test.ts b/src/lib/actions/inference-set-gateway-route-containment.test.ts index 56b20907e3e..1e4ea364182 100644 --- a/src/lib/actions/inference-set-gateway-route-containment.test.ts +++ b/src/lib/actions/inference-set-gateway-route-containment.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; +import { HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV } from "../inference/https-pin-runtime"; import type { ConfigObject } from "../security/credential-filter"; import type { SandboxEntry } from "../state/registry"; import { runInferenceSet } from "./inference-set"; @@ -28,6 +29,7 @@ const entry = (name: string, overrides: Partial = {}): SandboxEntr describe("runtime shared gateway route containment", () => { afterEach(() => { vi.unstubAllEnvs(); + delete process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]; }); it("rejects an ambient gateway endpoint before OpenShell prep or state mutation", async () => { @@ -213,7 +215,7 @@ describe("runtime shared gateway route containment", () => { const peer = entry("late-peer", { provider: "compatible-endpoint", model: "custom/model", - endpointUrl: "https://peer.example.test/v1", + endpointUrl: "http://peer.example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }); @@ -224,13 +226,16 @@ describe("runtime shared gateway route containment", () => { .mockReturnValue({ sandboxes: [alpha, peer], defaultSandbox: "alpha" }); deps.listSandboxes = listSandboxes; + // HTTP (not HTTPS) so this test exercises rewriteUrlWithDnsPinning directly + // to create the async validation gap; DNS-backed HTTPS endpoints route + // through the HTTPS-pin runtime adapter instead. await expect( runInferenceSet( { provider: "compatible-endpoint", model: "custom/model", sandboxName: "alpha", - endpointUrl: "https://alpha.example.test/v1", + endpointUrl: "http://alpha.example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", inferenceApi: "openai-completions", }, @@ -279,8 +284,11 @@ describe("runtime shared gateway route containment", () => { }); it("catches a DNS change between the preliminary and finalized gateway route checks", async () => { - const firstEndpoint = "https://first.example.test/v1"; - const secondEndpoint = "https://second.example.test/v1"; + // HTTP (not HTTPS) so this test exercises rewriteUrlWithDnsPinning directly; + // DNS-backed HTTPS endpoints route through the HTTPS-pin runtime adapter + // instead (see inference-set-https-pin-runtime.test.ts). + const firstEndpoint = "http://first.example.test/v1"; + const secondEndpoint = "http://second.example.test/v1"; const customRoute = { provider: "compatible-endpoint", model: "custom/model", @@ -304,6 +312,7 @@ describe("runtime shared gateway route containment", () => { sandboxes: [alpha, peer], }); const rewriteUrlWithDnsPinning = vi.fn().mockResolvedValueOnce(secondEndpoint); + const ensureHttpsPinRuntimeAdapter = vi.fn(); await expect( finalizeInferenceSetRoute({ @@ -315,11 +324,113 @@ describe("runtime shared gateway route containment", () => { onboardEndpointUrl: null, getSandboxes: () => [alpha, peer], rewriteUrlWithDnsPinning, + ensureHttpsPinRuntimeAdapter, }), ).rejects.toThrow("custom-peer"); expect(rewriteUrlWithDnsPinning).toHaveBeenCalledOnce(); expect(rewriteUrlWithDnsPinning).toHaveBeenCalledWith(firstEndpoint); + expect(ensureHttpsPinRuntimeAdapter).not.toHaveBeenCalled(); + }); + + it("serializes concurrent HTTPS-pin adapter route provisions for different gateways so their process.env token writes cannot interleave (#6141)", async () => { + const firstEndpoint = "https://race-a.example.test/v1"; + const secondEndpoint = "https://race-b.example.test/v1"; + const routeFor = (endpointUrl: string) => ({ + provider: "compatible-endpoint", + model: "custom/model", + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + // Different gateways (distinguished by gatewayPort -- resolveSandboxGatewayName + // derives the effective gateway name from the port, not the gatewayName field, + // whenever a valid port is present): the per-gateway route-mutation lock does + // not serialize these two against each other, so only the credential-env lock + // added for this race can prevent their process.env writes from interleaving. + const alpha = entry("alpha", routeFor(firstEndpoint)); + const beta = entry("beta", { + ...routeFor(secondEndpoint), + gatewayName: "nemoclaw-9091", + gatewayPort: 9091, + }); + const preparedFor = (target: SandboxEntry, endpointUrl: string) => + prepareInferenceSetRoute({ + entry: target, + sandboxName: target.name, + provider: "compatible-endpoint", + model: "custom/model", + customRoute: { + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + session: null, + sandboxes: [alpha, beta], + }); + + let releaseA: () => void = () => {}; + const aGate = new Promise((resolve) => { + releaseA = resolve; + }); + const callOrder: string[] = []; + const adapterBehaviorByEndpoint: Record< + string, + () => Promise<{ baseUrl: string; credentialEnv: string; token: string }> + > = { + [firstEndpoint]: async () => { + callOrder.push("a-start"); + await aGate; + callOrder.push("a-end"); + return { + baseUrl: "http://host.openshell.internal:1/route/a", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: "token-a", + }; + }, + [secondEndpoint]: async () => { + callOrder.push("b-start"); + return { + baseUrl: "http://host.openshell.internal:1/route/b", + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: "token-b", + }; + }, + }; + const ensureHttpsPinRuntimeAdapter = vi.fn(async (options: { endpointUrl: string }) => + adapterBehaviorByEndpoint[options.endpointUrl](), + ); + const rewriteUrlWithDnsPinning = vi.fn(async (value: unknown) => value as string); + + const finalize = (target: SandboxEntry, endpointUrl: string) => + finalizeInferenceSetRoute({ + prepared: preparedFor(target, endpointUrl), + sandboxName: target.name, + provider: "compatible-endpoint", + model: "custom/model", + canReuseRecordedRoute: false, + onboardEndpointUrl: null, + getSandboxes: () => [alpha, beta], + rewriteUrlWithDnsPinning, + ensureHttpsPinRuntimeAdapter, + }); + + const callA = finalize(alpha, firstEndpoint); + await vi.waitFor(() => expect(callOrder).toContain("a-start")); + const callB = finalize(beta, secondEndpoint); + + // While A is gated inside the lock, B must not reach its own adapter + // call (and process.env write) yet -- proving the two invocations are + // serialized rather than interleaved. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(callOrder).toEqual(["a-start"]); + + releaseA(); + const [resultA, resultB] = await Promise.all([callA, callB]); + + expect(callOrder).toEqual(["a-start", "a-end", "b-start"]); + expect(resultA.registryMetadata.endpointUrl).toBe("http://host.openshell.internal:1/route/a"); + expect(resultB.registryMetadata.endpointUrl).toBe("http://host.openshell.internal:1/route/b"); }); it("blocks an incomplete legacy custom target even without a peer (#6315)", async () => { diff --git a/src/lib/actions/inference-set-https-pin-runtime.test.ts b/src/lib/actions/inference-set-https-pin-runtime.test.ts new file mode 100644 index 00000000000..52270b307dc --- /dev/null +++ b/src/lib/actions/inference-set-https-pin-runtime.test.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// PRA-1 regression (#6141): the HTTPS-pin runtime adapter's HTTP server +// rejects every request that doesn't present the adapter's own bearer token. +// `inference set --endpoint-url` must therefore register the adapter's own +// credentialEnv/token as the sandbox-facing route credential, never the real +// upstream secret — while still forwarding the real secret to the adapter so +// it can authenticate the outbound leg to the real provider. This file +// exercises that handoff at the `inference set` orchestration level; the +// adapter's own auth/forwarding behavior is covered separately in +// https-pin-runtime-adapter.test.ts and +// https-pin-runtime-adapter-forward.test.ts. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV } from "../inference/https-pin-runtime"; +import type { ConfigObject } from "../security/credential-filter"; +import { runInferenceSet } from "./inference-set"; +import { baseSession, createDeps } from "./inference-set.test-support"; +import type { EnsureHttpsPinRuntimeAdapterOptions } from "./inference-set-route-containment"; + +const ADAPTER_TOKEN = "test-adapter-token"; +const ADAPTER_BASE_URL = "http://host.openshell.internal:11438/route/test-route"; + +function mockAdapter() { + return vi.fn(async (_options: EnsureHttpsPinRuntimeAdapterOptions) => ({ + baseUrl: ADAPTER_BASE_URL, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + token: ADAPTER_TOKEN, + })); +} + +describe("runInferenceSet HTTPS-pin runtime adapter credential handoff (#6141)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + delete process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]; + }); + + it.each([ + ["compatible-endpoint", "COMPATIBLE_API_KEY", "openai-completions"], + ["compatible-anthropic-endpoint", "COMPATIBLE_ANTHROPIC_API_KEY", "anthropic-messages"], + ] as const)("registers the adapter's own token as the %s route credential, not the real upstream secret", async (provider, realCredentialEnv, inferenceApi) => { + vi.stubEnv(realCredentialEnv, "real-upstream-secret"); + const adapter = mockAdapter(); + const config: ConfigObject = { + agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, + models: { providers: { inference: { api: "openai-completions", models: [] } } }, + }; + const deps = createDeps({ + config, + entry: { name: "alpha", agent: "openclaw", provider: "nvidia-prod", model: "nvidia/model-a" }, + session: baseSession({ + provider: "nvidia-prod", + model: "nvidia/model-a", + endpointUrl: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + }), + ensureHttpsPinRuntimeAdapter: adapter, + }); + + await runInferenceSet( + { + provider, + model: "mock-model", + noVerify: true, + endpointUrl: "https://compatible.example/v1", + credentialEnv: realCredentialEnv, + inferenceApi, + }, + deps, + ); + + // Leg 1 (adapter -> real upstream): the adapter is handed the real + // provider secret's *value* so it can authenticate outbound. + expect(adapter).toHaveBeenCalledWith( + expect.objectContaining({ credentialValue: "real-upstream-secret" }), + ); + + // Leg 2 (sandbox -> adapter): the persisted route credential is the + // adapter's own env var, never the real secret's name. + expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ + "alpha", + expect.objectContaining({ + provider, + endpointUrl: ADAPTER_BASE_URL, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + }), + ]); + expect(deps.getSession()).toMatchObject({ + provider, + endpointUrl: ADAPTER_BASE_URL, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + }); + + // The adapter's own token is staged so the sandbox-facing credential + // env var it was just registered under actually resolves to it. + expect(process.env[HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV]).toBe(ADAPTER_TOKEN); + }); + + it("never persists the real upstream secret name as the route credentialEnv even when unset", async () => { + // Explicitly unset rather than relying on the ambient shell environment + // not happening to have this set -- the "unset" case under test must not + // depend on the developer's or CI's actual environment. + vi.stubEnv("COMPATIBLE_API_KEY", undefined); + const adapter = mockAdapter(); + const deps = createDeps({ + config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } } }, + entry: { name: "alpha", agent: "openclaw", provider: "nvidia-prod", model: "nvidia/model-a" }, + session: baseSession({ + provider: "nvidia-prod", + model: "nvidia/model-a", + endpointUrl: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + }), + ensureHttpsPinRuntimeAdapter: adapter, + }); + + await runInferenceSet( + { + provider: "compatible-endpoint", + model: "mock-model", + noVerify: true, + endpointUrl: "https://compatible.example/v1", + credentialEnv: "COMPATIBLE_API_KEY", + inferenceApi: "openai-completions", + }, + deps, + ); + + // No upstream secret was ever staged in this process, so the adapter + // sees an empty credentialValue -- but the persisted route credential + // must still be the adapter's own env var, not "COMPATIBLE_API_KEY". + expect(adapter).toHaveBeenCalledWith(expect.objectContaining({ credentialValue: "" })); + const persisted = deps.calls.updateSandbox.mock.calls.at(-1)?.[1] as { credentialEnv?: string }; + expect(persisted.credentialEnv).toBe(HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV); + expect(persisted.credentialEnv).not.toBe("COMPATIBLE_API_KEY"); + }); +}); diff --git a/src/lib/actions/inference-set-provider-alias.test.ts b/src/lib/actions/inference-set-provider-alias.test.ts index 74c8b11a451..8fbaa3f0a4f 100644 --- a/src/lib/actions/inference-set-provider-alias.test.ts +++ b/src/lib/actions/inference-set-provider-alias.test.ts @@ -24,6 +24,7 @@ import { normalizeInferenceSetProvider, runInferenceSet, } from "./inference-set"; +import type { EnsureHttpsPinRuntimeAdapterOptions } from "./inference-set-route-containment"; import { baseSession, createDeps } from "./inference-set.test-support"; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -279,6 +280,28 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { }); } + // A DNS-backed HTTPS endpoint (the shape every URL in this suite uses) never + // reaches rewriteConfigUrlsWithDnsPinning/ssrfGuard above — it is eligible for + // the HTTPS-pin runtime adapter, whose real implementation runs its own SSRF + // preflight (assertEndpointResolvesPublic) before registering a route. This + // stand-in mirrors that preflight against the same STUB_INTERNAL_HOSTS set. + function httpsPinAdapterGuard() { + return vi.fn(async (options: EnsureHttpsPinRuntimeAdapterOptions) => { + const host = new URL(options.endpointUrl).hostname; + return STUB_INTERNAL_HOSTS.has(host) + ? Promise.reject( + new Error( + `URL hostname "${host}" resolves to private/internal address "10.48.203.205". This could expose internal services to the sandbox.`, + ), + ) + : Promise.resolve({ + baseUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN", + token: "test-adapter-token", + }); + }); + } + it("keeps the SSRF guard when same-endpoint onboarding provenance is missing", async () => { // Legacy registry rows have no machine-checkable endpoint source. Exact // string equality is insufficient because inference set also persists the @@ -294,7 +317,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }, - rewriteConfigUrlsWithDnsPinning: ssrfGuard(), + ensureHttpsPinRuntimeAdapter: httpsPinAdapterGuard(), }); const attempt = runInferenceSet( @@ -390,7 +413,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { }); it("keeps the SSRF guard for an inference-set-authored endpoint", async () => { - const guard = ssrfGuard(); + const guard = httpsPinAdapterGuard(); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, @@ -406,7 +429,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", }, - rewriteConfigUrlsWithDnsPinning: guard, + ensureHttpsPinRuntimeAdapter: guard, }); const attempt = runInferenceSet( @@ -500,7 +523,7 @@ describe("runInferenceSet SSRF-block guidance — facet 2 (#6321)", () => { provider: "nvidia-prod", model: "nvidia/model-a", }, - rewriteConfigUrlsWithDnsPinning: ssrfGuard(), + ensureHttpsPinRuntimeAdapter: httpsPinAdapterGuard(), }); const attempt = runInferenceSet( diff --git a/src/lib/actions/inference-set-route-containment.ts b/src/lib/actions/inference-set-route-containment.ts index 132f41d6287..bddb9042bb1 100644 --- a/src/lib/actions/inference-set-route-containment.ts +++ b/src/lib/actions/inference-set-route-containment.ts @@ -5,12 +5,34 @@ import { checkGatewayRouteCompatibility, formatGatewayRouteConflict, } from "../inference/gateway-route-compatibility"; +import { + type HttpsPinCredentialProviderType, + isHttpsPinRuntimeEligible, +} from "../inference/https-pin-runtime"; import { resolveSandboxGatewayName } from "../onboard/gateway-binding"; +import { ConfigUrlValidationError } from "../sandbox/config"; import type { ConfigValue } from "../security/credential-filter"; +import { withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; import type { Session } from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; import { InferenceSetError } from "./inference-set-error"; +/** + * Fixed key, not gateway- or sandbox-scoped: every explicit-metadata + * `inference set` call that provisions an HTTPS Pin Runtime adapter route + * stages the adapter's bearer token in the one shared + * `HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV` slot on `process.env` + * (see the comment on `ensureHttpsPinAdapterRoute` in + * `finalizeInferenceSetRoute`). The per-sandbox/per-gateway locks already + * held around this call (`withSandboxMutationLock`, + * `withGatewayRouteMutationLock`) do not serialize two calls targeting + * *different* sandboxes/gateways against each other, so without this lock two + * concurrent adapter-route provisions could interleave their read of the real + * upstream credential, their adapter call, and their write of the adapter's + * token, and clobber each other's staged value. + */ +const HTTPS_PIN_ADAPTER_CREDENTIAL_ENV_LOCK_KEY = "https-pin-adapter-credential-env"; + /** * Custom-route compatibility is intentionally checked twice. The invalid state * is a requested endpoint whose DNS-pinned identity differs from the route that @@ -36,6 +58,26 @@ export interface ExplicitCustomRouteOptions { type RewriteConfigUrlsWithDnsPinning = (value: ConfigValue) => Promise; +/** + * Resolves a DNS-backed HTTPS custom endpoint to a pinned, locally-terminated + * route base URL instead of the raw operator-supplied URL. OpenShell never + * sees the real hostname; the returned URL always targets the trusted + * `host.openshell.internal` bridge, matching the shape already exempted by + * {@link ALLOWED_PRIVATE_CUSTOM_ENDPOINT_HOSTS}. + */ +export interface EnsureHttpsPinRuntimeAdapterOptions { + gatewayName: string; + provider: string; + endpointUrl: string; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; +} +export type EnsureHttpsPinRuntimeAdapterFn = ( + options: EnsureHttpsPinRuntimeAdapterOptions, +) => Promise<{ baseUrl: string; credentialEnv: string; token: string }>; + +type EnsureHttpsPinAdapterRoute = (endpointUrl: string) => Promise; + export interface PreparedInferenceSetRoute { gatewayName: string; preliminaryExplicitMetadata: RegistryInferenceMetadata | null; @@ -105,6 +147,7 @@ function normalizeCustomEndpointUrlWithoutDns(value: string | null | undefined): export async function normalizeCustomEndpointUrl( value: string | null | undefined, rewriteUrlWithDnsPinning: RewriteConfigUrlsWithDnsPinning, + ensureHttpsPinAdapterRoute?: EnsureHttpsPinAdapterRoute, ): Promise { const normalized = normalizeCustomEndpointUrlWithoutDns(value); const shaped = normalizeEndpointUrlShape(normalized); @@ -124,13 +167,39 @@ export async function normalizeCustomEndpointUrl( return normalized; } + // A DNS-backed HTTPS endpoint cannot be pinned by IP substitution alone: the + // TLS certificate requires the real hostname as SNI, so OpenShell's own + // re-resolution at request time would race the SSRF preflight (TOCTOU) if + // it saw that hostname directly. Route it through the local HTTPS-pin + // runtime adapter instead, which re-validates the address immediately + // before connecting and hides the real hostname from the OpenShell runtime + // boundary entirely. + if (ensureHttpsPinAdapterRoute && isHttpsPinRuntimeEligible(normalized)) { + try { + const pinned = await ensureHttpsPinAdapterRoute(normalized); + if (typeof pinned !== "string") + throw new Error("HTTPS pin adapter returned a non-string value"); + return normalizeEndpointUrlShape(pinned).normalized; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}`, 2); + } + } + try { const validated = await rewriteUrlWithDnsPinning(normalized); if (typeof validated !== "string") throw new Error("URL validator returned a non-string value"); return normalizeEndpointUrlShape(validated).normalized; } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}`, 2); + // The generic DNS-pinning validator's message stays scoped to arbitrary + // persisted config values; only this inference-set call site knows the + // rejected field is an inference endpoint, so it adds the adapter hint. + const hint = + error instanceof ConfigUrlValidationError && error.reason === "dns_backed_https_unsupported" + ? " This endpoint should have been routed through the HTTPS Pin Runtime adapter; retry, and report a bug if this persists." + : ""; + throw new InferenceSetError(`${ENDPOINT_URL_NOT_ALLOWED_PREFIX} ${message}${hint}`, 2); } } @@ -331,6 +400,7 @@ export async function finalizeInferenceSetRoute(options: { onboardEndpointUrl: string | null; getSandboxes: () => SandboxEntry[]; rewriteUrlWithDnsPinning: RewriteConfigUrlsWithDnsPinning; + ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; }): Promise<{ registryMetadata: RegistryInferenceMetadata; explicitPreferredInferenceApi: string | null; @@ -342,6 +412,46 @@ export async function finalizeInferenceSetRoute(options: { explicitPreferredInferenceApi: null, }; } + // Bound once per finalize call: the credential env var name is fixed per + // provider (normalizeExplicitCredentialEnv already enforced this), and the + // real credential value is read directly from the host process environment + // at invocation time, never persisted, and never returned to the caller. + const httpsPinCredentialEnv = CUSTOM_COMPATIBLE_CREDENTIAL_ENV[options.provider]; + // Set only when the adapter route is actually used, so the sandbox-facing + // credential env name gets swapped to the adapter's own bearer token below. + // Left null for every other endpoint shape, which keeps the operator's real + // credential env name as the sandbox-facing identity, unchanged. + let adapterCredentialEnv: string | null = null; + const ensureHttpsPinAdapterRoute: EnsureHttpsPinAdapterRoute = async (endpointUrl) => + // Serializes the read-call-write sequence below against every other + // concurrent adapter-route provision process-wide -- see the lock key's + // doc comment for why the per-sandbox/per-gateway locks above cannot + // substitute for this. + withMcpLifecycleLock(HTTPS_PIN_ADAPTER_CREDENTIAL_ENV_LOCK_KEY, async () => { + // The credential env var's value, if any, is read directly from the host + // process environment at invocation time and handed straight to the + // adapter — never persisted, never returned to this caller. A missing + // value is validated by ensureHttpsPinRuntimeAdapter itself, after the + // endpoint's own reachability/SSRF check, so an unsafe URL is always + // rejected on that ground first. + const credentialValue = process.env[httpsPinCredentialEnv] ?? ""; + const adapter = await options.ensureHttpsPinRuntimeAdapter({ + gatewayName: prepared.gatewayName, + provider: options.provider, + endpointUrl, + providerType: options.provider === "compatible-anthropic-endpoint" ? "anthropic" : "openai", + credentialValue, + }); + // The real upstream credential above authenticates the adapter's own + // outbound leg to the real provider. Requests reaching the adapter's + // sandbox-facing route must instead carry the adapter's own bearer + // token — the adapter rejects anything else with 401 — so the env var + // that ends up registered as this route's credential must hold that + // token, not the operator's real secret. + process.env[adapter.credentialEnv] = adapter.token; + adapterCredentialEnv = adapter.credentialEnv; + return adapter.baseUrl; + }); let endpointUrl: string; let endpointSource: RegistryInferenceMetadata["endpointSource"]; try { @@ -363,6 +473,7 @@ export async function finalizeInferenceSetRoute(options: { endpointUrl = await normalizeCustomEndpointUrl( suppliedEndpoint, options.rewriteUrlWithDnsPinning, + ensureHttpsPinAdapterRoute, ); endpointSource = "inference-set"; } @@ -389,6 +500,7 @@ export async function finalizeInferenceSetRoute(options: { ...prepared.preliminaryExplicitMetadata, endpointUrl, endpointSource, + credentialEnv: adapterCredentialEnv ?? prepared.preliminaryExplicitMetadata.credentialEnv, }; assertGatewayRouteCompatibility({ gatewayName: prepared.gatewayName, diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 00de8f2a36c..99566834fa1 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -8,6 +8,7 @@ import type { ConfigObject, ConfigValue } from "../security/credential-filter"; import type { Session } from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; import type { InferenceSetDeps } from "./inference-set"; +import type { EnsureHttpsPinRuntimeAdapterFn } from "./inference-set-route-containment"; export const OPENCLAW_TARGET: AgentConfigTarget = { agentName: "openclaw", @@ -85,6 +86,7 @@ export function createDeps(options: { shieldsMutable?: boolean; prepareRunOpenshell?: () => void; rewriteConfigUrlsWithDnsPinning?: (value: ConfigValue) => Promise; + ensureHttpsPinRuntimeAdapter?: EnsureHttpsPinRuntimeAdapterFn; restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"]; seedHermesDashboardConfigResult?: "converged" | "absent" | "failed"; withGatewayRouteMutationLock?: InferenceSetDeps["withGatewayRouteMutationLock"]; @@ -104,6 +106,7 @@ export function createDeps(options: { resolveContextWindowForModel: ReturnType; prepareRunOpenshell: ReturnType; rewriteConfigUrlsWithDnsPinning: ReturnType; + ensureHttpsPinRuntimeAdapter: ReturnType; restartSandboxGateway: ReturnType; withGatewayRouteMutationLock: ReturnType; }; @@ -145,6 +148,14 @@ export function createDeps(options: { rewriteConfigUrlsWithDnsPinning: vi.fn( options.rewriteConfigUrlsWithDnsPinning ?? (async (value: ConfigValue) => value), ), + ensureHttpsPinRuntimeAdapter: vi.fn( + options.ensureHttpsPinRuntimeAdapter ?? + (async () => ({ + baseUrl: "http://host.openshell.internal:11438/route/test-route", + credentialEnv: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN", + token: "test-adapter-token", + })), + ), restartSandboxGateway: vi.fn( options.restartSandboxGateway ?? ((): ReturnType => ({ @@ -184,6 +195,8 @@ export function createDeps(options: { resolveContextWindowForModel: calls.resolveContextWindowForModel, isSandboxConfigMutable: () => options.shieldsMutable ?? true, rewriteConfigUrlsWithDnsPinning: calls.rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter: + calls.ensureHttpsPinRuntimeAdapter as unknown as EnsureHttpsPinRuntimeAdapterFn, withGatewayRouteMutationLock: calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"], restartSandboxGateway: calls.restartSandboxGateway, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index e5b76a1c408..e8b1fe3eb94 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -7,6 +7,7 @@ import { CLI_NAME } from "../cli/branding"; import { shellQuote } from "../core/shell-quote"; import { HERMES_PROXY_API_KEY_PLACEHOLDER } from "../hermes-proxy-api-key"; import { isBedrockRuntimeEndpoint } from "../inference/bedrock-runtime"; +import { ensureHttpsPinRuntimeAdapter } from "../inference/https-pin-runtime-adapter"; import { getProviderSelectionConfig, getSandboxInferenceConfig, @@ -63,6 +64,7 @@ import { readOpenClawPrimaryReplyBudget, } from "./inference-set-reply-budget"; import { + type EnsureHttpsPinRuntimeAdapterFn, finalizeInferenceSetRoute, prepareInferenceSetRoute, type RegistryInferenceMetadata, @@ -136,6 +138,7 @@ export interface InferenceSetDeps extends InferenceGatewayRestartDeps { resolveContextWindowForModel: (provider: string, model: string) => number | null; isSandboxConfigMutable: (sandboxName: string) => boolean; rewriteConfigUrlsWithDnsPinning: (value: ConfigValue) => Promise; + ensureHttpsPinRuntimeAdapter: EnsureHttpsPinRuntimeAdapterFn; withGatewayRouteMutationLock: typeof withGatewayRouteMutationLock; } @@ -237,6 +240,7 @@ function defaultDeps(): InferenceSetDeps { ensureLocalProviderReachable, resolveContextWindowForModel, rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter, withGatewayRouteMutationLock, restartSandboxGateway: defaultInferenceGatewayRestart, isSandboxConfigMutable: (sandboxName) => { @@ -742,6 +746,7 @@ async function runInferenceSetWithoutHostLock( : null, getSandboxes: () => deps.listSandboxes().sandboxes, rewriteUrlWithDnsPinning: deps.rewriteConfigUrlsWithDnsPinning, + ensureHttpsPinRuntimeAdapter: deps.ensureHttpsPinRuntimeAdapter, }); // Local providers (ollama-local, vllm-local) route through the sandbox-facing diff --git a/src/lib/core/ports.test.ts b/src/lib/core/ports.test.ts index 024f4929f5a..e1417040540 100644 --- a/src/lib/core/ports.test.ts +++ b/src/lib/core/ports.test.ts @@ -3,7 +3,12 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; // Import source directly so tests cannot pass against a stale build. -import { parseGatewayPort, parsePort, validateOpenRouterRuntimeAdapterPort } from "./ports"; +import { + parseGatewayPort, + parsePort, + validateHttpsPinRuntimeAdapterPort, + validateOpenRouterRuntimeAdapterPort, +} from "./ports"; const GATEWAY_VALIDATION_OPTIONS = { dashboardPort: 18789, @@ -15,6 +20,7 @@ const GATEWAY_VALIDATION_OPTIONS = { ollamaProxyPort: 11435, bedrockRuntimeAdapterPort: 11436, openrouterRuntimeAdapterPort: 11437, + httpsPinRuntimeAdapterPort: 11438, }; describe("parsePort", () => { @@ -117,6 +123,7 @@ describe("parseGatewayPort", () => { ["11435", "Ollama auth proxy"], ["11436", "Bedrock Runtime adapter"], ["11437", "OpenRouter Runtime adapter"], + ["11438", "HTTPS Pin Runtime adapter"], ])("rejects overlap with default port %s", (port, label) => { process.env[ENV_KEY] = port; expect(() => parseGatewayPort(ENV_KEY, 8080, GATEWAY_VALIDATION_OPTIONS)).toThrow(label); @@ -141,6 +148,16 @@ describe("parseGatewayPort", () => { }), ).toThrow("NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT"); }); + + it("rejects overlap with a configured HTTPS Pin Runtime adapter port", () => { + process.env[ENV_KEY] = "19004"; + expect(() => + parseGatewayPort(ENV_KEY, 8080, { + ...GATEWAY_VALIDATION_OPTIONS, + httpsPinRuntimeAdapterPort: 19004, + }), + ).toThrow("NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT"); + }); }); describe("validateOpenRouterRuntimeAdapterPort", () => { @@ -158,6 +175,7 @@ describe("validateOpenRouterRuntimeAdapterPort", () => { [11434, "Ollama inference"], [11435, "Ollama auth proxy"], [11436, "Bedrock Runtime adapter"], + [11438, "HTTPS Pin Runtime adapter"], [18790, "18789-18799"], ])("rejects OpenRouter adapter overlap with %s", (port, expectedMessage) => { expect(() => @@ -174,3 +192,36 @@ describe("validateOpenRouterRuntimeAdapterPort", () => { ).toThrow("NEMOCLAW_VLLM_PORT"); }); }); + +describe("validateHttpsPinRuntimeAdapterPort", () => { + const ENV_KEY = "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT"; + + it("allows the default HTTPS Pin Runtime adapter port", () => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, 11438, GATEWAY_VALIDATION_OPTIONS), + ).not.toThrow(); + }); + + it.each([ + [8080, "NEMOCLAW_GATEWAY_PORT"], + [8000, "vLLM / NIM inference"], + [11434, "Ollama inference"], + [11435, "Ollama auth proxy"], + [11436, "Bedrock Runtime adapter"], + [11437, "OpenRouter Runtime adapter"], + [18790, "18789-18799"], + ])("rejects HTTPS Pin adapter overlap with %s", (port, expectedMessage) => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, port, GATEWAY_VALIDATION_OPTIONS), + ).toThrow(expectedMessage); + }); + + it("rejects HTTPS Pin adapter overlap with configured service ports", () => { + expect(() => + validateHttpsPinRuntimeAdapterPort(ENV_KEY, 19001, { + ...GATEWAY_VALIDATION_OPTIONS, + vllmPort: 19001, + }), + ).toThrow("NEMOCLAW_VLLM_PORT"); + }); +}); diff --git a/src/lib/core/ports.ts b/src/lib/core/ports.ts index 7cd9ff0da50..31ddb4985f9 100644 --- a/src/lib/core/ports.ts +++ b/src/lib/core/ports.ts @@ -33,6 +33,7 @@ export interface GatewayPortValidationOptions { ollamaProxyPort: number; bedrockRuntimeAdapterPort: number; openrouterRuntimeAdapterPort: number; + httpsPinRuntimeAdapterPort: number; } export interface RuntimeAdapterPortValidationOptions extends GatewayPortValidationOptions { @@ -71,6 +72,11 @@ export const OPENROUTER_RUNTIME_ADAPTER_PORT = parsePort( "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", 11437, ); +/** HTTPS DNS-pinning reverse-proxy adapter port (default 11438, override via NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT). */ +export const HTTPS_PIN_RUNTIME_ADAPTER_PORT = parsePort( + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + 11438, +); export function validateGatewayPort( envVar: string, @@ -89,6 +95,7 @@ export function validateGatewayPort( { label: "Ollama auth proxy", port: 11435 }, { label: "Bedrock Runtime adapter", port: 11436 }, { label: "OpenRouter Runtime adapter", port: 11437 }, + { label: "HTTPS Pin Runtime adapter", port: 11438 }, ]; const reservedDefault = reservedDefaults.find((entry) => entry.port === port); if (reservedDefault) { @@ -110,6 +117,10 @@ export function validateGatewayPort( envVar: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", port: options.openrouterRuntimeAdapterPort, }, + { + envVar: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + port: options.httpsPinRuntimeAdapterPort, + }, ]; const conflict = conflicts.find((entry) => entry.port === port); if (conflict) { @@ -145,6 +156,55 @@ export function validateOpenRouterRuntimeAdapterPort( { label: "Ollama inference", port: 11434 }, { label: "Ollama auth proxy", port: 11435 }, { label: "Bedrock Runtime adapter", port: 11436 }, + { label: "HTTPS Pin Runtime adapter", port: 11438 }, + ]; + const reservedDefault = reservedDefaults.find((entry) => entry.port === port); + if (reservedDefault) { + throw new Error( + `Invalid port: ${envVar}="${port}" — must not overlap the ${reservedDefault.label} default port (${reservedDefault.port})`, + ); + } + + const conflicts = [ + { envVar: "NEMOCLAW_GATEWAY_PORT", port: options.gatewayPort }, + { envVar: "NEMOCLAW_DASHBOARD_PORT", port: options.dashboardPort }, + { envVar: "NEMOCLAW_VLLM_PORT", port: options.vllmPort }, + { envVar: "NEMOCLAW_OLLAMA_PORT", port: options.ollamaPort }, + { envVar: "NEMOCLAW_OLLAMA_PROXY_PORT", port: options.ollamaProxyPort }, + { + envVar: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", + port: options.bedrockRuntimeAdapterPort, + }, + { + envVar: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + port: options.httpsPinRuntimeAdapterPort, + }, + ]; + const conflict = conflicts.find((entry) => entry.port === port); + if (conflict) { + throw new Error( + `Invalid port: ${envVar}="${port}" — conflicts with ${conflict.envVar} (${conflict.port})`, + ); + } +} + +export function validateHttpsPinRuntimeAdapterPort( + envVar: string, + port: number, + options: RuntimeAdapterPortValidationOptions, +): void { + if (port >= options.dashboardRangeStart && port <= options.dashboardRangeEnd) { + throw new Error( + `Invalid port: ${envVar}="${port}" — must not overlap the ${options.dashboardRangeStart}-${options.dashboardRangeEnd} dashboard port range`, + ); + } + + const reservedDefaults = [ + { label: "vLLM / NIM inference", port: 8000 }, + { label: "Ollama inference", port: 11434 }, + { label: "Ollama auth proxy", port: 11435 }, + { label: "Bedrock Runtime adapter", port: 11436 }, + { label: "OpenRouter Runtime adapter", port: 11437 }, ]; const reservedDefault = reservedDefaults.find((entry) => entry.port === port); if (reservedDefault) { @@ -163,6 +223,10 @@ export function validateOpenRouterRuntimeAdapterPort( envVar: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", port: options.bedrockRuntimeAdapterPort, }, + { + envVar: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", + port: options.openrouterRuntimeAdapterPort, + }, ]; const conflict = conflicts.find((entry) => entry.port === port); if (conflict) { @@ -184,4 +248,5 @@ export const GATEWAY_PORT = parseGatewayPort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GA ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }); diff --git a/src/lib/inference/https-pin-runtime-adapter-forward.test.ts b/src/lib/inference/https-pin-runtime-adapter-forward.test.ts new file mode 100644 index 00000000000..42dc00323d2 --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter-forward.test.ts @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import type { AddressInfo } from "node:net"; + +import { afterAll, afterEach, describe, expect, it } from "vitest"; + +import { + type CaMaterial, + cleanupCaSetup, + resolveCaSetup, + startTlsServer, +} from "../../../test/helpers/corporate-ca-support"; +import { + forwardHttpsPinnedRequest, + HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES, + type HttpsPinTarget, +} from "./https-pin-runtime-adapter-forward"; + +const servers: http.Server[] = []; +const tlsServers: Array<{ close: () => Promise }> = []; + +afterEach(async () => { + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); + servers.length = 0; + await Promise.all(tlsServers.map((server) => server.close())); + tlsServers.length = 0; +}); + +function listen(server: http.Server): Promise<{ baseUrl: string; port: number }> { + servers.push(server); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve({ baseUrl: `http://127.0.0.1:${address.port}`, port: address.port }); + }); + }); +} + +const TEST_CREDENTIAL = { name: "x-api-key", value: "secret-upstream-credential" }; + +/** A minimal server that forwards every request through `forwardHttpsPinnedRequest` against `target`. */ +function createForwardTestServer( + target: HttpsPinTarget, + options: { upstreamTimeoutMs?: number; bodyTimeoutMs?: number } = {}, +): http.Server { + return http.createServer(async (req, res) => { + const url = new URL(req.url || "/", "http://127.0.0.1"); + await forwardHttpsPinnedRequest({ + req, + res, + forwardPath: url.pathname + url.search, + target, + upstreamTimeoutMs: options.upstreamTimeoutMs, + bodyTimeoutMs: options.bodyTimeoutMs, + }); + }); +} + +function readRequestBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +describe("forwardHttpsPinnedRequest header handling (#6141)", () => { + it("connects to the pinned address while sending the real hostname as Host, and injects the upstream credential", async () => { + const upstreamRequests: Array<{ + headers: http.IncomingHttpHeaders; + body: string; + url: string | undefined; + }> = []; + const upstream = http.createServer(async (req, res) => { + upstreamRequests.push({ + headers: req.headers, + body: await readRequestBody(req), + url: req.url, + }); + res.writeHead(200, { "Content-Type": "application/json", "X-Upstream-Marker": "yes" }); + res.end(JSON.stringify({ ok: true })); + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base/chat?trace=1`, { + method: "POST", + headers: { + Authorization: "Bearer client-supplied-should-be-dropped", + "Content-Type": "application/json", + "X-Trace-Id": "abc123", + }, + body: JSON.stringify({ hello: "world" }), + }); + + expect(response.status).toBe(200); + expect(response.headers.get("x-upstream-marker")).toBe("yes"); + expect(upstreamRequests).toHaveLength(1); + const [seen] = upstreamRequests; + // Host reflects the real target hostname (with its non-default port), + // not the pinned connect address. + expect(seen.headers.host).toBe(`forward-test.example:${upstreamPort}`); + // The adapter's own credential is what reaches upstream... + expect(seen.headers["x-api-key"]).toBe(TEST_CREDENTIAL.value); + // ...and the client-supplied Authorization header never does. + expect(seen.headers.authorization).toBeUndefined(); + expect(seen.headers["x-trace-id"]).toBe("abc123"); + expect(seen.url).toBe("/base/chat?trace=1"); + expect(seen.body).toBe(JSON.stringify({ hello: "world" })); + }); + + it("rejects a request body over the size limit before contacting upstream", async () => { + const upstreamHandler = () => { + throw new Error("upstream must not be contacted for an oversized body"); + }; + const upstream = http.createServer(upstreamHandler); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { + method: "POST", + body: "x".repeat(HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES + 1), + }); + + expect(response.status).toBe(413); + await expect(response.json()).resolves.toMatchObject({ error: { code: "request_too_large" } }); + }); + + it("delivers the 408 timeout body to the client instead of hanging up the shared socket (#6141)", async () => { + const upstreamHandler = () => { + throw new Error("upstream must not be contacted for a stalled request body"); + }; + const upstream = http.createServer(upstreamHandler); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { bodyTimeoutMs: 50 }); + const { port: adapterPort } = await listen(adapter); + + // A raw request that declares a body but never finishes sending it, so + // the adapter's body-read timeout fires instead of the client ever + // completing the write. Destroying `req` before `res` flushes (the bug + // this test guards) would tear down the shared socket and the client + // would see the connection drop instead of a 408 body. + const response = await new Promise<{ status: number | undefined; body: string }>( + (resolve, reject) => { + const req = http.request( + { + host: "127.0.0.1", + port: adapterPort, + path: "/base", + method: "POST", + headers: { "content-length": "100" }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + res.on("end", () => { + resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString("utf8") }); + }); + res.on("error", reject); + }, + ); + req.on("error", reject); + // Fewer bytes than the declared content-length, and `.end()` is + // deliberately never called. + req.write("partial-body"); + }, + ); + + expect(response.status).toBe(408); + expect(JSON.parse(response.body)).toMatchObject({ error: { code: "request_timeout" } }); + }); + + it("cancels the pinned upstream request when the client disconnects before the response finishes (#6141)", async () => { + let upstreamRequestSocket: import("node:net").Socket | undefined; + let resolveUpstreamClosed: () => void; + const upstreamClosed = new Promise((resolve) => { + resolveUpstreamClosed = resolve; + }); + const upstream = http.createServer((req, res) => { + upstreamRequestSocket = req.socket; + req.socket.once("close", () => resolveUpstreamClosed()); + res.writeHead(200, { "Content-Type": "application/json" }); + res.write('{"partial":true'); + // Never call res.end(): the upstream response is left open so the + // only way this promise resolves is via the adapter destroying the + // pinned outbound connection after the client disconnects. + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 30_000 }); + const { port: adapterPort } = await listen(adapter); + + await new Promise((resolve, reject) => { + const clientReq = http.request( + { host: "127.0.0.1", port: adapterPort, path: "/base", method: "POST" }, + (res) => { + res.once("data", () => { + // Simulate the original client abandoning the request once it + // has started receiving a response. + clientReq.destroy(); + resolve(); + }); + res.once("error", () => resolve()); + }, + ); + clientReq.on("error", () => { + /* destroying our own request triggers this; expected. */ + }); + clientReq.end("{}"); + setTimeout(() => reject(new Error("timed out waiting for client response data")), 2000); + }); + + // The pinned outbound connection must be torn down promptly, well under + // the 30s default/configured upstream timeout, rather than lingering + // until the abandoned upstream response finishes on its own. + await expect( + Promise.race([ + upstreamClosed, + new Promise((_, reject) => + setTimeout(() => reject(new Error("upstream connection was not canceled in time")), 2000), + ), + ]), + ).resolves.toBeUndefined(); + expect(upstreamRequestSocket?.destroyed).toBe(true); + }); + + it("times out a stalled upstream response without hanging (#6141)", async () => { + const upstream = http.createServer(async (req) => { + await readRequestBody(req); + // Never responds. + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target, { upstreamTimeoutMs: 50 }); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + expect(response.status).toBe(504); + await expect(response.json()).resolves.toMatchObject({ error: { code: "upstream_timeout" } }); + }); +}); + +describe("forwardHttpsPinnedRequest redirect fail-closed (#6141)", () => { + it.each([ + 301, 302, 303, 307, 308, + ])("blocks a %i upstream redirect instead of following or relaying it", async (status) => { + const upstream = http.createServer(async (req, res) => { + await readRequestBody(req); + res.writeHead(status, { Location: "http://169.254.169.254/latest/meta-data/" }); + res.end(); + }); + const { port: upstreamPort } = await listen(upstream); + + const target: HttpsPinTarget = { + targetUrl: new URL(`http://forward-test.example:${upstreamPort}/base`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/base`, { method: "POST", body: "{}" }); + expect(response.status).toBe(502); + const body = (await response.json()) as { error: { code: string } }; + expect(body.error.code).toBe("redirect_blocked"); + // The attacker-influenced Location header must never reach the client. + expect(response.headers.get("location")).toBeNull(); + expect(JSON.stringify(body)).not.toContain("169.254.169.254"); + }); +}); + +const sniPinSetup = resolveCaSetup("https-pin-runtime-adapter-forward SNI pinning"); + +afterAll(() => cleanupCaSetup(sniPinSetup)); + +describe.skipIf(!sniPinSetup.ok)("forwardHttpsPinnedRequest TLS SNI pinning (#6141)", () => { + const ca = sniPinSetup as CaMaterial; + + it("validates the certificate against the real target hostname while connecting to the pinned address", async () => { + const tlsServer = await startTlsServer(ca.serverKey, ca.serverCert); + tlsServers.push(tlsServer); + + const trustedAgent = new https.Agent({ ca: fs.readFileSync(ca.corporateCaCert) }); + const originalAgent = https.globalAgent; + https.globalAgent = trustedAgent; + try { + const target: HttpsPinTarget = { + // The leaf cert's SAN covers "localhost"; connecting via the pinned + // loopback address (not a fresh DNS lookup of the hostname) must still + // validate against this real hostname through TLS SNI. + targetUrl: new URL(`https://localhost:${tlsServer.port}/`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/`, { method: "POST", body: "{}" }); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ ok: true }); + } finally { + https.globalAgent = originalAgent; + trustedAgent.destroy(); + } + }); + + it("fails closed when the pinned target hostname is not covered by the upstream certificate", async () => { + const tlsServer = await startTlsServer(ca.serverKey, ca.serverCert); + tlsServers.push(tlsServer); + + const trustedAgent = new https.Agent({ ca: fs.readFileSync(ca.corporateCaCert) }); + const originalAgent = https.globalAgent; + https.globalAgent = trustedAgent; + try { + const target: HttpsPinTarget = { + // Same server/cert as the positive case, but a hostname the leaf + // certificate does not cover: certificate hostname verification must + // still reject this, proving the pin never disables verification. + targetUrl: new URL(`https://not-the-real-host.invalid:${tlsServer.port}/`), + pinnedAddress: "127.0.0.1", + credential: TEST_CREDENTIAL, + }; + const adapter = createForwardTestServer(target); + const { baseUrl } = await listen(adapter); + + const response = await fetch(`${baseUrl}/`, { method: "POST", body: "{}" }); + expect(response.status).toBe(502); + } finally { + https.globalAgent = originalAgent; + trustedAgent.destroy(); + } + }); +}); diff --git a/src/lib/inference/https-pin-runtime-adapter-forward.ts b/src/lib/inference/https-pin-runtime-adapter-forward.ts new file mode 100644 index 00000000000..4f5fa1dddfd --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter-forward.ts @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import http from "node:http"; +import https from "node:https"; + +import { compactText } from "../core/url-utils"; +import type { HttpsPinCredentialHeader } from "./https-pin-runtime"; + +export const HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES = 2 * 1024 * 1024; +const HTTPS_PIN_RUNTIME_ADAPTER_BODY_TIMEOUT_MS = 30_000; +const HTTPS_PIN_RUNTIME_ADAPTER_UPSTREAM_TIMEOUT_MS = 30_000; + +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "authorization", + "x-api-key", +]); + +export class ForwardHttpError extends Error { + constructor( + readonly status: number, + message: string, + readonly code: string, + ) { + super(message); + } +} + +/** + * The pinned outbound peer for one forwarded request: the validated public + * address to connect to, and the real hostname to present as TLS SNI / send + * as the Host header, so certificate validation still targets the real host + * while the TCP connection goes to the address the SSRF preflight validated. + */ +export interface HttpsPinTarget { + targetUrl: URL; + pinnedAddress: string; + credential: HttpsPinCredentialHeader; +} + +export function buildForwardRequestHeaders( + req: http.IncomingMessage, + credential: HttpsPinCredentialHeader, +): http.OutgoingHttpHeaders { + const headers: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (value === undefined || HOP_BY_HOP_HEADERS.has(name.toLowerCase())) continue; + headers[name] = value; + } + headers[credential.name] = credential.value; + return headers; +} + +function buildForwardResponseHeaders(source: http.IncomingHttpHeaders): http.OutgoingHttpHeaders { + const headers: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(source)) { + if (value === undefined || HOP_BY_HOP_HEADERS.has(name.toLowerCase())) continue; + headers[name] = value; + } + return headers; +} + +function readBoundedRequestBody( + req: http.IncomingMessage, + bodyTimeoutMs = HTTPS_PIN_RUNTIME_ADAPTER_BODY_TIMEOUT_MS, +): Promise { + return new Promise((resolve, reject) => { + const contentLength = Number(req.headers["content-length"] || 0); + if ( + Number.isFinite(contentLength) && + contentLength > HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES + ) { + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + // Destroying `req` here (rather than leaving that to the caller) would + // tear down the same underlying socket `res` needs to flush the 408 + // response on -- the client would see a dead connection instead of the + // documented JSON body. The caller destroys `req` itself, after the + // error response finishes writing. + req.removeAllListeners("data"); + reject(new ForwardHttpError(408, "Request body timed out.", "request_timeout")); + }, bodyTimeoutMs); + + req.on("data", (chunk: Buffer) => { + if (settled) return; + size += chunk.length; + if (size > HTTPS_PIN_RUNTIME_ADAPTER_MAX_BODY_BYTES) { + settled = true; + clearTimeout(timer); + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + chunks.push(Buffer.from(chunk)); + }); + req.on("end", () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(Buffer.concat(chunks)); + }); + req.on("error", (err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(err); + }); + }); +} + +export function sendForwardError( + res: http.ServerResponse, + err: unknown, + req?: http.IncomingMessage, +): number { + const status = err instanceof ForwardHttpError ? err.status : 502; + const code = err instanceof ForwardHttpError ? err.code : "https_pin_runtime_error"; + const message = err instanceof ForwardHttpError ? err.message : "Upstream request failed."; + // Only the body-read timeout leaves the client still writing indefinitely + // -- every other rejection (e.g. an oversized body) responds without + // needing the rest of the client's upload, and destroying the shared + // socket there would cut off a still-in-flight client write (EPIPE) + // instead of letting it drain normally. + const shouldDestroyRequest = Boolean(req) && code === "request_timeout"; + if (!res.headersSent) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: { message: compactText(message), type: code, code }, + }), + // Only destroy the request socket once the error response has finished + // writing, not before -- `req` and `res` share the same underlying + // socket, so destroying `req` any earlier would take the response down + // with it. + () => { + if (shouldDestroyRequest && !req?.destroyed) req?.destroy(); + }, + ); + } else { + res.destroy(err instanceof Error ? err : undefined); + } + return status; +} + +/** + * Forward one request to a pinned HTTPS peer: connects to `pinnedAddress` + * (the address the SSRF preflight already validated) while sending TLS SNI + * and the Host header for the real target hostname, so certificate + * validation still targets the real host — the Node equivalent of curl + * `--resolve` with strict hostname verification preserved. HTTP targets + * connect directly (no pinning needed; the address itself was already + * validated and substituted upstream of this adapter). + * + * Fails closed on any 3xx upstream response: a redirect is never followed or + * relayed, since a `Location` header is attacker-influenced content that + * could point at an internal address, silently defeating the pin. + */ +export async function forwardHttpsPinnedRequest(options: { + req: http.IncomingMessage; + res: http.ServerResponse; + forwardPath: string; + target: HttpsPinTarget; + upstreamTimeoutMs?: number; + bodyTimeoutMs?: number; +}): Promise { + const { req, res, forwardPath, target } = options; + let body: Buffer; + try { + body = await readBoundedRequestBody(req, options.bodyTimeoutMs); + } catch (err) { + return sendForwardError(res, err, req); + } + + const isHttps = target.targetUrl.protocol === "https:"; + const transport = isHttps ? https : http; + const port = target.targetUrl.port ? Number(target.targetUrl.port) : isHttps ? 443 : 80; + + return new Promise((resolve) => { + let settled = false; + const resolveOnce = (status: number) => { + if (settled) return; + settled = true; + res.off("close", onClientClose); + resolve(status); + }; + const failRequest = (err: unknown) => { + // Once the client-facing response is already finalized (normally, or + // via onClientClose below), res is no longer safe to write to. + if (settled) return; + resolveOnce(sendForwardError(res, err)); + }; + + const headers = buildForwardRequestHeaders(req, target.credential); + // `.host` (not `.hostname`) so a non-default port on the real endpoint is + // preserved in the Host header; TLS SNI below correctly stays bare + // hostname-only since SNI has no port component. + headers.host = target.targetUrl.host; + headers["content-length"] = String(body.length); + + const upstreamReq = transport.request( + { + hostname: target.pinnedAddress, + port, + path: forwardPath, + method: req.method, + headers, + ...(isHttps ? { servername: target.targetUrl.hostname } : {}), + }, + (upstreamRes) => { + const status = upstreamRes.statusCode || 502; + if (status >= 300 && status < 400) { + upstreamRes.resume(); + failRequest( + new ForwardHttpError( + 502, + "Upstream redirect blocked: the pinned adapter does not follow or relay redirects.", + "redirect_blocked", + ), + ); + return; + } + res.writeHead(status, buildForwardResponseHeaders(upstreamRes.headers)); + upstreamRes.once("aborted", () => { + failRequest( + new ForwardHttpError(502, "Upstream response aborted.", "upstream_response_aborted"), + ); + }); + upstreamRes.once("error", failRequest); + upstreamRes.pipe(res); + upstreamRes.once("end", () => resolveOnce(status)); + }, + ); + // If the original client disconnects before the response finishes, the + // pinned outbound connection would otherwise keep streaming from the real + // upstream until it finishes on its own or the upstream timeout fires -- + // an abandoned client could hold a pinned connection open indefinitely. + const onClientClose = () => { + if (res.writableEnded) return; + upstreamReq.destroy(); + resolveOnce(0); + }; + res.once("close", onClientClose); + upstreamReq.setTimeout( + options.upstreamTimeoutMs ?? HTTPS_PIN_RUNTIME_ADAPTER_UPSTREAM_TIMEOUT_MS, + () => { + upstreamReq.destroy( + new ForwardHttpError(504, "Upstream request timed out.", "upstream_timeout"), + ); + }, + ); + upstreamReq.on("error", (err) => { + failRequest(err); + }); + upstreamReq.end(body); + }); +} diff --git a/src/lib/inference/https-pin-runtime-adapter.test.ts b/src/lib/inference/https-pin-runtime-adapter.test.ts new file mode 100644 index 00000000000..47c38cbf845 --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter.test.ts @@ -0,0 +1,789 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { EndpointDnsLookupFn } from "./endpoint-ssrf-preflight"; +import { + __test, + createHttpsPinRuntimeAdapterServer, + ensureHttpsPinRuntimeAdapter, +} from "./https-pin-runtime-adapter"; + +const servers: http.Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); + servers.length = 0; +}); + +function listen(server: http.Server): Promise { + servers.push(server); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${address.port}`); + }); + }); +} + +const CONTROL_TOKEN = "test-control-plane-token"; +const ROUTE_TOKEN = "test-route-token"; + +function readRequestBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +describe("createHttpsPinRuntimeAdapterServer health and auth (#6141)", () => { + it("exposes an unauthenticated health endpoint without leaking the token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/health`); + expect(response.status).toBe(200); + const body = (await response.json()) as { ok: boolean; routeCount: number; tokenHash: string }; + expect(body).toMatchObject({ ok: true, routeCount: 0 }); + expect(typeof body.tokenHash).toBe("string"); + expect(JSON.stringify(body)).not.toContain(CONTROL_TOKEN); + }); + + it("rejects control-plane requests without a valid control token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + const putBody = JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-secret", + routeToken: ROUTE_TOKEN, + }); + + const missingAuth = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: putBody, + }); + expect(missingAuth.status).toBe(401); + + const wrongAuth = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: "Bearer wrong-token", "Content-Type": "application/json" }, + body: putBody, + }); + expect(wrongAuth.status).toBe(401); + + const body = (await wrongAuth.json()) as { error: { code: string } }; + expect(body.error.code).toBe("unauthorized"); + }); + + it("rejects route requests without that route's own valid bearer token", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-secret", + routeToken: ROUTE_TOKEN, + }), + }); + + const missingAuth = await fetch(`${baseUrl}/route/route-1`); + expect(missingAuth.status).toBe(401); + + const wrongAuth = await fetch(`${baseUrl}/route/route-1`, { + headers: { Authorization: "Bearer wrong-token" }, + }); + expect(wrongAuth.status).toBe(401); + + const body = (await wrongAuth.json()) as { error: { code: string } }; + expect(body.error.code).toBe("unauthorized"); + }); + + it("returns 404 for an unknown path", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/nonexistent`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(404); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer control plane (#6141)", () => { + it("registers a route via PUT and then forwards requests to it", async () => { + const upstreamRequests: Array<{ headers: http.IncomingHttpHeaders; body: string }> = []; + const upstream = http.createServer(async (req, res) => { + upstreamRequests.push({ headers: req.headers, body: await readRequestBody(req) }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const putResponse = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream-secret", + routeToken: ROUTE_TOKEN, + }), + }); + expect(putResponse.status).toBe(200); + await expect(putResponse.json()).resolves.toEqual({ ok: true, routeId: "route-1" }); + + const health = await fetch(`${baseUrl}/health`); + await expect(health.json()).resolves.toMatchObject({ routeCount: 1 }); + + const forwardResponse = await fetch(`${baseUrl}/route/route-1/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${ROUTE_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ hello: "world" }), + }); + expect(forwardResponse.status).toBe(200); + expect(upstreamRequests).toHaveLength(1); + expect(upstreamRequests[0].headers.authorization).toBe("Bearer sk-upstream-secret"); + expect(upstreamRequests[0].headers.host).toBe(`real-upstream.example:${upstreamPort}`); + expect(upstreamRequests[0].body).toBe(JSON.stringify({ hello: "world" })); + }); + + it("uses the anthropic credential header shape for an anthropic route", async () => { + const upstreamRequests: Array<{ headers: http.IncomingHttpHeaders }> = []; + const upstream = http.createServer(async (req, res) => { + upstreamRequests.push({ headers: req.headers }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + await fetch(`${baseUrl}/control/routes/route-anthropic`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "anthropic", + credentialValue: "sk-ant-secret", + routeToken: ROUTE_TOKEN, + }), + }); + + await fetch(`${baseUrl}/route/route-anthropic/v1/messages`, { + method: "POST", + headers: { Authorization: `Bearer ${ROUTE_TOKEN}`, "Content-Type": "application/json" }, + body: "{}", + }); + + expect(upstreamRequests[0].headers["x-api-key"]).toBe("sk-ant-secret"); + expect(upstreamRequests[0].headers.authorization).toBeUndefined(); + }); + + it("seeds routes from initialRoutes at construction, before any PUT", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: CONTROL_TOKEN, + initialRoutes: { + "bootstrap-route": { + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-bootstrap", + routeToken: ROUTE_TOKEN, + }, + }, + }); + const baseUrl = await listen(adapter); + + const health = await fetch(`${baseUrl}/health`); + await expect(health.json()).resolves.toMatchObject({ routeCount: 1 }); + + const response = await fetch(`${baseUrl}/route/bootstrap-route/`, { + headers: { Authorization: `Bearer ${ROUTE_TOKEN}` }, + }); + expect(response.status).toBe(200); + }); + + it("rejects PUT bodies missing required fields with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ targetBaseUrl: "http://example.com/" }), + }); + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "invalid_route" } }); + }); + + it("rejects PUT bodies with an unparseable targetBaseUrl with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: "not-a-url", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-secret", + routeToken: ROUTE_TOKEN, + }), + }); + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "invalid_route" } }); + }); + + it("rejects PUT bodies with an unsupported providerType with 400 invalid_route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "gemini", + credentialValue: "sk-secret", + routeToken: ROUTE_TOKEN, + }), + }); + expect(response.status).toBe(400); + }); + + it("rejects oversized control-plane bodies with 413", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: "http://example.com/", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "x".repeat(20 * 1024), + routeToken: ROUTE_TOKEN, + }), + }); + expect(response.status).toBe(413); + await expect(response.json()).resolves.toMatchObject({ error: { code: "request_too_large" } }); + }); + + it("returns 404 for a GET on the control-routes path (PUT only)", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/control/routes/route-1`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(404); + }); + + it("returns 404 route_not_found for an unregistered route id", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const response = await fetch(`${baseUrl}/route/never-registered`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(404); + await expect(response.json()).resolves.toMatchObject({ error: { code: "route_not_found" } }); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer orphaned route recovery (#6141)", () => { + it("returns 503 route_needs_recovery for a route orphaned by the last respawn, distinct from an unknown route", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: CONTROL_TOKEN, + orphanedRouteIds: ["orphan-1"], + }); + const baseUrl = await listen(adapter); + + const orphaned = await fetch(`${baseUrl}/route/orphan-1/v1/messages`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(orphaned.status).toBe(503); + await expect(orphaned.json()).resolves.toMatchObject({ + error: { code: "route_needs_recovery" }, + }); + + const neverKnown = await fetch(`${baseUrl}/route/never-known/v1/messages`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(neverKnown.status).toBe(404); + await expect(neverKnown.json()).resolves.toMatchObject({ error: { code: "route_not_found" } }); + }); + + it("prefers a live route over its own stale orphaned-route id once re-registered", async () => { + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + const upstreamBaseUrl = await listen(upstream); + const upstreamPort = new URL(upstreamBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ + controlToken: CONTROL_TOKEN, + orphanedRouteIds: ["healed-route"], + }); + const baseUrl = await listen(adapter); + + await fetch(`${baseUrl}/control/routes/healed-route`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream.example:${upstreamPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-healed", + routeToken: ROUTE_TOKEN, + }), + }); + + const response = await fetch(`${baseUrl}/route/healed-route/`, { + headers: { Authorization: `Bearer ${ROUTE_TOKEN}` }, + }); + expect(response.status).toBe(200); + }); +}); + +// Drives the server's request listener directly with a fake req/res instead +// of a real socket, so the simulated `remoteAddress` isn't at the mercy of +// how (or whether) a given host/CI sandbox routes secondary loopback +// addresses like 127.0.0.2 -- only the literal connection identity matters +// to `isLoopbackRemoteAddress`, not real network delivery. +function dispatchFakeRequest( + server: http.Server, + options: { + method: string; + url: string; + remoteAddress: string; + authorization?: string; + body?: unknown; + }, +): Promise<{ status: number; body: unknown }> { + const listener = server.listeners("request")[0] as ( + req: http.IncomingMessage, + res: http.ServerResponse, + ) => unknown; + + const req = new EventEmitter() as unknown as http.IncomingMessage; + Object.assign(req, { + method: options.method, + url: options.url, + headers: options.authorization ? { authorization: options.authorization } : {}, + socket: { remoteAddress: options.remoteAddress }, + }); + + return new Promise((resolve) => { + let status = 0; + const res = { + writeHead(code: number) { + status = code; + }, + end(payload?: string) { + resolve({ status, body: payload ? JSON.parse(payload) : undefined }); + }, + } as unknown as http.ServerResponse; + + void listener(req, res); + queueMicrotask(() => { + const chunks = options.body === undefined ? [] : [Buffer.from(JSON.stringify(options.body))]; + for (const chunk of chunks) (req as unknown as EventEmitter).emit("data", chunk); + (req as unknown as EventEmitter).emit("end"); + }); + }); +} + +describe("createHttpsPinRuntimeAdapterServer control-plane loopback restriction (#6141)", () => { + it("rejects a route registration whose connection did not arrive over loopback", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + + // The container-gateway address the sandbox actually connects from when + // it reaches the adapter through `host.openshell.internal` -- distinct + // from the literal 127.0.0.1 the host process itself always dials from. + const response = await dispatchFakeRequest(adapter, { + method: "PUT", + url: "/control/routes/route-1", + remoteAddress: "172.17.0.2", + authorization: `Bearer ${CONTROL_TOKEN}`, + body: { + targetBaseUrl: "http://internal.example/base", + pinnedAddresses: ["10.0.0.5"], + providerType: "openai", + credentialValue: "sk-should-not-register", + routeToken: ROUTE_TOKEN, + }, + }); + expect(response.status).toBe(404); + + const health = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/health", + remoteAddress: "172.17.0.2", + }); + expect(health.body).toMatchObject({ routeCount: 0 }); + }); + + it("still allows route registration over loopback", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + + const response = await dispatchFakeRequest(adapter, { + method: "PUT", + url: "/control/routes/route-1", + remoteAddress: "127.0.0.1", + authorization: `Bearer ${CONTROL_TOKEN}`, + body: { + targetBaseUrl: "http://real-upstream.example/base", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream-secret", + routeToken: ROUTE_TOKEN, + }, + }); + expect(response.status).toBe(200); + expect(response.body).toEqual({ ok: true, routeId: "route-1" }); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer route forwarding private-network restriction (#6141)", () => { + // These drive the gate itself, so an unregistered route ID is enough: a + // request that passes the private-network gate falls through to the + // "route_not_found" lookup (which never pipes a real upstream response), + // while a request blocked by the gate never reaches that lookup at all and + // instead gets the gate's own "not_found" code. + + it("rejects a route-forward request whose connection arrives from a public address", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + + // A peer that reached this 0.0.0.0-bound port from outside the intended + // Docker-bridge sandbox boundary -- an address the adapter should never + // trust a replayed bearer token from. 203.0.113.0/24 is the reserved + // TEST-NET-3 documentation range (RFC 5737), never a real bridge subnet. + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "203.0.113.5", + authorization: `Bearer ${CONTROL_TOKEN}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "not_found" } }); + }); + + it("still passes a route-forward request from the Docker-bridge sandbox address through to route lookup", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "172.17.0.2", + authorization: `Bearer ${CONTROL_TOKEN}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "route_not_found" } }); + }); + + it("still passes a route-forward request over loopback through to route lookup", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + + const response = await dispatchFakeRequest(adapter, { + method: "GET", + url: "/route/never-registered", + remoteAddress: "127.0.0.1", + authorization: `Bearer ${CONTROL_TOKEN}`, + }); + expect(response.status).toBe(404); + expect(response.body).toMatchObject({ error: { code: "route_not_found" } }); + }); +}); + +describe("createHttpsPinRuntimeAdapterServer per-route credential isolation (#6906)", () => { + it("rejects route A's token against route B and never forwards to route B's upstream", async () => { + const upstreamARequests: Array<{ headers: http.IncomingHttpHeaders }> = []; + const upstreamA = http.createServer(async (req, res) => { + upstreamARequests.push({ headers: req.headers }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, upstream: "a" })); + }); + const upstreamABaseUrl = await listen(upstreamA); + const upstreamAPort = new URL(upstreamABaseUrl).port; + + const upstreamBRequests: Array<{ headers: http.IncomingHttpHeaders }> = []; + const upstreamB = http.createServer(async (req, res) => { + upstreamBRequests.push({ headers: req.headers }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, upstream: "b" })); + }); + const upstreamBBaseUrl = await listen(upstreamB); + const upstreamBPort = new URL(upstreamBBaseUrl).port; + + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + const ROUTE_TOKEN_A = "route-a-token"; + const ROUTE_TOKEN_B = "route-b-token"; + + await fetch(`${baseUrl}/control/routes/route-a`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream-a.example:${upstreamAPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream-a-secret", + routeToken: ROUTE_TOKEN_A, + }), + }); + await fetch(`${baseUrl}/control/routes/route-b`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: `http://real-upstream-b.example:${upstreamBPort}/base`, + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-upstream-b-secret", + routeToken: ROUTE_TOKEN_B, + }), + }); + + // Route A's own token against route A succeeds and reaches upstream A. + const ownRouteA = await fetch(`${baseUrl}/route/route-a/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${ROUTE_TOKEN_A}`, "Content-Type": "application/json" }, + body: JSON.stringify({ hello: "a" }), + }); + expect(ownRouteA.status).toBe(200); + expect(upstreamARequests).toHaveLength(1); + + // Adversarial: route A's token replayed against route B must be + // rejected, and upstream B must never see the forwarded request. + const crossRoute = await fetch(`${baseUrl}/route/route-b/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${ROUTE_TOKEN_A}`, "Content-Type": "application/json" }, + body: JSON.stringify({ hello: "cross" }), + }); + expect(crossRoute.status).toBe(401); + expect(upstreamBRequests).toHaveLength(0); + const crossRouteBody = (await crossRoute.json()) as { error: { code: string } }; + expect(crossRouteBody.error.code).toBe("unauthorized"); + + // Route B's own token against route B still succeeds and reaches + // upstream B, proving the rejection above was scoping, not breakage. + const ownRouteB = await fetch(`${baseUrl}/route/route-b/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${ROUTE_TOKEN_B}`, "Content-Type": "application/json" }, + body: JSON.stringify({ hello: "b" }), + }); + expect(ownRouteB.status).toBe(200); + expect(upstreamBRequests).toHaveLength(1); + }); + + it("rejects the control-plane token when replayed against a route's data-plane path", async () => { + const adapter = createHttpsPinRuntimeAdapterServer({ controlToken: CONTROL_TOKEN }); + const baseUrl = await listen(adapter); + + await fetch(`${baseUrl}/control/routes/route-1`, { + method: "PUT", + headers: { Authorization: `Bearer ${CONTROL_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + targetBaseUrl: "http://real-upstream.example/base", + pinnedAddresses: ["127.0.0.1"], + providerType: "openai", + credentialValue: "sk-secret", + routeToken: ROUTE_TOKEN, + }), + }); + + const response = await fetch(`${baseUrl}/route/route-1`, { + headers: { Authorization: `Bearer ${CONTROL_TOKEN}` }, + }); + expect(response.status).toBe(401); + }); +}); + +describe("adapter recovery lock (#6141)", () => { + // The statically-imported `__test.LOCK_PATH` above is derived from this + // machine's real os.homedir() at module-evaluation time, same as a real, + // possibly-concurrently-running adapter's lock. Acquiring/deleting it here + // could steal or wedge that live adapter's lock. Give each test its own + // HOME (and therefore its own LOCK_PATH under a fresh temp `.nemoclaw`) via + // vi.resetModules() plus a fresh dynamic import, since STATE_DIR is only + // ever read once, at import time. + let tempHome: string; + let lockModule: typeof import("./https-pin-runtime-adapter"); + + beforeEach(async () => { + tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-adapter-lock-test-")); + vi.stubEnv("HOME", tempHome); + vi.resetModules(); + lockModule = await import("./https-pin-runtime-adapter"); + }); + + afterEach(() => { + try { + fs.unlinkSync(lockModule.__test.LOCK_PATH); + } catch { + /* nothing to clean up */ + } + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("blocks a second acquire while the first holder has not released", () => { + const release = lockModule.__test.tryAcquireAdapterLock(); + expect(release).not.toBeNull(); + expect(lockModule.__test.tryAcquireAdapterLock()).toBeNull(); + release?.(); + expect(lockModule.__test.tryAcquireAdapterLock()).not.toBeNull(); + }); + + it("serializes concurrent withAdapterLock operations instead of interleaving them", async () => { + const order: string[] = []; + const slow = lockModule.__test.withAdapterLock(async () => { + order.push("slow:start"); + await new Promise((resolve) => setTimeout(resolve, 50)); + order.push("slow:end"); + }); + // Give `slow` a head start so it wins the lock first. + await new Promise((resolve) => setTimeout(resolve, 5)); + const fast = lockModule.__test.withAdapterLock(async () => { + order.push("fast:start"); + order.push("fast:end"); + }); + + await Promise.all([slow, fast]); + + expect(order).toEqual(["slow:start", "slow:end", "fast:start", "fast:end"]); + }); +}); + +describe("computeRespawnState orphaned-route bookkeeping (#6141)", () => { + it("marks every persisted route except the one being bootstrapped as orphaned", () => { + const priorRoutes = { + a: { targetBaseUrl: "http://a.example/", pinnedAddresses: ["10.0.0.1"] }, + b: { targetBaseUrl: "http://b.example/", pinnedAddresses: ["10.0.0.2"] }, + c: { targetBaseUrl: "http://c.example/", pinnedAddresses: ["10.0.0.3"] }, + }; + + const { orphanedRouteIds, persistedRoutes } = __test.computeRespawnState(priorRoutes, "b"); + + expect(orphanedRouteIds.sort()).toEqual(["a", "c"]); + expect(Object.keys(persistedRoutes).sort()).toEqual(["a", "c"]); + expect(persistedRoutes.a).toMatchObject({ targetBaseUrl: "http://a.example/" }); + expect(typeof persistedRoutes.a.orphanedAt).toBe("string"); + expect(persistedRoutes.c).toMatchObject({ targetBaseUrl: "http://c.example/" }); + expect(typeof persistedRoutes.c.orphanedAt).toBe("string"); + expect(persistedRoutes.b).toBeUndefined(); + }); + + it("orphans nothing when there is no prior state to recover from", () => { + const { orphanedRouteIds, persistedRoutes } = __test.computeRespawnState({}, "bootstrap-only"); + + expect(orphanedRouteIds).toEqual([]); + expect(persistedRoutes).toEqual({}); + }); +}); + +describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#6141)", () => { + const privateLookup: EndpointDnsLookupFn = async () => [{ address: "10.48.203.205", family: 4 }]; + const publicLookup: EndpointDnsLookupFn = async () => [{ address: "93.184.216.34", family: 4 }]; + + it("rejects a DNS-private endpoint before ever considering the credential", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://internal.example.test/v1", + providerType: "openai", + // Deliberately empty: if the credential check ran first, the error + // message would mention "credential" instead of the SSRF reason. + credentialValue: "", + lookup: privateLookup, + }), + ).rejects.toThrow(/resolves to private\/internal address/); + }); + + it("rejects an empty credential only after the endpoint already resolved publicly", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://public.example.test/v1", + providerType: "openai", + credentialValue: " ", + lookup: publicLookup, + }), + ).rejects.toThrow(/requires a non-empty credential value/); + }); + + it("rejects a loopback endpoint (no pinnable address) before the credential check", async () => { + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://localhost/v1", + providerType: "openai", + credentialValue: "", + lookup: publicLookup, + }), + ).rejects.toThrow(/requires a DNS-resolved public address/); + }); + + it("surfaces the underlying resolver failure when DNS lookup itself errors", async () => { + const failingLookup: EndpointDnsLookupFn = async () => { + throw new Error("ENOTFOUND"); + }; + await expect( + ensureHttpsPinRuntimeAdapter({ + gatewayName: "gw", + provider: "compatible-endpoint", + endpointUrl: "https://does-not-resolve.example.test/v1", + providerType: "openai", + credentialValue: "sk-secret", + lookup: failingLookup, + }), + ).rejects.toThrow(/cannot resolve endpoint host/); + }); +}); diff --git a/src/lib/inference/https-pin-runtime-adapter.ts b/src/lib/inference/https-pin-runtime-adapter.ts new file mode 100644 index 00000000000..6c5fccd8499 --- /dev/null +++ b/src/lib/inference/https-pin-runtime-adapter.ts @@ -0,0 +1,1016 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTPS DNS-pinning runtime adapter server and lifecycle. + * + * Unlike the Bedrock/OpenRouter adapters (one process per singleton external + * endpoint), a host can have multiple DNS-backed HTTPS custom endpoints + * configured concurrently, so this adapter is one shared process serving many + * routes. Routes are registered on an already-running adapter through an + * authenticated control-plane `PUT /control/routes/:routeId` call instead of + * a full respawn, because respawning would lose every other route's + * credential value — those values are seeded into the process only at spawn + * time or via a control-plane call and are never written to disk (only a + * SHA-256 fingerprint is persisted, for diagnostics). + * + * If the adapter process dies, only the next route whose owning command calls + * `ensureHttpsPinRuntimeAdapter` recovers automatically; other previously + * registered routes stay unreachable until their owning command re-runs. This + * is an accepted consequence of never persisting plaintext credentials -- but + * the freshly spawned process is still told which route ids those are (never + * their credentials), so it can answer them with an actionable + * `route_needs_recovery` response instead of a 404 indistinguishable from a + * route that never existed (#6141). + */ + +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; + +import { + BEDROCK_RUNTIME_ADAPTER_PORT, + DASHBOARD_PORT, + DASHBOARD_PORT_RANGE_END, + DASHBOARD_PORT_RANGE_START, + GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, + OLLAMA_PORT, + OLLAMA_PROXY_PORT, + OPENROUTER_RUNTIME_ADAPTER_PORT, + VLLM_PORT, + validateHttpsPinRuntimeAdapterPort, +} from "../core/ports"; +import { compactText } from "../core/url-utils"; +import { ROOT, run, runCapture } from "../runner"; +import { buildMinimalCredentialAdapterEnv } from "../subprocess-env"; +import { assertEndpointResolvesPublic, type EndpointDnsLookupFn } from "./endpoint-ssrf-preflight"; +import { + buildHttpsPinRouteBaseUrl, + buildHttpsPinRouteLoopbackBaseUrl, + computeHttpsPinRouteId, + HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, + HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN, + HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + type HttpsPinCredentialProviderType, + resolveHttpsPinCredentialHeader, +} from "./https-pin-runtime"; +import { + ForwardHttpError, + forwardHttpsPinnedRequest, + type HttpsPinTarget, + sendForwardError, +} from "./https-pin-runtime-adapter-forward"; +import { + appendLocalAdapterJsonLine, + DEFAULT_LOCAL_ADAPTER_STATE_DIR, + ensureLocalAdapterStateDir, + isLocalAdapterProcess, + type JsonObject, + killLocalAdapterPid, + loadLocalAdapterPid, + localAdapterTokenHash, + persistLocalAdapterPid, + probeLocalAdapterHealth, + readLocalAdapterJsonFile, + readLocalAdapterTextFile, + removeLocalAdapterFile, + spawnDetachedNodeAdapter, + waitForLocalAdapterHealth, + writeLocalAdapterJsonFile, + writeLocalAdapterSecretFile, +} from "./local-adapter-lifecycle"; + +const STATE_DIR = DEFAULT_LOCAL_ADAPTER_STATE_DIR; +const TOKEN_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter-token"); +const PID_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.pid"); +const STATE_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.json"); +const LOCK_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.lock"); +export const LOG_PATH = path.join(STATE_DIR, "https-pin-runtime-adapter.log"); +const PROCESS_NEEDLE = "https-pin-runtime-adapter.js"; +const MAX_CONTROL_BODY_BYTES = 16 * 1024; +// Matches the sibling OpenRouter adapter's lock retry budget +// (openrouter-runtime-adapter-lifecycle.ts): long enough to outlast a normal +// spawn-and-health-check cycle, short enough to fail loudly on a truly stuck +// lock rather than hang the CLI command indefinitely. +const LOCK_RETRY_ATTEMPTS = 100; +const LOCK_RETRY_MS = 100; +const STALE_LOCK_MS = 30_000; + +interface RouteRuntime { + targetBaseUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + // Distinct random bearer token for this route only (#6906): the sandbox + // authorized for this route authenticates data-plane requests with this + // value, never the shared control-plane token, so a sandbox holding one + // route's token cannot replay it against a different route. + routeToken: string; +} + +interface RoutePersistedMeta { + targetBaseUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialHash: string; + registeredAt: string; +} + +type AdapterLogFields = Record; +type AdapterLogger = (event: string, fields?: AdapterLogFields) => void; + +function normalizeLogField( + value: string | number | boolean | null | undefined, +): string | number | boolean | null { + if (value === undefined) return null; + if (typeof value === "string") return compactText(value).slice(0, 180); + return value; +} + +function defaultAdapterLogger(event: string, fields: AdapterLogFields = {}): void { + try { + const payload: Record = { + ts: new Date().toISOString(), + event: normalizeLogField(event) as string, + }; + for (const [key, value] of Object.entries(fields)) { + payload[key] = normalizeLogField(value); + } + appendLocalAdapterJsonLine(LOG_PATH, payload); + } catch { + /* best-effort diagnostics only */ + } +} + +function logAdapterEvent( + logger: AdapterLogger, + event: string, + fields: AdapterLogFields = {}, +): void { + try { + logger(event, fields); + } catch { + /* best-effort diagnostics only */ + } +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value as JsonObject) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson((value as JsonObject)[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function authMatches(actual: string | string[] | undefined, token: string): boolean { + const header = Array.isArray(actual) ? actual[0] : actual; + if (!header) return false; + const expected = Buffer.from(`Bearer ${token}`); + const received = Buffer.from(header); + return received.length === expected.length && crypto.timingSafeEqual(received, expected); +} + +function isLoopbackRemoteAddress(remoteAddress: string | undefined): boolean { + if (!remoteAddress) return false; + const normalized = remoteAddress.replace(/^::ffff:/, ""); + return normalized === "127.0.0.1" || normalized === "::1"; +} + +/** + * Loopback plus the RFC1918 / unique-local ranges that cover the Docker + * bridge network a sandbox actually connects from when it reaches the + * adapter through `host.openshell.internal` (see the module doc comment on + * `HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST` for why the listener itself stays on + * `0.0.0.0`). This does not attempt to discover the real bridge subnet -- + * that varies by Docker/Colima/Podman setup -- it just excludes the case a + * `0.0.0.0` bind actually widens: a peer that reaches this host port over a + * public or otherwise routable address that was never the intended + * sandbox-to-host boundary. + */ +function isPrivateNetworkRemoteAddress(remoteAddress: string | undefined): boolean { + if (!remoteAddress) return false; + const normalized = remoteAddress.replace(/^::ffff:/, ""); + if (isLoopbackRemoteAddress(normalized)) return true; + const ipv4 = normalized.match(/^(\d{1,3})\.(\d{1,3})\.\d{1,3}\.\d{1,3}$/); + if (ipv4) { + const a = Number(ipv4[1]); + const b = Number(ipv4[2]); + if (a === 10) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + return false; + } + const lower = normalized.toLowerCase(); + // fc00::/7 (unique local) and fe80::/10 (link-local) + return /^f[cd][0-9a-f]{2}:/.test(lower) || /^fe[89ab][0-9a-f]:/.test(lower); +} + +function adapterTokenHash(token: string): string { + return localAdapterTokenHash(token); +} + +function routeCredentialHash( + endpointUrl: string, + providerType: HttpsPinCredentialProviderType, + credentialValue: string, +): string { + return crypto + .createHash("sha256") + .update(stableJson({ endpointUrl, providerType, credentialValue })) + .digest("hex"); +} + +function sendJson(res: http.ServerResponse, status: number, body: unknown): void { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); +} + +function safeHostname(rawUrl: string): string { + try { + return new URL(rawUrl).hostname; + } catch { + return "unknown"; + } +} + +function readControlRequestJson(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + req.on("data", (chunk: Buffer) => { + if (settled) return; + size += chunk.length; + if (size > MAX_CONTROL_BODY_BYTES) { + // Reject without destroying the socket: destroying `req` mid-stream + // tears down the underlying connection before the 413 response can + // flush, so the caller sees a raw connection reset instead of a + // clean error. Draining the remainder of a small control-plane body + // (16 KB cap) to let `res.end()` reach the client is cheap. + settled = true; + reject(new ForwardHttpError(413, "Request body is too large.", "request_too_large")); + return; + } + chunks.push(Buffer.from(chunk)); + }); + req.on("end", () => { + if (settled) return; + settled = true; + try { + const raw = Buffer.concat(chunks).toString("utf8"); + const parsed = raw ? JSON.parse(raw) : {}; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("expected a JSON object"); + } + resolve(parsed as JsonObject); + } catch { + reject(new ForwardHttpError(400, "Request body must be valid JSON.", "invalid_json")); + } + }); + req.on("error", (err) => { + if (settled) return; + settled = true; + reject(err); + }); + }); +} + +function parseRoutePutBody(raw: JsonObject): RouteRuntime { + const targetBaseUrl = typeof raw.targetBaseUrl === "string" ? raw.targetBaseUrl.trim() : ""; + const providerType = + raw.providerType === "anthropic" || raw.providerType === "openai" ? raw.providerType : null; + const credentialValue = typeof raw.credentialValue === "string" ? raw.credentialValue : ""; + const routeToken = typeof raw.routeToken === "string" ? raw.routeToken : ""; + const pinnedAddresses = Array.isArray(raw.pinnedAddresses) + ? raw.pinnedAddresses.filter( + (entry): entry is string => typeof entry === "string" && entry.length > 0, + ) + : []; + if ( + !targetBaseUrl || + !providerType || + !credentialValue || + !routeToken || + pinnedAddresses.length === 0 + ) { + throw new ForwardHttpError( + 400, + "targetBaseUrl, providerType, credentialValue, routeToken, and pinnedAddresses are required.", + "invalid_route", + ); + } + try { + new URL(targetBaseUrl); + } catch { + throw new ForwardHttpError(400, `"${targetBaseUrl}" is not a valid URL.`, "invalid_route"); + } + return { targetBaseUrl, pinnedAddresses, providerType, credentialValue, routeToken }; +} + +/** + * Builds the shared adapter server. Routes live only in memory (`routes`), + * seeded from `initialRoutes` at startup and otherwise populated by + * authenticated `PUT /control/routes/:routeId` calls from + * `ensureHttpsPinRuntimeAdapter`. + * + * Two distinct bearer credentials are in play (#6906): `controlToken` + * authenticates only the host-only, loopback-restricted control plane + * (`PUT /control/routes/:id`); each route's own `routeToken` authenticates + * only data-plane requests to that exact route (`/route/:id`). A sandbox + * holding one route's token never learns or can pass the control token, and + * cannot authenticate against any other route's data-plane path with it. + */ +export function createHttpsPinRuntimeAdapterServer(options: { + controlToken: string; + initialRoutes?: Record; + orphanedRouteIds?: string[]; + logger?: AdapterLogger; +}): http.Server { + const logger = options.logger || defaultAdapterLogger; + const routes = new Map(Object.entries(options.initialRoutes || {})); + const orphanedRouteIds = new Set(options.orphanedRouteIds || []); + + return http.createServer(async (req, res) => { + const started = Date.now(); + let routeId = "unknown"; + try { + const url = new URL(req.url || "/", "http://127.0.0.1"); + + if (req.method === "GET" && url.pathname === "/health") { + sendJson(res, 200, { + ok: true, + tokenHash: adapterTokenHash(options.controlToken), + routeCount: routes.size, + }); + return; + } + + const controlMatch = url.pathname.match(/^\/control\/routes\/([^/]+)$/); + if (controlMatch) { + routeId = controlMatch[1]; + if (!isLoopbackRemoteAddress(req.socket.remoteAddress)) { + // Route registration accepts a caller-supplied targetBaseUrl and + // pinnedAddresses with no SSRF re-validation here -- that only + // happens host-side in ensureHttpsPinRuntimeAdapter before it + // calls this endpoint over loopback. A sandbox authenticates + // data-plane requests with its own route token, never the control + // token, so without this check it could still try to reach + // /control/routes/:id directly and register a route pointed at an + // internal address. + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "control_plane_non_loopback", + durationMs: Date.now() - started, + }); + return; + } + if (!authMatches(req.headers.authorization, options.controlToken)) { + sendJson(res, 401, { + error: { message: "Unauthorized", type: "unauthorized", code: "unauthorized" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 401, + reason: "control_plane_unauthorized", + durationMs: Date.now() - started, + }); + return; + } + if (req.method !== "PUT") { + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + return; + } + const body = await readControlRequestJson(req); + const route = parseRoutePutBody(body); + routes.set(routeId, route); + sendJson(res, 200, { ok: true, routeId }); + logAdapterEvent(logger, "route_registered", { + routeId, + targetHost: safeHostname(route.targetBaseUrl), + providerType: route.providerType, + routeCount: routes.size, + durationMs: Date.now() - started, + }); + return; + } + + const routeMatch = url.pathname.match(/^\/route\/([^/]+)(\/.*)?$/); + if (routeMatch) { + routeId = routeMatch[1]; + if (!isPrivateNetworkRemoteAddress(req.socket.remoteAddress)) { + // Each route's token is scoped to that route alone, but a peer + // that reaches this port from outside the intended sandbox-to-host + // boundary still must not be able to probe route state at all. + sendJson(res, 404, { + error: { message: "Not found", type: "not_found", code: "not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "route_non_private_network", + durationMs: Date.now() - started, + }); + return; + } + const route = routes.get(routeId); + if (!route) { + if (orphanedRouteIds.has(routeId)) { + // Known before the adapter's last restart but not recovered by + // it -- distinct from a route that never existed, so the caller + // gets an actionable signal instead of an indistinguishable 404. + sendJson(res, 503, { + error: { + message: + "This route was registered before the adapter's last restart and was not recovered. Re-run the original `inference set --endpoint-url` command for this endpoint.", + type: "unavailable", + code: "route_needs_recovery", + }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 503, + reason: "route_needs_recovery", + durationMs: Date.now() - started, + }); + return; + } + sendJson(res, 404, { + error: { message: "Unknown route", type: "not_found", code: "route_not_found" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 404, + reason: "route_not_found", + durationMs: Date.now() - started, + }); + return; + } + // Route-scoped auth (#6906): compared against this specific route's + // own token, never the shared control token or any other route's + // token, so a sandbox authorized for a different route (or holding + // no credential at all) is rejected here before the request ever + // reaches this route's real upstream. + if (!authMatches(req.headers.authorization, route.routeToken)) { + sendJson(res, 401, { + error: { message: "Unauthorized", type: "unauthorized", code: "unauthorized" }, + }); + logAdapterEvent(logger, "request_rejected", { + routeId, + status: 401, + reason: "route_unauthorized", + durationMs: Date.now() - started, + }); + return; + } + const forwardPath = (routeMatch[2] || "/") + url.search; + const target: HttpsPinTarget = { + targetUrl: new URL(route.targetBaseUrl), + pinnedAddress: route.pinnedAddresses[0], + credential: resolveHttpsPinCredentialHeader(route.providerType, route.credentialValue), + }; + const status = await forwardHttpsPinnedRequest({ req, res, forwardPath, target }); + logAdapterEvent(logger, "request_forwarded", { + routeId, + status, + targetHost: safeHostname(route.targetBaseUrl), + durationMs: Date.now() - started, + }); + return; + } + + sendJson(res, 404, { error: { message: "Not found", type: "not_found", code: "not_found" } }); + } catch (err) { + const status = err instanceof ForwardHttpError ? err.status : 502; + const code = err instanceof ForwardHttpError ? err.code : "https_pin_runtime_error"; + logAdapterEvent(logger, "request_failed", { + routeId, + status, + code, + durationMs: Date.now() - started, + }); + sendForwardError(res, err); + } + }); +} + +function parseBootstrapRoute( + raw: string | undefined, +): { routeId: string; route: RouteRuntime } | null { + if (!raw) return null; + try { + const parsed = JSON.parse(raw) as { + routeId?: unknown; + targetBaseUrl?: unknown; + pinnedAddresses?: unknown; + providerType?: unknown; + credentialValue?: unknown; + routeToken?: unknown; + }; + if (typeof parsed.routeId !== "string" || !parsed.routeId) return null; + const route = parseRoutePutBody(parsed as JsonObject); + return { routeId: parsed.routeId, route }; + } catch { + return null; + } +} + +function parseOrphanedRouteIds(raw: string | undefined): string[] { + if (!raw) return []; + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed.filter((id): id is string => typeof id === "string") : []; + } catch { + return []; + } +} + +export function startHttpsPinRuntimeAdapterFromEnv(): http.Server { + const controlToken = process.env[HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV]; + const port = Number( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT || HTTPS_PIN_RUNTIME_ADAPTER_PORT, + ); + + if (!controlToken) { + throw new Error(`${HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV} is required`); + } + if (!Number.isInteger(port) || port <= 0) { + throw new Error("NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT must be a valid port"); + } + + const bootstrap = parseBootstrapRoute( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_BOOTSTRAP_ROUTE, + ); + const initialRoutes: Record = bootstrap + ? { [bootstrap.routeId]: bootstrap.route } + : {}; + const orphanedRouteIds = parseOrphanedRouteIds( + process.env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTE_IDS, + ); + + const server = createHttpsPinRuntimeAdapterServer({ + controlToken, + initialRoutes, + orphanedRouteIds, + }); + server.listen(port, HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, () => { + defaultAdapterLogger("adapter_ready", { + bindHost: HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST, + port, + routeCount: Object.keys(initialRoutes).length, + orphanedRouteCount: orphanedRouteIds.length, + logPath: LOG_PATH, + }); + console.log( + `HTTPS Pin Runtime adapter listening on ${HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST}:${port}; log ${LOG_PATH}`, + ); + }); + return server; +} + +function loadPersistedPid(): number | null { + return loadLocalAdapterPid(PID_PATH); +} + +function isAdapterProcess(pid: number | null | undefined): boolean { + return isLocalAdapterProcess(pid, PROCESS_NEEDLE, runCapture); +} + +function killStaleAdapter(): void { + killLocalAdapterPid({ pidPath: PID_PATH, processMatcher: PROCESS_NEEDLE, run, runCapture }); +} + +/** + * Unlike the Bedrock/OpenRouter adapters' hand-maintained `scripts/*.js` + * wrappers, this adapter is spawned directly from its own compiled output so + * the entrypoint stays TypeScript-only (see the `require.main` guard below). + */ +function getAdapterScriptPath(): string { + return path.join(ROOT, "dist", "lib", "inference", "https-pin-runtime-adapter.js"); +} + +function probeAdapterHealth( + options: { port?: number; tokenHash?: string | null } = {}, +): Promise { + return probeLocalAdapterHealth({ + host: HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + port: options.port || HTTPS_PIN_RUNTIME_ADAPTER_PORT, + expectedTokenHash: options.tokenHash || null, + }); +} + +async function waitForAdapterHealth( + token: string, + port = HTTPS_PIN_RUNTIME_ADAPTER_PORT, +): Promise { + const tokenHash = adapterTokenHash(token); + return waitForLocalAdapterHealth(() => probeAdapterHealth({ port, tokenHash }), { + attempts: 20, + intervalMs: 100, + }); +} + +function putRoute(options: { + controlToken: string; + routeId: string; + targetBaseUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + routeToken: string; +}): Promise { + return new Promise((resolve, reject) => { + const payload = JSON.stringify({ + targetBaseUrl: options.targetBaseUrl, + pinnedAddresses: options.pinnedAddresses, + providerType: options.providerType, + credentialValue: options.credentialValue, + routeToken: options.routeToken, + }); + const req = http.request( + { + hostname: HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST, + port: HTTPS_PIN_RUNTIME_ADAPTER_PORT, + path: `/control/routes/${options.routeId}`, + method: "PUT", + headers: { + Authorization: `Bearer ${options.controlToken}`, + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + }, + timeout: 3000, + }, + (res) => { + res.on("data", () => {}); + res.on("end", () => { + if (res.statusCode === 200) { + resolve(); + } else { + reject( + new Error( + `HTTPS Pin Runtime adapter rejected route registration (status ${res.statusCode}).`, + ), + ); + } + }); + }, + ); + req.on("timeout", () => { + req.destroy(); + reject(new Error("HTTPS Pin Runtime adapter route registration timed out.")); + }); + req.on("error", reject); + req.end(payload); + }); +} + +function extractPersistedRoutes(prior: JsonObject | null): Record { + if (!prior?.routes || typeof prior.routes !== "object" || Array.isArray(prior.routes)) return {}; + return prior.routes as Record; +} + +function persistRouteState(routeId: string, meta: RoutePersistedMeta): void { + const prior = readLocalAdapterJsonFile(STATE_PATH); + const priorRoutes = extractPersistedRoutes(prior); + writeLocalAdapterJsonFile(STATE_PATH, { + pid: (prior?.pid as number | null | undefined) ?? loadPersistedPid(), + updatedAt: new Date().toISOString(), + // Re-registering a route (fresh `meta`, no `orphanedAt`) always + // supersedes any prior orphaned entry for the same id -- this is how a + // route heals after its owner re-runs `inference set` post-recovery. + routes: { ...priorRoutes, [routeId]: meta }, + }); +} + +/** + * Computes which previously-registered routes a fresh adapter respawn will + * NOT recover (every one except the route currently being bootstrapped), + * since credentials are only ever seeded into the process at spawn/PUT time + * and are never persisted to disk (see the module doc comment). Returns + * their ids -- so the freshly spawned process can tell "this route was + * orphaned by a restart" apart from "this route never existed" and respond + * accordingly instead of a bare 404 either way -- plus the persisted-state + * shape that keeps them recorded (still without credentials) until their + * owner re-runs `inference set` and `persistRouteState` supersedes them. + */ +function computeRespawnState( + priorRoutes: Record, + bootstrapRouteId: string, +): { orphanedRouteIds: string[]; persistedRoutes: Record } { + const orphanedRouteIds: string[] = []; + const persistedRoutes: Record = {}; + const orphanedAt = new Date().toISOString(); + for (const [id, meta] of Object.entries(priorRoutes)) { + if (id === bootstrapRouteId) continue; + orphanedRouteIds.push(id); + persistedRoutes[id] = { ...meta, orphanedAt }; + } + return { orphanedRouteIds, persistedRoutes }; +} + +/** + * Ensures the shared adapter process is running and holds a current, + * pin-validated route for `(gatewayName, provider, endpointUrl)`, then + * returns the sandbox-facing base URL OpenShell should be registered with. + * + * Re-runs the SSRF preflight on every call so the pinned address is never + * older than this call — the address that gets registered is the one that + * gets connected to, closing the TOCTOU window between validation and the + * OpenShell gateway's own (would-be) resolution. + */ +export async function ensureHttpsPinRuntimeAdapter(options: { + gatewayName: string; + provider: string; + endpointUrl: string; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + lookup?: EndpointDnsLookupFn; +}): Promise<{ + baseUrl: string; + localBaseUrl: string; + logPath: string; + credentialEnv: string; + token: string; + routeId: string; + pinnedAddresses: string[]; +}> { + const preflight = await assertEndpointResolvesPublic(options.endpointUrl, options.lookup); + if (!preflight.ok) { + throw new Error( + `HTTPS Pin Runtime adapter cannot validate "${options.endpointUrl}": ${preflight.reason}`, + ); + } + const pinnedAddresses = + preflight.addresses && preflight.addresses.length > 0 ? preflight.addresses : []; + if (pinnedAddresses.length === 0) { + throw new Error( + `HTTPS Pin Runtime adapter requires a DNS-resolved public address for "${options.endpointUrl}".`, + ); + } + // Checked only after the endpoint itself is proven safe to pin: an + // unreachable/private endpoint must fail on that ground, not report a + // confusing credential error for a URL that was never going to be allowed. + if (!options.credentialValue || !options.credentialValue.trim()) { + throw new Error( + `HTTPS Pin Runtime adapter requires a non-empty credential value for "${options.endpointUrl}".`, + ); + } + + const routeId = computeHttpsPinRouteId( + options.gatewayName, + options.provider, + options.endpointUrl, + ); + // Minted fresh on every call, distinct from every other route's token and + // from the adapter's own control-plane token (#6906): this is the only + // credential the sandbox that owns this route ever receives, and it + // authenticates data-plane requests to this route alone. + const routeToken = crypto.randomBytes(24).toString("hex"); + const controlToken = await ensureAdapterProcess({ + routeId, + endpointUrl: options.endpointUrl, + pinnedAddresses, + providerType: options.providerType, + credentialValue: options.credentialValue, + routeToken, + }); + + await putRoute({ + controlToken, + routeId, + targetBaseUrl: options.endpointUrl, + pinnedAddresses, + providerType: options.providerType, + credentialValue: options.credentialValue, + routeToken, + }); + persistRouteState(routeId, { + targetBaseUrl: options.endpointUrl, + pinnedAddresses, + providerType: options.providerType, + credentialHash: routeCredentialHash( + options.endpointUrl, + options.providerType, + options.credentialValue, + ), + registeredAt: new Date().toISOString(), + }); + + return { + baseUrl: buildHttpsPinRouteBaseUrl(routeId, options.endpointUrl), + localBaseUrl: buildHttpsPinRouteLoopbackBaseUrl(routeId, options.endpointUrl), + logPath: LOG_PATH, + credentialEnv: HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV, + // Route-scoped, not the adapter's shared control-plane token (#6906): + // this is what the caller stages as the sandbox-facing credential, so + // each route's sandbox only ever learns its own token. + token: routeToken, + routeId, + pinnedAddresses, + }; +} + +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function removeStaleLock(): void { + try { + const ageMs = Date.now() - fs.statSync(LOCK_PATH).mtimeMs; + if (ageMs > STALE_LOCK_MS) fs.unlinkSync(LOCK_PATH); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } +} + +function tryAcquireAdapterLock(): (() => void) | null { + ensureLocalAdapterStateDir(STATE_DIR); + removeStaleLock(); + try { + const fd = fs.openSync(LOCK_PATH, "wx", 0o600); + fs.writeFileSync(fd, `${process.pid}\n${new Date().toISOString()}\n`); + fs.closeSync(fd); + return () => { + try { + fs.unlinkSync(LOCK_PATH); + } catch { + /* best-effort lock cleanup */ + } + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "EEXIST") return null; + throw err; + } +} + +/** + * Serializes the read-check-kill-spawn recovery decision in + * `ensureAdapterProcess` across concurrent `inference set` invocations. + * Without this, two callers can both see no healthy prior process, both kill + * and respawn, and race to bind the same port and overwrite + * PID_PATH/TOKEN_PATH/STATE_PATH -- leaking a process and potentially + * leaving the persisted token out of sync with whichever process actually + * won the port. + */ +async function withAdapterLock(operation: () => Promise): Promise { + for (let attempt = 0; attempt < LOCK_RETRY_ATTEMPTS; attempt++) { + const release = tryAcquireAdapterLock(); + if (release) { + try { + return await operation(); + } finally { + release(); + } + } + await sleepMs(LOCK_RETRY_MS); + } + throw new Error("HTTPS Pin Runtime adapter startup is already in progress"); +} + +function validateAdapterPortConfiguration(): void { + validateHttpsPinRuntimeAdapterPort( + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + HTTPS_PIN_RUNTIME_ADAPTER_PORT, + { + dashboardPort: DASHBOARD_PORT, + dashboardRangeStart: DASHBOARD_PORT_RANGE_START, + dashboardRangeEnd: DASHBOARD_PORT_RANGE_END, + gatewayPort: GATEWAY_PORT, + vllmPort: VLLM_PORT, + ollamaPort: OLLAMA_PORT, + ollamaProxyPort: OLLAMA_PROXY_PORT, + bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, + openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, + }, + ); +} + +/** Returns a live adapter control token, reusing the running process when possible or spawning a fresh one. */ +async function ensureAdapterProcessLocked(bootstrap: { + routeId: string; + endpointUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + routeToken: string; +}): Promise { + validateAdapterPortConfiguration(); + const priorToken = readLocalAdapterTextFile(TOKEN_PATH); + const priorPid = loadPersistedPid(); + if ( + priorToken && + isAdapterProcess(priorPid) && + (await probeAdapterHealth({ tokenHash: adapterTokenHash(priorToken) })) + ) { + return priorToken; + } + + killStaleAdapter(); + // Reusing a still-valid persisted control token (rather than always + // minting a new one) keeps the running adapter process's identity stable + // across a respawn whenever possible. This is the host-only control-plane + // token (#6906) -- never staged into a sandbox. + const controlToken = priorToken || crypto.randomBytes(24).toString("hex"); + // A fresh process starts with an empty in-memory route map -- every route + // other than the one being bootstrapped now is unrecoverable this restart, + // since credentials are never persisted to disk (see module doc comment). + // Tell the freshly spawned process which route ids those are so it can + // answer them with an actionable "needs recovery" response instead of a + // bare 404 indistinguishable from a route that never existed (#6141). + const priorState = readLocalAdapterJsonFile(STATE_PATH); + const { orphanedRouteIds, persistedRoutes } = computeRespawnState( + extractPersistedRoutes(priorState), + bootstrap.routeId, + ); + const child = spawnDetachedNodeAdapter({ + scriptPath: getAdapterScriptPath(), + env: { + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT: String(HTTPS_PIN_RUNTIME_ADAPTER_PORT), + [HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV]: controlToken, + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_BOOTSTRAP_ROUTE: JSON.stringify({ + routeId: bootstrap.routeId, + targetBaseUrl: bootstrap.endpointUrl, + pinnedAddresses: bootstrap.pinnedAddresses, + providerType: bootstrap.providerType, + credentialValue: bootstrap.credentialValue, + routeToken: bootstrap.routeToken, + }), + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTE_IDS: JSON.stringify(orphanedRouteIds), + }, + // This is a long-lived, credential-bearing process, so it gets a + // purpose-built minimal environment rather than the general subprocess + // allowlist -- it must not inherit DOCKER_HOST/KUBECONFIG/SSH_AUTH_SOCK/ + // proxy capabilities that an ordinary short-lived CLI subprocess might + // legitimately need. See #6141. + buildEnv: buildMinimalCredentialAdapterEnv, + }); + try { + persistLocalAdapterPid(PID_PATH, child.pid); + if (!(await waitForAdapterHealth(controlToken))) { + throw new Error( + `HTTPS Pin Runtime adapter did not become healthy on ${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}`, + ); + } + writeLocalAdapterSecretFile(TOKEN_PATH, controlToken); + // Keep the orphaned routes recorded (still without credentials) instead + // of dropping them: `persistRouteState` supersedes an entry here the + // moment its owner re-runs `inference set`, which is how a route heals. + writeLocalAdapterJsonFile(STATE_PATH, { + pid: child.pid ?? null, + updatedAt: new Date().toISOString(), + routes: persistedRoutes, + }); + } catch (err) { + killStaleAdapter(); + removeLocalAdapterFile(STATE_PATH); + throw err; + } + return controlToken; +} + +function ensureAdapterProcess(bootstrap: { + routeId: string; + endpointUrl: string; + pinnedAddresses: string[]; + providerType: HttpsPinCredentialProviderType; + credentialValue: string; + routeToken: string; +}): Promise { + return withAdapterLock(() => ensureAdapterProcessLocked(bootstrap)); +} + +export const __test = { + routeCredentialHash, + getAdapterScriptPath, + probeAdapterHealth, + tryAcquireAdapterLock, + withAdapterLock, + computeRespawnState, + LOCK_PATH, +}; + +// Detached-process entrypoint: `spawnDetachedNodeAdapter` runs this compiled +// file directly with plain `node` (see `getAdapterScriptPath`), so this guard +// is the only thing that distinguishes that invocation from the normal +// `require()` used by the rest of the CLI. +if (require.main === module) { + try { + startHttpsPinRuntimeAdapterFromEnv(); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} diff --git a/src/lib/inference/https-pin-runtime.test.ts b/src/lib/inference/https-pin-runtime.test.ts new file mode 100644 index 00000000000..dec59989f1a --- /dev/null +++ b/src/lib/inference/https-pin-runtime.test.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN, + HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN, + buildHttpsPinRouteBaseUrl, + buildHttpsPinRouteLoopbackBaseUrl, + computeHttpsPinRouteId, + isHttpsPinRuntimeEligible, + resolveHttpsPinCredentialHeader, +} from "./https-pin-runtime"; + +describe("isHttpsPinRuntimeEligible (#6141)", () => { + it("is eligible for a DNS-backed HTTPS hostname", () => { + expect(isHttpsPinRuntimeEligible("https://api.example.com/v1")).toBe(true); + }); + + it("is not eligible for HTTP, even with a DNS-backed hostname", () => { + expect(isHttpsPinRuntimeEligible("http://api.example.com/v1")).toBe(false); + }); + + it("is not eligible for an HTTPS IPv4 literal", () => { + expect(isHttpsPinRuntimeEligible("https://93.184.216.34/v1")).toBe(false); + }); + + it("is not eligible for an HTTPS IPv6 literal", () => { + expect(isHttpsPinRuntimeEligible("https://[2001:db8::1]/v1")).toBe(false); + }); + + it("is not eligible for NemoClaw's own OpenShell-managed host alias", () => { + expect(isHttpsPinRuntimeEligible("https://host.openshell.internal/v1")).toBe(false); + }); + + it("is not eligible for other OpenShell-managed aliases (inference.local, host.docker.internal)", () => { + expect(isHttpsPinRuntimeEligible("https://inference.local/v1")).toBe(false); + expect(isHttpsPinRuntimeEligible("https://host.docker.internal/v1")).toBe(false); + }); + + it("is not eligible for an unparseable URL", () => { + expect(isHttpsPinRuntimeEligible("not-a-url")).toBe(false); + }); + + it("is not eligible for null/undefined/empty input", () => { + expect(isHttpsPinRuntimeEligible(null)).toBe(false); + expect(isHttpsPinRuntimeEligible(undefined)).toBe(false); + expect(isHttpsPinRuntimeEligible("")).toBe(false); + }); + + it("accepts a URL instance the same as an equivalent string", () => { + expect(isHttpsPinRuntimeEligible(new URL("https://api.example.com/v1"))).toBe(true); + }); +}); + +describe("resolveHttpsPinCredentialHeader (#6141)", () => { + it("uses x-api-key for the anthropic provider type", () => { + expect(resolveHttpsPinCredentialHeader("anthropic", "sk-ant-secret")).toEqual({ + name: "x-api-key", + value: "sk-ant-secret", + }); + }); + + it("uses a Bearer authorization header for the openai provider type", () => { + expect(resolveHttpsPinCredentialHeader("openai", "sk-secret")).toEqual({ + name: "authorization", + value: "Bearer sk-secret", + }); + }); +}); + +describe("computeHttpsPinRouteId (#6141)", () => { + it("is deterministic for the same (gateway, provider, endpoint) triple", () => { + const a = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + const b = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + expect(a).toBe(b); + }); + + it("differs when any input differs", () => { + const base = computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.example.com/v1"); + expect( + computeHttpsPinRouteId("other-gw", "compatible-endpoint", "https://api.example.com/v1"), + ).not.toBe(base); + expect( + computeHttpsPinRouteId("gw", "compatible-anthropic-endpoint", "https://api.example.com/v1"), + ).not.toBe(base); + expect( + computeHttpsPinRouteId("gw", "compatible-endpoint", "https://api.other.com/v1"), + ).not.toBe(base); + }); + + it("never contains the endpoint hostname or path (safe to persist and log)", () => { + const id = computeHttpsPinRouteId( + "gw", + "compatible-endpoint", + "https://api.example.com/some/secret/path", + ); + expect(id).not.toContain("api.example.com"); + expect(id).not.toContain("secret"); + expect(id).toMatch(/^[0-9a-f]{20}$/); + }); +}); + +describe("buildHttpsPinRouteBaseUrl / buildHttpsPinRouteLoopbackBaseUrl (#6141)", () => { + it("builds the sandbox-facing origin with the route id and the endpoint's path preserved", () => { + const url = buildHttpsPinRouteBaseUrl("routeid1234567890abc", "https://api.example.com/v1/"); + expect(url).toBe(`${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/routeid1234567890abc/v1`); + }); + + it("builds the host-side loopback equivalent for the same route", () => { + const url = buildHttpsPinRouteLoopbackBaseUrl( + "routeid1234567890abc", + "https://api.example.com/v1/", + ); + expect(url).toBe(`${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}/route/routeid1234567890abc/v1`); + }); + + it("omits the suffix entirely for a root-path endpoint", () => { + expect(buildHttpsPinRouteBaseUrl("routeid1234567890abc", "https://api.example.com/")).toBe( + `${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/routeid1234567890abc`, + ); + }); + + it("does not leak the real hostname into either base URL", () => { + const sandboxUrl = buildHttpsPinRouteBaseUrl( + "routeid1234567890abc", + "https://api.example.com/v1", + ); + const loopbackUrl = buildHttpsPinRouteLoopbackBaseUrl( + "routeid1234567890abc", + "https://api.example.com/v1", + ); + expect(sandboxUrl).not.toContain("api.example.com"); + expect(loopbackUrl).not.toContain("api.example.com"); + }); +}); diff --git a/src/lib/inference/https-pin-runtime.ts b/src/lib/inference/https-pin-runtime.ts new file mode 100644 index 00000000000..42e41febf38 --- /dev/null +++ b/src/lib/inference/https-pin-runtime.ts @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTPS DNS-pinning runtime adapter: classification and hidden adapter + * constants for arbitrary DNS-backed HTTPS custom inference endpoints + * (`compatible-endpoint` / `compatible-anthropic-endpoint`). + * + * A DNS-backed HTTPS custom endpoint cannot be registered with OpenShell + * directly: OpenShell's gateway re-resolves the hostname when it makes its + * own outbound connection, which can race the SSRF preflight's resolution + * (TOCTOU/DNS rebinding) and exposes the real hostname to that runtime + * boundary. This module classifies which endpoints need the adapter; the + * adapter itself (`https-pin-runtime-adapter.ts`) terminates the pinned + * outbound HTTPS connection on the host, immediately after the addresses it + * connects to were validated, and registers only its own loopback-adjacent + * `host.openshell.internal` route with OpenShell. + */ + +import crypto from "node:crypto"; +import { isIP } from "node:net"; + +import { HTTPS_PIN_RUNTIME_ADAPTER_PORT } from "../core/ports"; +import { isOpenShellManagedHost } from "./endpoint-ssrf-preflight"; + +/** + * Env var name under which a sandbox's own route-scoped data-plane bearer + * token is staged (one distinct random value per route, minted by + * `ensureHttpsPinRuntimeAdapter`). Never the real upstream credential, and + * never shared across routes -- a sandbox authorized for one route must not + * be able to replay this value against a different route on the same shared + * adapter (#6906). + */ +export const HTTPS_PIN_RUNTIME_ADAPTER_PROVIDER_CREDENTIAL_ENV = + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_TOKEN"; +/** + * Env var name for the adapter process's own control-plane bearer token, + * used only for the host-only, loopback-restricted `PUT /control/routes/:id` + * call. Kept separate from the per-route data-plane token above: this value + * is never given to a sandbox (#6906). + */ +export const HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN_ENV = + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN"; +export const HTTPS_PIN_RUNTIME_ADAPTER_BIND_HOST = "0.0.0.0"; +export const HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST = "127.0.0.1"; +export const HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST = "host.openshell.internal"; +export const HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN = `http://${HTTPS_PIN_RUNTIME_ADAPTER_SANDBOX_HOST}:${HTTPS_PIN_RUNTIME_ADAPTER_PORT}`; +export const HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN = `http://${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_HOST}:${HTTPS_PIN_RUNTIME_ADAPTER_PORT}`; + +export type HttpsPinCredentialProviderType = "openai" | "anthropic"; + +export interface HttpsPinCredentialHeader { + name: string; + value: string; +} + +/** Upstream credential header for the real endpoint, matching each provider type's existing convention. */ +export function resolveHttpsPinCredentialHeader( + providerType: HttpsPinCredentialProviderType, + credentialValue: string, +): HttpsPinCredentialHeader { + if (providerType === "anthropic") { + return { name: "x-api-key", value: credentialValue }; + } + return { name: "authorization", value: `Bearer ${credentialValue}` }; +} + +function parseUrl(value: string | URL | null | undefined): URL | null { + const raw = value instanceof URL ? value.href : String(value || "").trim(); + if (!raw) return null; + try { + return new URL(raw); + } catch { + return null; + } +} + +/** + * True when `endpointUrl` is exactly the shape that the DNS-pinning adapter + * exists for: HTTPS, a DNS-backed hostname (not an IP literal), and not one + * of NemoClaw's own trusted OpenShell-managed aliases. HTTP endpoints are + * already handled by direct IP substitution; HTTPS IP-literal endpoints + * already connect to an address the caller can see up front; OpenShell + * aliases are already exempt loopback-equivalent routes. + */ +export function isHttpsPinRuntimeEligible(endpointUrl: string | URL | null | undefined): boolean { + const url = parseUrl(endpointUrl); + if (!url || url.protocol !== "https:") return false; + const hostname = url.hostname; + const bare = + hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + if (isIP(bare)) return false; + if (isOpenShellManagedHost(hostname)) return false; + return true; +} + +/** Deterministic, stable identifier for one (gateway, provider, endpoint) route. Safe to persist and log. */ +export function computeHttpsPinRouteId( + gatewayName: string, + provider: string, + endpointUrl: string, +): string { + return crypto + .createHash("sha256") + .update(`${gatewayName} ${provider} ${endpointUrl}`) + .digest("hex") + .slice(0, 20); +} + +/** Sandbox-facing base URL for a route: adapter origin + `/route/` + the endpoint's own path. */ +export function buildHttpsPinRouteBaseUrl(routeId: string, endpointUrl: string): string { + const url = parseUrl(endpointUrl); + const suffix = url ? url.pathname.replace(/\/+$/, "") : ""; + return `${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/${routeId}${suffix}`; +} + +/** Host-side (loopback) equivalent of {@link buildHttpsPinRouteBaseUrl}, for health checks and the control plane. */ +export function buildHttpsPinRouteLoopbackBaseUrl(routeId: string, endpointUrl: string): string { + const url = parseUrl(endpointUrl); + const suffix = url ? url.pathname.replace(/\/+$/, "") : ""; + return `${HTTPS_PIN_RUNTIME_ADAPTER_LOOPBACK_ORIGIN}/route/${routeId}${suffix}`; +} diff --git a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts index c901ca2b4ed..36894e3990f 100644 --- a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts +++ b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts @@ -11,6 +11,7 @@ import { DASHBOARD_PORT_RANGE_END, DASHBOARD_PORT_RANGE_START, GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT, OPENROUTER_RUNTIME_ADAPTER_PORT, @@ -222,6 +223,7 @@ function validateAdapterPortConfiguration(): void { ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }, ); } diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index fbec2303249..79ebd7ffc38 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -12,6 +12,7 @@ import { DASHBOARD_PORT_RANGE_END, DASHBOARD_PORT_RANGE_START, GATEWAY_PORT, + HTTPS_PIN_RUNTIME_ADAPTER_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT, OPENROUTER_RUNTIME_ADAPTER_PORT, @@ -104,6 +105,7 @@ function resolveGatewayRecoveryTarget(options: StartGatewayForRecoveryOptions = ollamaProxyPort: OLLAMA_PROXY_PORT, bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }); return { gatewayName, gatewayPort }; } diff --git a/src/lib/onboard/inference-providers/hermes.ts b/src/lib/onboard/inference-providers/hermes.ts index 3f5fb848e09..f1527d53330 100644 --- a/src/lib/onboard/inference-providers/hermes.ts +++ b/src/lib/onboard/inference-providers/hermes.ts @@ -53,9 +53,10 @@ export async function setupHermesProviderInference( } // DNS-resolving + pinning validation closes the DNS-rebinding gap a // string-only hostname check leaves open. For HTTP this returns the - // pinned-IP URL. DNS-backed HTTPS fails closed until NemoClaw has a - // runtime-aware transport that can preserve TLS SNI/Host while pinning the - // resolved peer IP across the downstream OpenShell boundary. + // pinned-IP URL. DNS-backed HTTPS fails closed here: onboarding does not + // wire the HTTPS Pin Runtime adapter (see inference-set-route-containment.ts), + // so a DNS-backed HTTPS endpoint must be set after onboarding via + // `inference set --endpoint-url`. try { const validated = await rewriteConfigUrlsWithDnsPinning(endpointUrl, deps.lookup); resolvedEndpointUrl = typeof validated === "string" ? validated : endpointUrl; diff --git a/src/lib/sandbox/config.ts b/src/lib/sandbox/config.ts index a7b8d906ea8..7e30d3ebd4f 100644 --- a/src/lib/sandbox/config.ts +++ b/src/lib/sandbox/config.ts @@ -153,10 +153,11 @@ function buildConfigSetRestartGuidance(sandboxName: string, agentName: string): ]; } -class ConfigUrlValidationError extends Error { +export class ConfigUrlValidationError extends Error { constructor( readonly urlValue: string, message: string, + readonly reason: "dns_backed_https_unsupported" | "invalid" = "invalid", ) { super(message); this.name = "ConfigUrlValidationError"; @@ -950,16 +951,22 @@ async function rewriteConfigUrlsWithDnsPinning( // HTTPS endpoints fail closed for generic persisted config because the // downstream consumer would otherwise perform a second DNS lookup while // NemoClaw cannot pin the peer IP and preserve TLS SNI/Host across the - // OpenShell runtime boundary. + // OpenShell runtime boundary. This validator handles arbitrary persisted + // config values, not just inference endpoints, so the message stays + // generic; callers that know the field is an inference endpoint add + // their own guidance by checking `reason` on the thrown error. if (validated.protocol === "https:" && validated.pinnedUrl !== validated.originalUrl) { - throw new Error( - "DNS-backed HTTPS URLs are not supported for persisted sandbox config yet. " + - "Use an HTTPS IP-literal endpoint, an HTTP endpoint that can be DNS-pinned, " + - "or wait for the runtime-aware HTTPS pinning transport.", + throw new ConfigUrlValidationError( + trimmed, + "DNS-backed HTTPS URLs are not supported for arbitrary persisted sandbox config " + + "values. Use an HTTPS IP-literal endpoint or an HTTP endpoint that can be " + + "DNS-pinned.", + "dns_backed_https_unsupported", ); } return validated.protocol === "http:" ? validated.pinnedUrl : validated.originalUrl; } catch (err: unknown) { + if (err instanceof ConfigUrlValidationError) throw err; const message = err instanceof Error ? err.message : String(err); throw new ConfigUrlValidationError(trimmed, message); } diff --git a/src/lib/subprocess-env.test.ts b/src/lib/subprocess-env.test.ts index 97fd9022e76..2e417b539d7 100644 --- a/src/lib/subprocess-env.test.ts +++ b/src/lib/subprocess-env.test.ts @@ -196,3 +196,75 @@ describe("buildSubprocessEnv NO_PROXY injection", () => { expect(env.NO_PROXY).toBe(LOCAL_NO_PROXY); }); }); + +describe("buildMinimalCredentialAdapterEnv", () => { + const originalEnv = process.env; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("excludes toolchain, proxy, and unrelated secret-like variables even when ambient", async () => { + process.env.DOCKER_HOST = "unix:///var/run/docker.sock"; + process.env.KUBECONFIG = "/home/user/.kube/config"; + process.env.SSH_AUTH_SOCK = "/tmp/ssh-agent.sock"; + process.env.HTTP_PROXY = "http://proxy.example.com:8888"; + process.env.HTTPS_PROXY = "http://proxy.example.com:8888"; + process.env.NO_PROXY = "corp.internal"; + process.env.NVIDIA_INFERENCE_API_KEY = "should-not-leak"; + process.env.GITHUB_TOKEN = "should-not-leak"; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.DOCKER_HOST).toBeUndefined(); + expect(env.KUBECONFIG).toBeUndefined(); + expect(env.SSH_AUTH_SOCK).toBeUndefined(); + expect(env.HTTP_PROXY).toBeUndefined(); + expect(env.HTTPS_PROXY).toBeUndefined(); + expect(env.NO_PROXY).toBeUndefined(); + expect(env.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + }); + + it("includes only the documented runtime and TLS names", async () => { + process.env.HOME = "/home/user"; + process.env.PATH = "/usr/bin:/bin"; + process.env.NODE_ENV = "production"; + process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/corp-ca.pem"; + process.env.SSL_CERT_FILE = "/etc/ssl/cert.pem"; + process.env.USER = "someone-else-is-not-included"; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.HOME).toBe("/home/user"); + expect(env.PATH).toBe("/usr/bin:/bin"); + expect(env.NODE_ENV).toBe("production"); + expect(env.NODE_EXTRA_CA_CERTS).toBe("/etc/ssl/corp-ca.pem"); + expect(env.SSL_CERT_FILE).toBe("/etc/ssl/cert.pem"); + expect(env.USER).toBeUndefined(); + }); + + it("merges explicit adapter bootstrap fields via extra", async () => { + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv({ + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT: "9999", + }); + expect(env.NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT).toBe("9999"); + }); + + it("does not inject NO_PROXY the way buildSubprocessEnv does, since PROXY vars are never forwarded", async () => { + process.env.HTTP_PROXY = "http://proxy.example.com:8888"; + delete process.env.NO_PROXY; + delete process.env.no_proxy; + + const { buildMinimalCredentialAdapterEnv } = await import("./subprocess-env"); + const env = buildMinimalCredentialAdapterEnv(); + expect(env.NO_PROXY).toBeUndefined(); + expect(env.no_proxy).toBeUndefined(); + }); +}); diff --git a/src/lib/subprocess-env.ts b/src/lib/subprocess-env.ts index 54067365810..f334e79ef6a 100644 --- a/src/lib/subprocess-env.ts +++ b/src/lib/subprocess-env.ts @@ -132,3 +132,34 @@ export function buildSubprocessEnv(extra?: Record): Record, +): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value === undefined) continue; + if (ADAPTER_RUNTIME_NAMES.includes(key) || TLS.includes(key)) { + env[key] = value; + } + } + if (extra) { + Object.assign(env, extra); + } + return env; +} diff --git a/test/e2e/live/https-pin-compatible-server.ts b/test/e2e/live/https-pin-compatible-server.ts new file mode 100644 index 00000000000..0792cc3a9c2 --- /dev/null +++ b/test/e2e/live/https-pin-compatible-server.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; + +import { + closeServer, + writeJsonResponse as jsonResponse, + listenServer as listenOnRandomPort, + readRequestBody, +} from "../fixtures/http-protocol.ts"; +import type { StartedHttpServer } from "./mcp-bridge-servers.ts"; + +export interface FakeHttpsCompatibleRequest { + readonly method: string; + readonly path: string; + readonly hostHeader?: string; + readonly auth: "ok" | "missing" | "invalid"; + readonly body: string; +} + +export interface FakeHttpsCompatibleServer extends StartedHttpServer { + requests(): readonly FakeHttpsCompatibleRequest[]; +} + +function requireTcpPort(server: https.Server): number { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("fake HTTPS compatible endpoint did not bind to a TCP port"); + } + return address.port; +} + +function generateEphemeralTlsMaterial(): { dir: string; cert: Buffer; key: Buffer } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-https-pin-tls-")); + const keyPath = path.join(dir, "server.key"); + const certPath = path.join(dir, "server.crt"); + // Only the loopback hop from cloudflared to this process consumes this + // certificate, and the quick tunnel is launched with --no-tls-verify for + // that local origin (test/e2e/setup-mcp-test-tls.sh documents the identical + // rationale for the MCP HTTPS fixture). A self-signed leaf with no separate + // CA is sufficient; the sandbox only ever sees the public tunnel origin and + // its real, publicly trusted trycloudflare.com certificate. + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-sha256", + "-nodes", + "-days", + "1", + "-subj", + "/CN=nemoclaw-https-pin-e2e", + "-keyout", + keyPath, + "-out", + certPath, + ], + { stdio: "ignore" }, + ); + return { dir, cert: fs.readFileSync(certPath), key: fs.readFileSync(keyPath) }; +} + +/** + * A minimal OpenAI-compatible HTTPS endpoint for proving the HTTPS-pin + * runtime adapter's live routing (#6141): authenticated `/v1/models` and + * `/v1/chat/completions`, plus a request ledger so the test can assert on + * exactly what the pinned adapter forwarded (Host header, credential, + * method/path) once this server sits behind a real public tunnel. + */ +export async function startFakeHttpsCompatibleServer(options: { + apiKey: string; + model: string; + chatContent?: string; +}): Promise { + const tls = generateEphemeralTlsMaterial(); + const requests: FakeHttpsCompatibleRequest[] = []; + const chatContent = options.chatContent ?? "ok"; + + const server = https.createServer({ cert: tls.cert, key: tls.key }, async (req, res) => { + const requestPath = new URL(req.url ?? "/", "https://https-pin.local").pathname; + const body = req.method === "HEAD" ? "" : await readRequestBody(req); + const authHeader = req.headers.authorization ?? ""; + const auth: FakeHttpsCompatibleRequest["auth"] = + authHeader === `Bearer ${options.apiKey}` ? "ok" : authHeader ? "invalid" : "missing"; + + // The public quick-tunnel readiness probe issues an unauthenticated HEAD + // request. Keep it out of the request ledger so offset-based assertions + // in the test only ever measure real chat-completion traffic. + if (req.method !== "HEAD") { + requests.push({ + method: req.method ?? "", + path: requestPath, + hostHeader: req.headers.host, + auth, + body, + }); + } + + if (auth !== "ok") { + jsonResponse(res, 401, { error: { message: "missing bearer credential" } }); + return; + } + if ( + ["GET", "HEAD"].includes(req.method ?? "") && + ["/models", "/v1/models"].includes(requestPath) + ) { + jsonResponse(res, 200, { object: "list", data: [{ id: options.model, object: "model" }] }); + return; + } + if ( + req.method === "POST" && + ["/chat/completions", "/v1/chat/completions"].includes(requestPath) + ) { + jsonResponse(res, 200, { + id: "chatcmpl-https-pin", + object: "chat.completion", + created: 0, + model: options.model, + choices: [ + { index: 0, message: { role: "assistant", content: chatContent }, finish_reason: "stop" }, + ], + }); + return; + } + jsonResponse(res, 404, { error: { message: "not found" } }); + }); + + await listenOnRandomPort(server); + return { + port: requireTcpPort(server), + requests: () => requests, + close: async () => { + await closeServer(server); + fs.rmSync(tls.dir, { recursive: true, force: true }); + }, + }; +} diff --git a/test/e2e/live/inference-routing.test.ts b/test/e2e/live/inference-routing.test.ts index 048e75e4881..0d24265b501 100644 --- a/test/e2e/live/inference-routing.test.ts +++ b/test/e2e/live/inference-routing.test.ts @@ -5,11 +5,19 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN } from "../../../src/lib/inference/https-pin-runtime.ts"; +import { REGISTRY_FILE, type SandboxEntry } from "../../../src/lib/state/registry.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import { + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./dns-rebinding-hosts-fixture.ts"; +import { startFakeHttpsCompatibleServer } from "./https-pin-compatible-server.ts"; import { CREDENTIAL_CLASSIFICATION_PATTERN, cleanupSandbox, @@ -27,6 +35,7 @@ import { TRANSPORT_CLASSIFICATION_PATTERN, writeFakeOpenShellForBlueprintFailClosed, } from "./inference-routing-helpers.ts"; +import { startPublicMcpHttpsTunnel } from "./mcp-bridge-servers.ts"; // This is the PR-required inference-routing lane. Credential-backed provider // smokes live in inference-routing-provider-smoke.test.ts and are never selected @@ -322,3 +331,218 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere }), ); }); + +test("TC-INF-11 DNS-backed HTTPS custom endpoint routes through the local pinning adapter (#6141)", { + timeout: 20 * 60_000, +}, async ({ artifacts, cleanup, host, sandbox, skip }) => { + await requireLivePrerequisites(host, skip); + const model = "nemoclaw-e2e-https-pin"; + const apiKey = "sk-https-pin-TEST-NOT-A-REAL-VALUE"; + const sandboxName = inferenceSandboxName("e2e-https-pin"); + cleanup.add(`best-effort inference-routing https-pin cleanup for ${sandboxName}`, () => + cleanupSandbox(host, sandbox, sandboxName), + ); + await cleanupSandbox(host, sandbox, sandboxName); + + const fake = await startFakeHttpsCompatibleServer({ apiKey, chatContent: "PONG", model }); + cleanup.add("close https-pin fake HTTPS compatible server", async () => { + try { + await artifacts.writeJson("tc-inf-11-https-pin-endpoint-requests.json", fake.requests()); + } finally { + await fake.close(); + } + }); + + // A genuinely public, DNS-resolvable, publicly-trusted-certificate origin + // is required: the adapter's SSRF preflight rejects loopback/private + // addresses, and only a real TLS trust chain exercises its SNI-pinned + // certificate validation. This reuses the same trycloudflare.com quick + // tunnel mechanism as the MCP-bridge DNS-rebinding coverage. + const tunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "https-pin inference routing", + readinessPath: "/v1/models", + readinessStatus: 401, + server: fake, + }); + const endpointUrl = `${tunnel.origin}/v1`; + const endpointHostname = new URL(tunnel.origin).hostname; + + await artifacts.target.declare({ + id: "https-pin-runtime-adapter-dns-backed-endpoint", + issue: 6141, + contract: [ + "inference set routes a DNS-backed HTTPS endpoint through the local pinning adapter", + "the real upstream hostname is never persisted to the NemoClaw sandbox registry", + "OpenShell's own policy view never references the real upstream hostname", + "a real chat completion round-trips through the pinned TLS connection to the public endpoint", + "a DNS rebind of the upstream hostname after inference set does not redirect adapter traffic", + ], + endpointUrl, + model, + }); + + // Onboarding's own SSRF preflight (assertEndpointResolvesPublic) only + // rejects private/internal addresses; it does not fail closed on + // DNS-backed HTTPS the way the HTTPS Pin Runtime adapter's call site does, + // and onboarding never wires that adapter itself (only + // inference-set-route-containment.ts's normalizeCustomEndpointUrl does, on + // the `inference set --endpoint-url` path). Onboard with a disposable + // plain-HTTP placeholder endpoint first -- the same shape TC-INF-09 already + // onboards successfully with -- then switch to the DNS-backed HTTPS + // endpoint through `inference set --endpoint-url`, the actual #6141 call + // site this test exercises. + const placeholder = await startFakeOpenAiCompatibleServer({ + apiKey, + chatContent: "placeholder", + model, + publicHost: "localhost", + requireAuth: true, + requireAuthModels: true, + }); + cleanup.add("close https-pin onboarding placeholder endpoint", () => placeholder.close()); + + const onboard = await onboardSandbox( + artifacts, + sandboxName, + { + COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_ENDPOINT_URL: placeholder.baseUrl, + NEMOCLAW_MODEL: model, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }, + [apiKey], + "tc-inf-11-onboard-https-pin-placeholder", + 15 * 60_000, + ); + expectOnboardSuccess(onboard, "TC-INF-11 https-pin-endpoint placeholder onboard"); + cleanup.add(`strict inference-routing https-pin cleanup for ${sandboxName}`, () => + cleanupSandbox(host, sandbox, sandboxName, { strict: true }), + ); + + const inferenceSet = await runNemoclawCli( + [ + "inference", + "set", + "--provider", + "compatible-endpoint", + "--model", + model, + "--sandbox", + sandboxName, + "--endpoint-url", + endpointUrl, + "--credential-env", + "COMPATIBLE_API_KEY", + "--inference-api", + "openai-completions", + ], + { + artifactName: "tc-inf-11-inference-set-https-pin-endpoint", + artifacts, + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect( + inferenceSet.exitCode, + `TC-INF-11 inference set https-pin endpoint failed\n${redactedResultText(inferenceSet)}`, + ).toBe(0); + + // The real hostname must never reach the NemoClaw sandbox registry on + // disk: only the local adapter's host.openshell.internal route is + // persisted (#6141 requirement: hostname hidden from the runtime + // boundary; credential-bearing URL state is never persisted in plaintext). + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const registryEntry = registry.sandboxes?.[sandboxName]; + expect(registryEntry?.endpointUrl ?? "").toContain( + `${HTTPS_PIN_RUNTIME_ADAPTER_BASE_ORIGIN}/route/`, + ); + expect(registryEntry?.endpointUrl ?? "").not.toContain(endpointHostname); + + const provider = await sandbox.openshell( + ["provider", "get", "-g", "nemoclaw", "compatible-endpoint"], + { + artifactName: "tc-inf-11-provider-get-compatible-endpoint", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + const providerText = resultText(provider).replace(/\u001b\[[0-9;]*m/g, ""); + expect(provider.exitCode, providerText).toBe(0); + expect(providerText).toContain("Type: openai"); + expect(providerText).toContain("Credential keys: COMPATIBLE_API_KEY"); + expect(providerText).toContain("Config keys: OPENAI_BASE_URL"); + + // OpenShell's own network-policy view is a second, independent witness: + // it must never learn the real upstream hostname either, only the local + // adapter's host.openshell.internal boundary that everything else here + // already resolves through. + const policy = await sandbox.openshell(["policy", "get", "--full", sandboxName], { + artifactName: "tc-inf-11-policy-get-https-pin", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const policyText = resultText(policy).replace(/\u001b\[[0-9;]*m/g, ""); + expect(policy.exitCode, policyText).toBe(0); + expect(policyText).not.toContain(endpointHostname); + + const sandboxRequestOffset = fake.requests().length; + await expectOpenAiChatThroughSandbox( + sandbox, + sandboxName, + model, + [apiKey], + "https-pin-endpoint-inference-local-chat", + ); + expect(fake.requests().slice(sandboxRequestOffset)).toContainEqual( + expect.objectContaining({ + auth: "ok", + method: "POST", + path: "/v1/chat/completions", + }), + ); + + // The assertions above only prove the *initial* `inference set` reached + // the real target. They do not prove the adapter is resistant to a DNS + // record changing after the route is already pinned -- the exact + // SSRF/DNS-rebinding vulnerability the pinning mechanism exists to close. + // Rebind the tunnel hostname to a reserved, unreachable documentation + // address (RFC 5737 TEST-NET-1) now that the route is registered: if the + // adapter re-resolved DNS per request instead of using the addresses it + // already pinned, this chat call would fail to connect instead of + // succeeding. + const hostsFixture = await setupDnsRebindingHostsFixture(host, sandboxName, endpointHostname); + cleanup.add(`restore https-pin DNS rebinding hosts fixture for ${sandboxName}`, () => + restoreDnsRebindingHostsFixture(host, sandboxName, hostsFixture), + ); + await remapDnsRebindingHostname( + host, + sandboxName, + hostsFixture, + "192.0.2.1", + "tc-inf-11-dns-rebind-after-inference-set", + ); + + const rebindRequestOffset = fake.requests().length; + await expectOpenAiChatThroughSandbox( + sandbox, + sandboxName, + model, + [apiKey], + "https-pin-endpoint-dns-rebinding-chat", + ); + expect(fake.requests().slice(rebindRequestOffset)).toContainEqual( + expect.objectContaining({ + auth: "ok", + method: "POST", + path: "/v1/chat/completions", + }), + ); + + await restoreDnsRebindingHostsFixture(host, sandboxName, hostsFixture); +}); diff --git a/test/e2e/live/mcp-bridge-servers.ts b/test/e2e/live/mcp-bridge-servers.ts index 2320a330c8f..6736079cf21 100644 --- a/test/e2e/live/mcp-bridge-servers.ts +++ b/test/e2e/live/mcp-bridge-servers.ts @@ -190,37 +190,53 @@ export function buildCloudflaredQuickTunnelArgs(port: number): string[] { ]; } -async function probePublicTunnel(origin: string): Promise<{ +async function probePublicTunnel( + origin: string, + readinessPath: string, + readinessStatus: number, +): Promise<{ ready: boolean; diagnostic: string; }> { try { - const response = await fetch(`${origin}/mcp`, { + const response = await fetch(`${origin}${readinessPath}`, { method: "HEAD", redirect: "manual", signal: AbortSignal.timeout(5_000), }); await response.body?.cancel(); return { - ready: response.status === 405, - diagnostic: `public HEAD /mcp returned HTTP ${response.status}`, + ready: response.status === readinessStatus, + diagnostic: `public HEAD ${readinessPath} returned HTTP ${response.status}`, }; } catch (error) { return { ready: false, // Avoid reflecting request URLs or child output here. The error class is // enough to distinguish DNS/transport failure without risking headers. - diagnostic: `public HEAD /mcp failed (${error instanceof Error ? error.name : "unknown error"})`, + diagnostic: `public HEAD ${readinessPath} failed (${error instanceof Error ? error.name : "unknown error"})`, }; } } +/** + * Publishes a local HTTPS origin behind a real `trycloudflare.com` quick + * tunnel: a genuinely public, DNS-resolvable, publicly-trusted-certificate + * endpoint. Named for its original MCP-bridge fixture caller; reused as-is + * (via the optional readiness override below) for the HTTPS-pin runtime + * adapter's live coverage, since both need the identical real-tunnel proof + * and only differ in which local path/status means "ready". + */ export async function startPublicMcpHttpsTunnel(options: { cleanup: TunnelCleanupRegistry; label: string; server: StartedHttpServer; cloudflaredBin?: string; + readinessPath?: string; + readinessStatus?: number; }): Promise { + const readinessPath = options.readinessPath ?? "/mcp"; + const readinessStatus = options.readinessStatus ?? 405; const args = buildCloudflaredQuickTunnelArgs(options.server.port); let lastFailure = "cloudflared did not publish a quick-tunnel URL"; @@ -269,7 +285,7 @@ export async function startPublicMcpHttpsTunnel(options: { break; } if (origin) { - const probe = await probePublicTunnel(origin); + const probe = await probePublicTunnel(origin, readinessPath, readinessStatus); if (probe.ready) { consecutiveReadyProbes += 1; if (consecutiveReadyProbes >= QUICK_TUNNEL_CONSECUTIVE_READY_PROBES) {