diff --git a/docs/inference/custom-endpoint-security.mdx b/docs/inference/custom-endpoint-security.mdx index 2768e64e55e..8fdc60e3733 100644 --- a/docs/inference/custom-endpoint-security.mdx +++ b/docs/inference/custom-endpoint-security.mdx @@ -55,7 +55,9 @@ For those paths, use an HTTPS IP-literal endpoint with a certificate valid for t `$$nemoclaw inference set --endpoint-url ` on an already-onboarded sandbox supports a DNS-backed HTTPS custom endpoint through the HTTPS Pin Runtime adapter. After SSRF validation passes, NemoClaw starts a local reverse-proxy adapter on the host that terminates a pinned, SNI-correct outbound TLS connection to the real upstream hostname, re-validating that the resolved peer IP is still public. The sandbox, its OpenShell provider configuration and network policy, and the persisted sandbox registry only ever see the opaque local base `http://host.openshell.internal:/route/`. -The real upstream hostname and path never reach the sandbox or the persisted registry; host recovery state stores only the opaque route ID, provider type, a non-secret token generation value, and timestamps. +The real upstream hostname and path never reach the sandbox or the persisted registry. +Host recovery state contains adapter process metadata, the source-subnet policy, opaque route IDs, provider types, non-secret token generation values, and timestamps. +It contains no upstream URLs, pinned addresses, or credentials. Endpoint URLs containing userinfo, a query string, or a fragment are rejected rather than stripped or persisted. For an OpenAI-compatible endpoint entered as a bare origin, the adapter preserves the incoming `/v1` request path. For an endpoint with a path prefix, the adapter keeps forwarded requests beneath that prefix and rejects traversal-shaped paths. @@ -71,6 +73,10 @@ NemoClaw refuses to expose the adapter when it cannot discover a valid bridge su Adapter reuse also requires an authenticated health proof for the same source-subnet policy, so a running process with a stale or different policy is replaced. After an adapter restart, routes other than the one that triggered recovery return a recovery-needed response until their original `inference set --endpoint-url` command is rerun. Switching away from a route or destroying its last sandbox reference revokes it; a scoped uninstall that leaves sibling gateways in place preserves the shared adapter and its remaining routes. +Revocation has to authenticate the running adapter first, and the control-plane proof binds the source-subnet policy that adapter was started with, so NemoClaw records that policy in host recovery state when it starts or reuses the adapter and reads it back at revocation time. +It never re-derives the policy from the current host: a bridge that has since been recreated or renumbered would produce a value the running adapter never used. +An adapter started before NemoClaw recorded that policy cannot be authenticated for revocation, so the superseded route stays registered and `inference set` reports that its upstream credentials are still resident. +Rerun the `inference set --endpoint-url` command that registered the route: that records the policy for the running adapter, and the next switch revokes cleanly. This support is specific to `inference set` on an already-onboarded sandbox. Hermes Provider setup, host-side `config set`, and a direct blueprint run still reject DNS-backed HTTPS URLs as described above. diff --git a/src/lib/inference/https-pin-runtime-adapter.test.ts b/src/lib/inference/https-pin-runtime-adapter.test.ts index 1cb2dc54e53..82c23f29781 100644 --- a/src/lib/inference/https-pin-runtime-adapter.test.ts +++ b/src/lib/inference/https-pin-runtime-adapter.test.ts @@ -1125,6 +1125,7 @@ describe("adapter recovery lock (#6141)", () => { lockModule.__test.revokeRouteLocked("a".repeat(64), { loadPid: () => null, readControlToken: () => "persisted-control-token", + readAllowedSourceCidrs: () => ["172.18.0.0/16"], probeHealth: async () => true, deleteRoute, isAdapterProcess: () => false, @@ -1145,6 +1146,7 @@ describe("adapter recovery lock (#6141)", () => { lockModule.__test.revokeRouteLocked("a".repeat(64), { loadPid: () => null, readControlToken: () => "persisted-control-token", + readAllowedSourceCidrs: () => ["172.18.0.0/16"], probeHealth: async () => true, deleteRoute: async () => { throw new Error("delete failed"); diff --git a/src/lib/inference/https-pin-runtime-adapter.ts b/src/lib/inference/https-pin-runtime-adapter.ts index 00e531bfdfe..b8920ba171b 100644 --- a/src/lib/inference/https-pin-runtime-adapter.ts +++ b/src/lib/inference/https-pin-runtime-adapter.ts @@ -1135,12 +1135,45 @@ function extractPersistedRoutes(prior: JsonObject | null): Record typeof entry === "string" && entry.trim())) return null; + try { + const canonical = buildAllowedRouteSourceMatcher(raw as string[]).cidrs; + return canonical.length > 0 ? canonical : null; + } catch { + return null; + } +} + +function persistRouteState( + routeId: string, + meta: RoutePersistedMeta, + allowedSourceCidrs?: readonly string[], +): void { const prior = readLocalAdapterJsonFile(STATE_PATH); const priorRoutes = extractPersistedRoutes(prior); writeLocalAdapterJsonFile(STATE_PATH, { pid: (prior?.pid as number | null | undefined) ?? loadPersistedPid(), updatedAt: new Date().toISOString(), + // Record the policy this registration proved the live adapter is running + // under, so a later revocation can authenticate it without re-discovery. + allowedSourceCidrs: allowedSourceCidrs + ? [...allowedSourceCidrs] + : extractPersistedAllowedSourceCidrs(prior), // Re-registering a route (fresh `meta`, no `orphanedAt`) always // supersedes any prior orphaned entry for the same id -- this is how a // route heals after its owner re-runs `inference set` post-recovery. @@ -1155,6 +1188,9 @@ function removeRouteState(routeId: string): void { writeLocalAdapterJsonFile(STATE_PATH, { pid: (prior?.pid as number | null | undefined) ?? loadPersistedPid(), updatedAt: new Date().toISOString(), + // The adapter keeps running with the same policy after one of its routes + // is dropped, so carry the record forward instead of erasing it. + allowedSourceCidrs: extractPersistedAllowedSourceCidrs(prior), routes, }); } @@ -1298,11 +1334,18 @@ export async function ensureHttpsPinRuntimeAdapter(options: { credentialValue: options.credentialValue, generation, }); - persistRouteState(routeId, { - providerType: options.providerType, - generation, - registeredAt: new Date().toISOString(), - }); + // `ensureAdapterProcessLocked` authenticated the live adapter against this + // policy just above -- on the reuse path that is the only proof of what it + // is running under, so record it here too and not just at spawn (#7878). + persistRouteState( + routeId, + { + providerType: options.providerType, + generation, + registeredAt: new Date().toISOString(), + }, + allowedSourceCidrs, + ); // Only this route-scoped value leaves the host lifecycle boundary. The // control token stays in its 0600 host state file and the adapter process @@ -1338,13 +1381,19 @@ async function revokeRouteLocked( deps: { loadPid: () => number | null; readControlToken: () => string | null; - probeHealth: (options: { controlToken: string }) => Promise; + readAllowedSourceCidrs: () => readonly string[] | null; + probeHealth: (options: { + controlToken: string; + expectedSourceCidrs?: readonly string[]; + }) => Promise; deleteRoute: (controlToken: string, candidateRouteId: string) => Promise; isAdapterProcess: (pid: number | null) => boolean; removeRouteState: (candidateRouteId: string) => void; } = { loadPid: loadPersistedPid, readControlToken: () => readLocalAdapterTextFile(TOKEN_PATH), + readAllowedSourceCidrs: () => + extractPersistedAllowedSourceCidrs(readLocalAdapterJsonFile(STATE_PATH)), probeHealth: (options) => probeAdapterControlHealth(options), deleteRoute, isAdapterProcess, @@ -1353,13 +1402,38 @@ async function revokeRouteLocked( ): Promise { const pid = deps.loadPid(); const controlToken = deps.readControlToken(); + // The control-plane challenge proof binds the adapter's route-source policy, + // so probing without it silently authenticates against the `127.0.0.1/32` + // default while every sandbox-facing adapter runs on the OpenShell bridge + // range. That mismatch is why a superseded route could not be revoked and + // its upstream credentials stayed resident (#7878). Use the policy recorded + // when the adapter was started -- never one re-derived here, which stops + // matching as soon as the bridge is recreated or renumbered. + const allowedSourceCidrs = deps.readAllowedSourceCidrs(); const authenticatedLiveAdapter = Boolean( - controlToken && (await deps.probeHealth({ controlToken: controlToken as string })), + controlToken && + allowedSourceCidrs && + (await deps.probeHealth({ + controlToken: controlToken as string, + expectedSourceCidrs: allowedSourceCidrs, + })), ); if (authenticatedLiveAdapter && controlToken) { await deps.deleteRoute(controlToken, routeId); - } else if (deps.isAdapterProcess(pid)) { - throw new Error("Cannot authenticate the live HTTPS Pin Runtime adapter for revocation."); + } else if (controlToken || deps.isAdapterProcess(pid)) { + // Fail closed. An adapter started before this record existed cannot be + // authenticated for revocation, and falling back to a re-derived or + // default policy would authenticate it against a value this process just + // made up. A missing PID record does not prove that the adapter stopped; + // preserve the route state while either identity artifact remains. Its + // next route registration records the policy and heals it. + throw new Error( + allowedSourceCidrs + ? "Cannot authenticate the live HTTPS Pin Runtime adapter for revocation." + : "Cannot authenticate the live HTTPS Pin Runtime adapter for revocation: " + + "its route-source policy was not recorded. Re-run the inference set that " + + "registered this route to record it, then retry.", + ); } deps.removeRouteState(routeId); return true; @@ -1530,6 +1604,10 @@ async function ensureAdapterProcessLocked(options: { writeLocalAdapterJsonFile(STATE_PATH, { pid: child.pid ?? null, updatedAt: new Date().toISOString(), + // Provenance for the control-plane challenge proof: this is the policy + // the child was actually spawned with, which revocation must reuse + // rather than re-derive (#7878). + allowedSourceCidrs: [...options.allowedSourceCidrs], routes: persistedRoutes, }); } catch (err) { @@ -1556,6 +1634,7 @@ export const __test = { computeRespawnState, buildAllowedRouteSourceMatcher, discoverOpenShellBridgeSourceCidrs, + extractPersistedAllowedSourceCidrs, findReusableAdapterControlToken, revokeRouteLocked, LOCK_PATH, diff --git a/src/lib/inference/https-pin-runtime-revoke-source-policy.test.ts b/src/lib/inference/https-pin-runtime-revoke-source-policy.test.ts new file mode 100644 index 00000000000..732970af230 --- /dev/null +++ b/src/lib/inference/https-pin-runtime-revoke-source-policy.test.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Revocation has to authenticate the running adapter, and the control-plane +// proof binds the source-subnet policy that adapter was started with. Probing +// without it authenticates against the loopback default while every +// sandbox-facing adapter runs on the OpenShell bridge range, which is why a +// superseded route survived an endpoint update with its upstream credentials +// resident (#7878). + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { __test } from "./https-pin-runtime-adapter"; + +describe("HTTPS Pin Runtime revocation source policy", () => { + let lockModule: typeof import("./https-pin-runtime-adapter"); + + beforeEach(async () => { + vi.resetModules(); + lockModule = await import("./https-pin-runtime-adapter"); + }); + + it("authenticates revocation with the recorded route-source policy (#7878)", async () => { + const deleteRoute = vi.fn(async () => {}); + const removeRouteState = vi.fn(); + const probeHealth = vi.fn(async () => true); + + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => 4242, + readControlToken: () => "persisted-control-token", + readAllowedSourceCidrs: () => ["172.18.0.0/16"], + probeHealth, + deleteRoute, + isAdapterProcess: () => true, + removeRouteState, + }), + ).resolves.toBe(true); + // The proof binds the policy digest, so the probe must carry the recorded + // policy rather than falling back to the 127.0.0.1/32 default. + expect(probeHealth).toHaveBeenCalledWith({ + controlToken: "persisted-control-token", + expectedSourceCidrs: ["172.18.0.0/16"], + }); + expect(deleteRoute).toHaveBeenCalledWith("persisted-control-token", "a".repeat(64)); + }); + + it("preserves route state when a persisted token means an adapter may still hold the credential (#7878)", async () => { + const deleteRoute = vi.fn(async () => {}); + const removeRouteState = vi.fn(); + const probeHealth = vi.fn(async () => true); + + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => null, + readControlToken: () => "persisted-control-token", + readAllowedSourceCidrs: () => null, + probeHealth, + deleteRoute, + isAdapterProcess: () => false, + removeRouteState, + }), + ).rejects.toThrow("route-source policy was not recorded"); + // The token means the adapter may still hold the credential. Do not probe + // with a made-up policy or drop its route record; the caller must warn. + expect(probeHealth).not.toHaveBeenCalled(); + expect(deleteRoute).not.toHaveBeenCalled(); + expect(removeRouteState).not.toHaveBeenCalled(); + }); + + it("keeps clearing state for an absent adapter without a recorded policy (#7878)", async () => { + const deleteRoute = vi.fn(async () => {}); + const removeRouteState = vi.fn(); + + // Regression lock: no live adapter means nothing holds the credential, so + // the pre-existing "drop the stale record" behaviour must not change. + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => null, + readControlToken: () => null, + readAllowedSourceCidrs: () => null, + probeHealth: async () => false, + deleteRoute, + isAdapterProcess: () => false, + removeRouteState, + }), + ).resolves.toBe(true); + expect(deleteRoute).not.toHaveBeenCalled(); + expect(removeRouteState).toHaveBeenCalledWith("a".repeat(64)); + }); + + it("still reports the authentication failure when the recorded policy no longer matches (#7878)", async () => { + const removeRouteState = vi.fn(); + + await expect( + lockModule.__test.revokeRouteLocked("a".repeat(64), { + loadPid: () => 4242, + readControlToken: () => "persisted-control-token", + readAllowedSourceCidrs: () => ["172.18.0.0/16"], + probeHealth: async () => false, + deleteRoute: async () => {}, + isAdapterProcess: () => true, + removeRouteState, + }), + ).rejects.toThrow("Cannot authenticate the live HTTPS Pin Runtime adapter for revocation."); + expect(removeRouteState).not.toHaveBeenCalled(); + }); + + it.each([ + ["absent", undefined], + ["empty", []], + ["non-array", "172.18.0.0/16"], + ["non-string entries", [17218]], + ["blank entries", [" "]], + ])("treats a %s recorded route-source policy as unusable (#7878)", (_label, recorded) => { + expect( + __test.extractPersistedAllowedSourceCidrs( + recorded === undefined ? {} : { allowedSourceCidrs: recorded }, + ), + ).toBeNull(); + }); + + it("canonicalises a recorded route-source policy before use (#7878)", () => { + expect( + __test.extractPersistedAllowedSourceCidrs({ allowedSourceCidrs: ["172.18.0.0/16"] }), + ).toEqual(__test.buildAllowedRouteSourceMatcher(["172.18.0.0/16"]).cidrs); + }); +});