Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
7485a8d
feat(inference): add route-scoped HTTPS pinning transport
apurvvkumaria Jul 19, 2026
20d7d80
test(inference): keep HTTPS pin regressions linear
apurvvkumaria Jul 19, 2026
67b6a25
fix(inference): address HTTPS pin review findings
apurvvkumaria Jul 19, 2026
68a4b63
ci(e2e): provision pinned routing tunnel
apurvvkumaria Jul 19, 2026
5625f90
test(e2e): cover routing tunnel digest pin
apurvvkumaria Jul 19, 2026
6832280
fix(e2e): bootstrap pinned routing tunnel binary
apurvvkumaria Jul 19, 2026
1b137c5
fix(e2e): expose HTTPS pin placeholder to sandbox
apurvvkumaria Jul 19, 2026
0eda93d
test(e2e): pin HTTPS route placeholder port
apurvvkumaria Jul 19, 2026
e4b93da
test(e2e): wait for pinned route refresh
apurvvkumaria Jul 19, 2026
ac47c21
test(e2e): keep route polling linear
apurvvkumaria Jul 19, 2026
0f65fb8
docs(inference): define orphan recovery exit
apurvvkumaria Jul 19, 2026
2059f0b
merge(main): reconcile route cleanup updates
apurvvkumaria Jul 19, 2026
10b9f60
fix(inference): reject stale pin adapters
apurvvkumaria Jul 19, 2026
ba42c5b
fix(inference): translate pinned route API paths
apurvvkumaria Jul 19, 2026
26a70da
merge: sync scoped HTTPS routing with main
apurvvkumaria Jul 25, 2026
918e1d2
merge(main): sync PR #7188 with current main
apurvvkumaria Jul 25, 2026
b9c66f1
docs(inference): document HTTPS pin port conflicts
apurvvkumaria Jul 25, 2026
1deb665
fix(inference): defer provider binding updates
apurvvkumaria Jul 25, 2026
9eba53f
test(e2e): verify cloudflared prerequisite
apurvvkumaria Jul 25, 2026
6eba0ac
fix(inference): restrict adapter bridge sources
apurvvkumaria Jul 25, 2026
62ad5b5
docs(inference): document adapter source boundary
apurvvkumaria Jul 25, 2026
4b2f944
fix(inference): bound pinned upstream lifecycle
apurvvkumaria Jul 25, 2026
7e6703e
fix(inference): preserve bare-origin gateway paths
apurvvkumaria Jul 25, 2026
3f431c5
fix(inference): recognize persisted adapter ports
apurvvkumaria Jul 25, 2026
a3cc3d7
fix(inference): restrict adapter health metadata
apurvvkumaria Jul 25, 2026
da41cc6
fix(inference): keep route secrets off child env
apurvvkumaria Jul 25, 2026
fbbacd8
docs(inference): record adapter lifecycle guards
apurvvkumaria Jul 25, 2026
2d40456
test(e2e): prove credential URL state rejection
apurvvkumaria Jul 25, 2026
54954d4
ci(e2e): describe HTTPS pin coverage
apurvvkumaria Jul 25, 2026
0fc5cd9
Merge branch 'main' into codex/6141-route-scoped-credentials
senthilr-nv Jul 25, 2026
821969b
fix(inference): restore selection after provider failure
apurvvkumaria Jul 25, 2026
5d63951
Merge remote-tracking branch 'origin/main' into codex/pr7188-current-…
apurvvkumaria Jul 25, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 28 additions & 5 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1288,12 +1288,35 @@ jobs:
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Install and verify cloudflared prerequisite
# Keep the public HTTPS routing fixture on the same reviewed binary as
# the MCP and tunnel-lifecycle lanes. The checksum and package metadata
# checks prevent a mutable package source from entering PR-safe E2E.
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"

- name: Run inference routing live test
# Direct E2E coverage. The always-on PR-safe slices prove invalid-key,
# unreachable-endpoint, and localhost-compatible gateway routing
# without spending live provider quota. Credential-backed isolation
# and provider smokes live in inference-routing-provider-smoke.test.ts;
# any future secret-bearing lane must run that file from trusted main.
# Direct PR-safe E2E coverage proves invalid-key, unreachable-endpoint,
# localhost-compatible routing, namespace-aware HTTPS pinning, DNS
# rebinding resistance, private-target redirect rejection, and
# credential-bearing URL state rejection without live provider quota.
# Provider smokes live in inference-routing-provider-smoke.test.ts; any
# future secret-bearing lane must run that file from trusted main.
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/inference-routing.test.ts
Expand Down
16 changes: 16 additions & 0 deletions ci/env-var-doc-allowlist.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,22 @@
"name": "NEMOCLAW_BEDROCK_RUNTIME_REGION",
"reason": "Internal child-process setting used only to pass the resolved Bedrock Runtime region to the hidden local adapter. Users should rely on the endpoint URL or standard AWS region environment variables."
},
{
"name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT",
"reason": "Internal child-process setting used only when launching the hidden HTTPS Pin Runtime adapter. The port is a fixed internal constant, not a public user-facing configuration knob."
},
{
"name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_CONTROL_TOKEN",
"reason": "Internal host-only child-process secret used to authenticate HTTPS Pin Runtime adapter control-plane calls. It is generated by NemoClaw, stored in a private local state file, and never registered with OpenShell or supplied by users."
},
{
"name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ALLOWED_SOURCE_CIDRS",
"reason": "Internal child-process setting carrying only the JSON-encoded OpenShell Docker IPAM subnets allowed to reach the hidden HTTPS Pin Runtime adapter. It contains no endpoint or credential data and is never user-set."
},
{
"name": "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_ORPHANED_ROUTES",
"reason": "Internal child-process setting carrying a JSON-encoded map of opaque route ids to provider types and non-secret token generations (no URLs or credentials) that a fresh HTTPS Pin Runtime adapter respawn could not recover, so it can authenticate and respond to them distinctly from a route that never existed. Never user-set."
},
{
"name": "NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE",
"reason": "Internal installer sentinel exported only during OpenShell gateway replacement so onboard restores the pre-upgrade sandbox backup. Not user-facing."
Expand Down
35 changes: 31 additions & 4 deletions docs/inference/custom-endpoint-security.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ keywords: ["nemoclaw endpoint security", "inference endpoint ssrf", "custom endp
content:
type: "concept"
---

NemoClaw keeps provider credentials on the host and validates explicit custom endpoint URLs before saving them through security-sensitive configuration paths.

## Protect Provider Credentials
Expand Down Expand Up @@ -44,9 +43,37 @@ Legacy entries without a source, endpoints recorded by `inference set`, and diff

For a public HTTP URL, NemoClaw stores the validated IP address so the downstream runtime cannot resolve the hostname again and reach another address.

NemoClaw rejects DNS-backed HTTPS URLs in these paths because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary.
Use an HTTPS IP-literal endpoint with a certificate valid for that address.
Use a public HTTP endpoint only when your deployment permits unencrypted traffic.
NemoClaw rejects DNS-backed HTTPS URLs for Hermes Provider setup, host-side `config set`, and a direct blueprint run because it cannot pin the downstream peer address while preserving TLS Server Name Indication and host validation across the OpenShell runtime boundary.
For those paths, use an HTTPS IP-literal endpoint with a certificate valid for that address, or use a public HTTP endpoint if your deployment permits unencrypted traffic.

<AgentOnly variant="openclaw,hermes">

## Use a DNS-Backed HTTPS Endpoint with `inference set`

`$$nemoclaw inference set --endpoint-url <url>` on an already-onboarded sandbox supports a DNS-backed HTTPS custom endpoint through the HTTPS Pin Runtime adapter.
After SSRF validation passes, NemoClaw starts a local reverse-proxy adapter on the host that terminates a pinned, SNI-correct outbound TLS connection to the real upstream hostname, re-validating that the resolved peer IP is still public.
The sandbox, its OpenShell provider configuration and network policy, and the persisted sandbox registry only ever see the opaque local base `http://host.openshell.internal:<port>/route/<id>`.
The real upstream hostname and path never reach the sandbox or the persisted registry; host recovery state stores only the opaque route ID, provider type, a non-secret token generation value, and timestamps.
Endpoint URLs containing userinfo, a query string, or a fragment are rejected rather than stripped or persisted.
For an OpenAI-compatible endpoint entered as a bare origin, the adapter preserves the incoming `/v1` request path.
For an endpoint with a path prefix, the adapter keeps forwarded requests beneath that prefix and rejects traversal-shaped paths.
One 30-second total upstream deadline covers connection setup, TLS negotiation, and the complete response; the adapter closes a response that arrives after the deadline instead of relaying it.

Each opaque route has its own sandbox-facing adapter credential, distinct from both the real upstream credential and the host-only control credential; a credential issued for one route cannot authorize another route.
NemoClaw does not place upstream route credentials in the adapter child-process environment.
After startup, the host CLI registers each route and its credential in adapter memory through an authenticated loopback-only control plane.
Before starting or reusing the adapter, NemoClaw inspects the exact IPAM subnets assigned to the `openshell-docker` network.
The adapter accepts route-forwarding and non-control health requests only from loopback or those inspected subnets and returns a not-found response to peers on other private or LAN networks.
Authenticated control health and route-registration requests remain loopback-only.
NemoClaw refuses to expose the adapter when it cannot discover a valid bridge subnet.
Adapter reuse also requires an authenticated health proof for the same source-subnet policy, so a running process with a stale or different policy is replaced.
After an adapter restart, routes other than the one that triggered recovery return a recovery-needed response until their original `inference set --endpoint-url` command is rerun.
Switching away from a route or destroying its last sandbox reference revokes it; a scoped uninstall that leaves sibling gateways in place preserves the shared adapter and its remaining routes.

This support is specific to `inference set` on an already-onboarded sandbox.
Hermes Provider setup, host-side `config set`, and a direct blueprint run still reject DNS-backed HTTPS URLs as described above.

</AgentOnly>

## Use the Sandbox Host Alias

Expand Down
5 changes: 5 additions & 0 deletions docs/inference/switch-providers.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,11 @@ An explicit different API family is rejected for that route.
To point a sandbox at a different custom endpoint, re-run onboarding with the new endpoint.
A rebuild reuses the recorded endpoint and cannot change it.

If updating an existing compatible provider fails after OpenShell selects the new route, NemoClaw attempts to restore the previously recorded provider and model.
The command still exits nonzero because the provider binding might be partially updated.
Retry the switch or re-run onboarding to reconcile the provider.
If NemoClaw reports that it could not restore the previous selection, do not use the route until you re-run onboarding.

</AgentOnly>

## Account for Shared Gateways
Expand Down
7 changes: 4 additions & 3 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3091,7 +3091,8 @@ NemoClaw rejects loopback, link-local, private, and internal endpoint addresses,
For a same-provider model change, pass `--endpoint-url` with the exact canonical endpoint URL that the target sandbox registry identifies as onboarding-established.
Missing or `inference set` provenance and every different URL remain subject to the full address validation above.
For public HTTP URLs, NemoClaw stores the validated IP address to prevent DNS rebinding.
DNS-backed HTTPS URLs are rejected because NemoClaw cannot pin the downstream peer address while preserving TLS SNI and host validation across the OpenShell runtime boundary; HTTPS IP-literal URLs remain supported.
For a DNS-backed HTTPS URL, NemoClaw routes the endpoint through a local HTTPS Pin Runtime adapter that terminates a pinned, SNI-correct outbound connection to the real upstream hostname; the sandbox and the persisted registry only ever see a local `host.openshell.internal` route, never the real hostname.
HTTPS IP-literal URLs remain supported and do not need the adapter.
NemoClaw accepts `http://host.openshell.internal:<port>` only with an explicit port from `1024` through `65535`; this narrow exception supports NemoClaw's sandbox-to-host inference routes and is not a general private-endpoint bypass.
`--credential-env` may also be supplied for compatible provider metadata; supported `--inference-api` values are `openai-completions`, `anthropic-messages`, and `openai-responses`.

Expand Down Expand Up @@ -3402,8 +3403,8 @@ Passthrough commands do not consume flags intended for the downstream command as
</AgentOnly>

If a port value is not a valid integer or falls outside the allowed range, the CLI exits with an error.
`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, or OpenRouter runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `11434`, `11435`, and `11437`.
When you select OpenRouter, `NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT` must also be distinct from the gateway, vLLM, Ollama, and Ollama proxy ports.
`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, OpenRouter runtime adapter, or HTTPS Pin Runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `11434`, `11435`, `11437`, and `11438`.
When you select OpenRouter, `NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT` must also be distinct from the gateway, vLLM, Ollama, Ollama proxy, and HTTPS Pin Runtime adapter ports.
When you run multiple NemoClaw gateways with different `NEMOCLAW_GATEWAY_PORT` values, NemoClaw derives a separate gateway name, state directory, and compatibility container name from the port so one gateway does not tear down another.
Only port `8080` uses a NemoClaw-managed Linux systemd user service or macOS Homebrew service.
NemoClaw-managed gateways on custom ports run as detached processes and do not change the default gateway service.
Expand Down
3 changes: 2 additions & 1 deletion docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1994,7 +1994,8 @@ After the upgrade, recreate the sandbox with `$$nemoclaw onboard`.
NemoClaw rejects an explicit custom endpoint when it resolves a public HTTPS hostname but cannot pin the same peer address across the downstream OpenShell runtime boundary while preserving TLS SNI and host validation.
This can appear during a direct blueprint run, custom-endpoint onboarding, or a host-side `config set` write.
<AgentOnly variant="openclaw,hermes">
It can also appear during a runtime `$$nemoclaw inference set` switch.
It does not appear during a runtime `$$nemoclaw inference set` switch on an already-onboarded sandbox; that command routes a DNS-backed HTTPS endpoint through a local HTTPS Pin Runtime adapter instead of rejecting it.
Refer to [Commands](commands) for details.
</AgentOnly>

Use an HTTPS IP-literal endpoint whose certificate is valid for that address.
Expand Down
11 changes: 6 additions & 5 deletions nemoclaw/src/blueprint/ssrf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,12 @@ export async function validateEndpointUrl(url: string): Promise<ValidatedEndpoin
export function safeEndpointUrlForDownstream(validated: ValidatedEndpoint): string {
if (validated.protocol === "https:" && validated.dnsResolved) {
throw new Error(
`DNS-backed HTTPS endpoint '${validated.hostname}' is not supported yet because ` +
"NemoClaw cannot guarantee the downstream provider connects to the same IP " +
"that passed SSRF validation across the OpenShell runtime boundary. " +
"Use an HTTPS IP-literal endpoint, an HTTP endpoint that can be DNS-pinned, " +
"or wait for the runtime-aware HTTPS pinning transport.",
`DNS-backed HTTPS endpoint '${validated.hostname}' is not supported at onboarding ` +
"time because this path cannot guarantee the downstream provider connects to the " +
"same IP that passed SSRF validation across the OpenShell runtime boundary. " +
"Use an HTTPS IP-literal endpoint or an HTTP endpoint that can be DNS-pinned to " +
"onboard. After onboarding, `inference set --endpoint-url` supports a DNS-backed " +
"HTTPS endpoint through the HTTPS Pin Runtime adapter.",
);
}

Expand Down
18 changes: 9 additions & 9 deletions scripts/checks/check-cloudflared-update.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# invalidState: the three reviewed E2E consumers drift to different cloudflared
# invalidState: the four reviewed E2E consumers drift to different cloudflared
# versions/digests, or their shared pin no longer matches the upstream asset.
# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all three
# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all four
# workflow pins and independently verifies the downloaded bytes.
# whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use.
# regressionTest: cloudflared-update-check-workflow.test.ts covers three-pin
# regressionTest: cloudflared-update-check-workflow.test.ts covers four-pin
# parity, asset URL identity, digest mismatch, and update instructions.
# removalCondition: remove this checker when the three consumers share one
# removalCondition: remove this checker when the four consumers share one
# machine-readable dependency manifest with equivalent live asset verification.

set -euo pipefail
Expand Down Expand Up @@ -48,10 +48,10 @@ done < <(
"${E2E_WORKFLOW}"
)

[[ "${#version_pins[@]}" -eq 3 ]] \
|| fail "expected exactly three CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}"
[[ "${#sha_pins[@]}" -eq 3 ]] \
|| fail "expected exactly three CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}"
[[ "${#version_pins[@]}" -eq 4 ]] \
|| fail "expected exactly four CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}"
[[ "${#sha_pins[@]}" -eq 4 ]] \
|| fail "expected exactly four CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}"

pinned_version="${version_pins[0]}"
pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')"
Expand Down Expand Up @@ -127,7 +127,7 @@ print_update_instructions() {
'Update locations:' \
" ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \
" ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \
'Set all three version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2
'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2
}

if [[ "${latest_version}" != "${pinned_version}" ]]; then
Expand Down
Loading
Loading