diff --git a/.github/workflows/portable-profile-e2e.yaml b/.github/workflows/portable-profile-e2e.yaml
index 86d6667d3c..d9ec9eb2ec 100644
--- a/.github/workflows/portable-profile-e2e.yaml
+++ b/.github/workflows/portable-profile-e2e.yaml
@@ -3,10 +3,19 @@
name: E2E / Portable Profile
-run-name: "Portable profile rootless E2E for ${{ github.sha }}"
+run-name: "Portable profile rootless E2E for ${{ github.event.pull_request.head.sha || github.sha }}"
on:
workflow_dispatch:
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths:
+ - ".github/workflows/portable-profile-e2e.yaml"
+ - "src/lib/onboard/experimental/portable-host-preparation.ts"
+ - "src/lib/onboard/experimental/portable-profile.ts"
+ - "src/lib/onboard/experimental/portable-retired-subnet-recovery.test.ts"
+ - "test/e2e/live/portable-profile-rootless-linux.test.ts"
+ - "test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts"
push:
branches:
- main
@@ -32,7 +41,7 @@ permissions:
contents: read
concurrency:
- group: portable-profile-e2e-${{ github.ref }}
+ group: portable-profile-e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
@@ -40,12 +49,14 @@ jobs:
runs-on: ubuntu-26.04
timeout-minutes: 25
env:
+ E2E_SOURCE_REVISION: ${{ github.event.pull_request.head.sha || github.sha }}
PODMAN_APT_VERSION: "5.7.0+ds2-3build1"
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
@@ -125,7 +136,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: portable-profile-e2e-artifacts
+ name: portable-profile-e2e-artifacts-${{ github.event.pull_request.head.sha || github.sha }}
path: e2e-artifacts/portable-profile/
include-hidden-files: false
if-no-files-found: ignore
diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json
index 079a35a9e2..8d4a264a83 100644
--- a/ci/source-shape-test-budget.json
+++ b/ci/source-shape-test-budget.json
@@ -66,6 +66,11 @@
"test": "keeps live E2E on the accepted rootless runtime and local registry authority (#9006)",
"category": "compatibility"
},
+ {
+ "file": "test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts",
+ "test": "selects exact-commit rootless evidence for Portable recovery changes (#9707)",
+ "category": "security"
+ },
{
"file": "test/growth-guardrails-workflow-boundary.test.ts",
"test": "runs the trusted Vitest guardrails against pull request data",
diff --git a/docs/inference/set-up-openai-compatible-endpoint.mdx b/docs/inference/set-up-openai-compatible-endpoint.mdx
index 1cff806354..d1da794de7 100644
--- a/docs/inference/set-up-openai-compatible-endpoint.mdx
+++ b/docs/inference/set-up-openai-compatible-endpoint.mdx
@@ -132,10 +132,10 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_AGENT=langchain-deepage
-The portable profile creates the `openshell-docker` network on `169.254.1.0/24` and assigns `169.254.2.2/32` to the host loopback interface.
+The portable profile creates the `openshell-docker` network on `10.87.0.0/24` and assigns `169.254.2.2/32` to the host loopback interface.
The address `169.254.2.2` is outside the Portable sandbox subnet and is the host-gateway address for `host.openshell.internal`.
It carries authenticated mTLS plus sandbox-JWT callbacks to OpenShell on port `8080`.
-The managed local registry uses the distinct address `169.254.1.3` on port `5000`.
+The managed local registry uses the distinct address `10.87.0.3` on port `5000`.
Keeping the host-gateway address outside the sandbox subnet prevents a Portable workload from receiving that address.
@@ -153,6 +153,26 @@ $$nemoclaw onboard --experimental-profile portable
Do not run the loopback deletion command when onboarding reports another interface or prefix.
Investigate and resolve the conflicting assignment before you rerun onboarding.
+
+If Portable onboarding reports that `openshell-docker` still uses `169.254.1.0/24`, use only the commands in that error. Before it shows a command, NemoClaw verifies the current-user Podman socket, the complete retired network record, and every running or stopped container connected to that network. Each command includes the verified socket URL and the complete 64-character resource ID.
+
+When no container is connected, onboarding prints one network command:
+
+```bash
+podman --url 'unix://' network rm <64-character-network-id>
+```
+
+When the exact NemoClaw-managed registry is the only connected container, onboarding prints the registry commands before the network command. A stopped registry does not need the first command. Run each printed command in order, and continue only after it succeeds:
+
+```bash
+podman --url 'unix://' container stop <64-character-registry-id>
+podman --url 'unix://' container rm <64-character-registry-id>
+podman --url 'unix://' network rm <64-character-network-id>
+```
+
+Do not change the socket or IDs, and do not add `--force` to any command. If inspection is inconclusive or any other container is connected, onboarding prints no removal command. Leave the network and containers unchanged. This recovery does not remove sandboxes, inference containers, or Hermes resources. Use the lifecycle that created each resource, then retry onboarding.
+
+Rerun Portable onboarding once the network is gone.
The portable profile handles the descriptor as follows:
diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-inference.test.ts b/src/lib/onboard/experimental/hermes-portable-ollama-inference.test.ts
index 9aee04267e..6fb0861ff0 100644
--- a/src/lib/onboard/experimental/hermes-portable-ollama-inference.test.ts
+++ b/src/lib/onboard/experimental/hermes-portable-ollama-inference.test.ts
@@ -140,7 +140,7 @@ function createRuntimeFixture() {
});
harness.state.networkId = NETWORK_ID;
harness.state.networkName = "openshell-docker";
- harness.state.networkGatewayIp = "169.254.1.1";
+ harness.state.networkGatewayIp = "10.87.0.1";
harness.state.ollamaPsModels = [
{
name: "qwen3-vl:4b",
@@ -378,7 +378,7 @@ describe("Hermes Portable Ollama inference activation", () => {
model: "qwen3-vl:4b",
networkName: "openshell-docker",
networkId: NETWORK_ID,
- networkGatewayIp: "169.254.1.1",
+ networkGatewayIp: "10.87.0.1",
networkListenerIp: PORTABLE_HOST_GATEWAY_IP,
gpuDevices: [GPU_DEVICE],
},
diff --git a/src/lib/onboard/experimental/portable-host-preparation.test.ts b/src/lib/onboard/experimental/portable-host-preparation.test.ts
index 1dc3e86844..b2a88fe361 100644
--- a/src/lib/onboard/experimental/portable-host-preparation.test.ts
+++ b/src/lib/onboard/experimental/portable-host-preparation.test.ts
@@ -153,10 +153,9 @@ describe("preparePortableExperimentalHost", () => {
const portableSandboxAddresses = new BlockList();
portableSandboxAddresses.addSubnet(networkAddress!, Number(prefixText), "ipv4");
- expect(PORTABLE_DOCKER_NETWORK_SUBNET).toBe("169.254.1.0/24");
- expect(PORTABLE_REGISTRY_IP).toBe("169.254.1.3");
+ expect(PORTABLE_DOCKER_NETWORK_SUBNET).toBe("10.87.0.0/24");
+ expect(PORTABLE_REGISTRY_IP).toBe("10.87.0.3");
expect(PORTABLE_HOST_GATEWAY_IP).toBe("169.254.2.2");
- expect(portableSandboxAddresses.check("169.254.1.2", "ipv4")).toBe(true);
expect(portableSandboxAddresses.check(PORTABLE_REGISTRY_IP, "ipv4")).toBe(true);
expect(portableSandboxAddresses.check(PORTABLE_HOST_GATEWAY_IP, "ipv4")).toBe(false);
expect(PORTABLE_HOST_GATEWAY_IP).not.toBe(PORTABLE_REGISTRY_IP);
@@ -645,7 +644,7 @@ describe("preparePortableExperimentalHost", () => {
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result())
.mockReturnValueOnce(result(0, JSON.stringify([{ Subnet: PORTABLE_DOCKER_NETWORK_SUBNET }])))
- .mockReturnValueOnce(result(0, `1 true ${PORTABLE_REGISTRY_IP}`));
+ .mockReturnValueOnce(result(0, `1|true|${PORTABLE_REGISTRY_IP}`));
const ip = vi
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result(0, NO_RETIRED_GATEWAY_EVIDENCE))
@@ -743,7 +742,7 @@ describe("preparePortableExperimentalHost", () => {
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result())
.mockReturnValueOnce(result(0, JSON.stringify([{ Subnet: PORTABLE_DOCKER_NETWORK_SUBNET }])))
- .mockReturnValueOnce(result(0, `1 true ${PORTABLE_REGISTRY_IP}`));
+ .mockReturnValueOnce(result(0, `1|true|${PORTABLE_REGISTRY_IP}`));
preparePortableExperimentalHost(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
@@ -767,7 +766,7 @@ describe("preparePortableExperimentalHost", () => {
it.each<[SpawnResult, string]>([
[
result(0, JSON.stringify([{ Subnet: "10.88.0.0/16" }])),
- "Refusing to reuse network 'openshell-docker' with unexpected subnet '10.88.0.0/16'. Expected 169.254.1.0/24.",
+ "Refusing to reuse network 'openshell-docker' with unexpected subnet '10.88.0.0/16'. Expected 10.87.0.0/24.",
],
[
{
@@ -1049,7 +1048,7 @@ describe("preparePortableExperimentalHost", () => {
const docker = vi
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result())
- .mockReturnValueOnce(result(0, "1 true"))
+ .mockReturnValueOnce(result(0, "1|true|"))
.mockReturnValueOnce(result());
preparePortableExperimentalHost(
@@ -1113,7 +1112,7 @@ describe("preparePortableExperimentalHost", () => {
const docker = vi
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result())
- .mockReturnValueOnce(result(0, `1 true ${PORTABLE_REGISTRY_IP}`));
+ .mockReturnValueOnce(result(0, `1|true|${PORTABLE_REGISTRY_IP}`));
try {
const prepared = preparePortableExperimentalHost(scope.env, {
@@ -1370,7 +1369,7 @@ describe("preparePortableExperimentalHost", () => {
const docker = vi
.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
.mockReturnValueOnce(result())
- .mockReturnValueOnce(result(0, `1 true ${PORTABLE_REGISTRY_IP}`));
+ .mockReturnValueOnce(result(0, `1|true|${PORTABLE_REGISTRY_IP}`));
const prepared = preparePortableExperimentalHost(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
diff --git a/src/lib/onboard/experimental/portable-host-preparation.ts b/src/lib/onboard/experimental/portable-host-preparation.ts
index 9afd004756..077b3352aa 100644
--- a/src/lib/onboard/experimental/portable-host-preparation.ts
+++ b/src/lib/onboard/experimental/portable-host-preparation.ts
@@ -21,6 +21,7 @@ import {
DOCKER_NETWORK_IPAM_INSPECT_FORMAT,
isPortableExperimentalProfile,
parseDockerNetworkIpamEntries,
+ PORTABLE_DOCKER_NETWORK_NAME,
PORTABLE_DOCKER_NETWORK_SUBNET,
PORTABLE_HOST_GATEWAY_IP,
PORTABLE_LOCAL_REGISTRY,
@@ -41,11 +42,26 @@ import {
} from "./portable-cpu-delegation-preflight";
const REGISTRY_CONTAINER = "nemoclaw-portable-registry";
-const REGISTRY_LABEL = "com.nvidia.nemoclaw.portable=1";
+const REGISTRY_LABEL_NAME = "com.nvidia.nemoclaw.portable";
+const REGISTRY_LABEL_VALUE = "1";
+const REGISTRY_LABEL = `${REGISTRY_LABEL_NAME}=${REGISTRY_LABEL_VALUE}`;
// Portable onboarding assigned this address to loopback before #9587 moved the
// host gateway outside the sandbox subnet. Keep the retired value here so an
// upgraded host cannot silently retain a route that captures sandbox traffic.
const RETIRED_PORTABLE_HOST_GATEWAY_IP = "169.254.1.2";
+// Go template fields are joined with an explicit separator because a missing
+// label renders as ``, which contains a space and would corrupt
+// whitespace splitting.
+const DOCKER_FIELD_SEPARATOR = "|";
+// Portable onboarding created the sandbox network on this subnet before #9707
+// moved it out of the link-local block that netavark refuses. Name the retired
+// value so an upgraded host gets the removal command instead of the generic
+// unexpected-subnet refusal.
+const RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET = "169.254.1.0/24";
+const RETIRED_PORTABLE_DOCKER_NETWORK_GATEWAY = "169.254.1.1";
+const RETIRED_PORTABLE_REGISTRY_IP = "169.254.1.3";
+const FULL_PODMAN_ID_PATTERN = /^[a-f0-9]{64}$/u;
+const AUTO_PODMAN_BRIDGE_INTERFACE_PATTERN = /^podman(?:0|[1-9][0-9]{0,8})$/u;
const REGISTRY_IMAGE =
"docker.io/library/registry:2@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
const HOST_COMMAND_TIMEOUT_MS = 30_000;
@@ -115,6 +131,117 @@ function isRecord(value: unknown): value is Record {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
+function parseFullPodmanIds(raw: string): readonly string[] {
+ if (raw === "") return [];
+ const lines = raw.split(/\r?\n/u);
+ if (lines.at(-1) === "") lines.pop();
+ if (
+ lines.length === 0 ||
+ lines.some(
+ (line) => line === "" || line !== line.trim() || !FULL_PODMAN_ID_PATTERN.test(line),
+ ) ||
+ new Set(lines).size !== lines.length
+ ) {
+ throw new Error("invalid Podman IDs");
+ }
+ return lines;
+}
+
+function quoteRecoveryArgument(value: string): string {
+ return `'${value.replaceAll("'", `'\\''`)}'`;
+}
+
+interface RetiredPortableNetworkEvidence {
+ readonly networkId: string;
+}
+
+function parseRetiredPortableNetwork(
+ raw: string,
+ expectedName: string,
+): RetiredPortableNetworkEvidence {
+ const parsed: unknown = JSON.parse(raw);
+ if (!Array.isArray(parsed) || parsed.length !== 1 || !isRecord(parsed[0])) {
+ throw new Error("invalid retired network evidence");
+ }
+ const network = parsed[0];
+ const subnets = network.subnets;
+ const subnet = Array.isArray(subnets) && subnets.length === 1 ? subnets[0] : null;
+ const ipamOptions = network.ipam_options;
+ const labels = network.labels;
+ const options = network.options;
+ const routes = network.routes;
+ if (
+ typeof network.id !== "string" ||
+ !FULL_PODMAN_ID_PATTERN.test(network.id) ||
+ expectedName !== PORTABLE_DOCKER_NETWORK_NAME ||
+ network.name !== expectedName ||
+ network.driver !== "bridge" ||
+ network.internal !== false ||
+ network.ipv6_enabled !== false ||
+ network.dns_enabled !== true ||
+ typeof network.network_interface !== "string" ||
+ !AUTO_PODMAN_BRIDGE_INTERFACE_PATTERN.test(network.network_interface) ||
+ Object.hasOwn(network, "network_dns_servers") ||
+ !isRecord(subnet) ||
+ subnet.subnet !== RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET ||
+ subnet.gateway !== RETIRED_PORTABLE_DOCKER_NETWORK_GATEWAY ||
+ Object.hasOwn(subnet, "lease_range") ||
+ !isRecord(ipamOptions) ||
+ ipamOptions.driver !== "host-local" ||
+ (labels !== undefined &&
+ labels !== null &&
+ (!isRecord(labels) || Object.keys(labels).length)) ||
+ (options !== undefined &&
+ options !== null &&
+ (!isRecord(options) || Object.keys(options).length)) ||
+ (routes !== undefined && routes !== null && (!Array.isArray(routes) || routes.length !== 0))
+ ) {
+ throw new Error("ambiguous retired network evidence");
+ }
+ return { networkId: network.id };
+}
+
+interface RetiredPortableRegistryEvidence {
+ readonly containerId: string;
+ readonly running: boolean;
+}
+
+function parseOwnedRetiredRegistry(
+ raw: string,
+ expectedContainerId: string,
+ networkId: string,
+ networkName: string,
+): RetiredPortableRegistryEvidence {
+ const parsed: unknown = JSON.parse(raw);
+ if (!Array.isArray(parsed) || parsed.length !== 1 || !isRecord(parsed[0])) {
+ throw new Error("invalid retired registry evidence");
+ }
+ const registry = parsed[0];
+ const config = isRecord(registry.Config) ? registry.Config : null;
+ const labels = config && isRecord(config.Labels) ? config.Labels : null;
+ const state = isRecord(registry.State) ? registry.State : null;
+ const networkSettings = isRecord(registry.NetworkSettings) ? registry.NetworkSettings : null;
+ const networks =
+ networkSettings && isRecord(networkSettings.Networks) ? networkSettings.Networks : null;
+ const networkNames = networks ? Object.keys(networks) : [];
+ const attachment = networks && isRecord(networks[networkName]) ? networks[networkName] : null;
+ if (
+ typeof registry.Id !== "string" ||
+ registry.Id !== expectedContainerId ||
+ !FULL_PODMAN_ID_PATTERN.test(registry.Id) ||
+ registry.Name !== REGISTRY_CONTAINER ||
+ labels?.[REGISTRY_LABEL_NAME] !== REGISTRY_LABEL_VALUE ||
+ typeof state?.Running !== "boolean" ||
+ networkNames.length !== 1 ||
+ networkNames[0] !== networkName ||
+ attachment?.NetworkID !== networkId ||
+ attachment.IPAddress !== RETIRED_PORTABLE_REGISTRY_IP
+ ) {
+ throw new Error("ambiguous retired registry evidence");
+ }
+ return { containerId: registry.Id, running: state.Running };
+}
+
/**
* The portable profile points DOCKER_HOST at the rootless Podman socket but still
* drives the managed registry — and the rest of onboarding's runtime preflight —
@@ -428,10 +555,97 @@ function validateOwnedConfigAuthority(input: {
if (socketPath) assertOwnedDescendants(runtimeDir, path.dirname(socketPath));
}
+function retiredPortableNetworkRecovery(
+ env: NodeJS.ProcessEnv,
+ podman: NonNullable,
+ podmanUrl: string,
+ networkName: string,
+ assertSocketAuthority: () => void,
+): never {
+ const run = (args: readonly string[]): SpawnResult => {
+ assertSocketAuthority();
+ try {
+ return podman(["--url", podmanUrl, ...args], env, HOST_COMMAND_TIMEOUT_MS);
+ } finally {
+ assertSocketAuthority();
+ }
+ };
+ const blocked = (): never => {
+ throw new Error(
+ `Network '${networkName}' still uses the retired portable subnet ${RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET}, ` +
+ "but NemoClaw could not verify that it is safe to show recovery commands. No removal " +
+ "commands were produced. Leave the network and connected containers unchanged. Resolve " +
+ "each resource through the lifecycle that created it, then rerun " +
+ "`nemoclaw onboard --experimental-profile portable`.",
+ );
+ };
+
+ let networkId: string;
+ let attachmentIds: readonly string[];
+ let registry: RetiredPortableRegistryEvidence | null = null;
+ try {
+ const networkInspection = run(["network", "inspect", networkName]);
+ requireCommand(networkInspection, "Inspecting the retired portable sandbox network");
+ ({ networkId } = parseRetiredPortableNetwork(
+ String(networkInspection.stdout ?? ""),
+ networkName,
+ ));
+
+ const attachmentInspection = run([
+ "ps",
+ "--all",
+ "--no-trunc",
+ "--filter",
+ `network=${networkId}`,
+ "--format",
+ "{{.ID}}",
+ ]);
+ requireCommand(attachmentInspection, "Inspecting retired portable network attachments");
+ attachmentIds = parseFullPodmanIds(String(attachmentInspection.stdout ?? ""));
+ if (attachmentIds.length > 1) return blocked();
+ if (attachmentIds.length === 1) {
+ const containerId = attachmentIds[0]!;
+ const registryInspection = run(["container", "inspect", containerId]);
+ requireCommand(registryInspection, "Inspecting the retired portable registry candidate");
+ registry = parseOwnedRetiredRegistry(
+ String(registryInspection.stdout ?? ""),
+ containerId,
+ networkId,
+ networkName,
+ );
+ }
+ } catch {
+ return blocked();
+ }
+
+ const commandPrefix = `podman --url ${quoteRecoveryArgument(podmanUrl)}`;
+ const networkCommand = `${commandPrefix} network rm ${networkId}`;
+ if (attachmentIds.length === 0) {
+ throw new Error(
+ `Network '${networkName}' still uses the retired portable subnet ${RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET}. ` +
+ `Run \`${networkCommand}\`. Continue only after this succeeds. Then rerun ` +
+ "`nemoclaw onboard --experimental-profile portable`.",
+ );
+ }
+ if (!registry || attachmentIds[0] !== registry.containerId) return blocked();
+ const stopStep = registry.running
+ ? `Run \`${commandPrefix} container stop ${registry.containerId}\`. Continue only after this succeeds. `
+ : "";
+ throw new Error(
+ `Network '${networkName}' still uses the retired portable subnet ${RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET}. ` +
+ `${stopStep}Run \`${commandPrefix} container rm ${registry.containerId}\`. Continue only after this succeeds. ` +
+ `Then run \`${networkCommand}\`. Continue only after this succeeds. Then rerun ` +
+ "`nemoclaw onboard --experimental-profile portable`.",
+ );
+}
+
function ensurePortableSandboxNetwork(
env: NodeJS.ProcessEnv,
docker: NonNullable,
+ podman: NonNullable,
+ podmanUrl: string,
networkName: string,
+ assertSocketAuthority: () => void,
): void {
const networkInspection = docker(
["network", "inspect", "--format", DOCKER_NETWORK_IPAM_INSPECT_FORMAT, networkName],
@@ -444,6 +658,9 @@ function ensurePortableSandboxNetwork(
const subnets = (parseDockerNetworkIpamEntries(String(networkInspection.stdout ?? "")) ?? [])
.map((entry) => entry.subnet)
.filter((subnet): subnet is string => Boolean(subnet));
+ if (subnets.length === 1 && subnets[0] === RETIRED_PORTABLE_DOCKER_NETWORK_SUBNET) {
+ retiredPortableNetworkRecovery(env, podman, podmanUrl, networkName, assertSocketAuthority);
+ }
if (subnets.length !== 1 || subnets[0] !== PORTABLE_DOCKER_NETWORK_SUBNET) {
throw new Error(
`Refusing to reuse network '${networkName}' with unexpected subnet '${subnets.join(", ") || "none"}'. Expected ${PORTABLE_DOCKER_NETWORK_SUBNET}.`,
@@ -457,27 +674,28 @@ function ensurePortableSandboxNetwork(
}
}
+function registryInspectionArgs(networkName: string): readonly string[] {
+ return [
+ "inspect",
+ "--format",
+ `{{ index .Config.Labels "com.nvidia.nemoclaw.portable" }}${DOCKER_FIELD_SEPARATOR}{{.State.Running}}${DOCKER_FIELD_SEPARATOR}{{with index .NetworkSettings.Networks ${JSON.stringify(networkName)}}}{{.IPAddress}}{{end}}`,
+ REGISTRY_CONTAINER,
+ ];
+}
+
function ensureRegistryContainer(
env: NodeJS.ProcessEnv,
docker: NonNullable,
networkName: string,
): void {
- const inspection = docker(
- [
- "inspect",
- "--format",
- `{{ index .Config.Labels "com.nvidia.nemoclaw.portable" }} {{.State.Running}} {{with index .NetworkSettings.Networks ${JSON.stringify(networkName)}}}{{.IPAddress}}{{end}}`,
- REGISTRY_CONTAINER,
- ],
- env,
- );
+ const inspection = docker(registryInspectionArgs(networkName), env);
if (inspection.error) {
requireCommand(inspection, "Inspecting the managed portable registry");
}
const exists = inspection.status === 0;
const [owner, running, networkIp] = String(inspection.stdout ?? "")
.trim()
- .split(/\s+/u);
+ .split(DOCKER_FIELD_SEPARATOR);
if (exists && owner !== "1") {
throw new Error(
`Refusing to replace existing unmanaged container '${REGISTRY_CONTAINER}'. Rename or remove it and retry.`,
@@ -700,6 +918,12 @@ export function preparePortableExperimentalHost(
const socketAuthority = readiness.authority;
deps.qualifyPodman?.(socketAuthority);
const dockerHost = readiness.dockerHost;
+ const assertVerifiedSocketAuthority = (): void =>
+ (
+ deps.assertSocketAuthority ??
+ deps.runtimeReadiness?.assertSocketAuthority ??
+ assertPodmanSocketAuthority
+ )(socketAuthority);
console.log(
` Portable Podman readiness: ${readiness.timing.mode}; activation ${String(readiness.timing.activationMs)} ms; API ${String(readiness.timing.apiMs)} ms; total ${String(readiness.timing.totalMs)} ms.`,
);
@@ -732,16 +956,17 @@ export function preparePortableExperimentalHost(
timeout: HOST_COMMAND_TIMEOUT_MS,
}));
rejectRetiredPortableHostGatewayAlias(podmanEnv, ip);
- ensurePortableSandboxNetwork(podmanEnv, docker, dockerNetworkName);
+ ensurePortableSandboxNetwork(
+ podmanEnv,
+ docker,
+ podman,
+ dockerHost,
+ dockerNetworkName,
+ assertVerifiedSocketAuthority,
+ );
ensurePortableHostGatewayAlias(podmanEnv, ip, sudo);
ensureRegistryContainer(podmanEnv, docker, dockerNetworkName);
- if (socketAuthority) {
- (
- deps.assertSocketAuthority ??
- deps.runtimeReadiness?.assertSocketAuthority ??
- assertPodmanSocketAuthority
- )(socketAuthority);
- }
+ assertVerifiedSocketAuthority();
return {
authority: runtimeAuthority,
socketAuthority,
diff --git a/src/lib/onboard/experimental/portable-profile.ts b/src/lib/onboard/experimental/portable-profile.ts
index 58978d06b6..95ac6c94f5 100644
--- a/src/lib/onboard/experimental/portable-profile.ts
+++ b/src/lib/onboard/experimental/portable-profile.ts
@@ -6,9 +6,15 @@ import { DEFAULT_DOCKER_DRIVER_NETWORK_NAME } from "./docker-network-authority";
export const EXPERIMENTAL_PROFILE_ENV = "NEMOCLAW_EXPERIMENTAL_PROFILE";
export const PORTABLE_EXPERIMENTAL_PROFILE = "portable";
export const PORTABLE_HOST_GATEWAY_IP = "169.254.2.2";
-export const PORTABLE_REGISTRY_IP = "169.254.1.3";
+// Netavark refuses bridge networks inside the RFC 3927 link-local block.
+// 10.87.0.0/24 is outside stock Podman's default bridge and automatic pools,
+// and outside Docker's built-in local-scope bridge pools. A host route or
+// custom pool can still overlap it, so explicit network creation remains the
+// authoritative conflict check. The host gateway keeps its link-local address
+// because netavark validates bridge subnets, not loopback aliases (#9707).
+export const PORTABLE_REGISTRY_IP = "10.87.0.3";
export const PORTABLE_DOCKER_NETWORK_NAME = DEFAULT_DOCKER_DRIVER_NETWORK_NAME;
-export const PORTABLE_DOCKER_NETWORK_SUBNET = "169.254.1.0/24";
+export const PORTABLE_DOCKER_NETWORK_SUBNET = "10.87.0.0/24";
export const PORTABLE_LOCAL_REGISTRY = "localhost:5000";
export type ExperimentalOnboardProfile = typeof PORTABLE_EXPERIMENTAL_PROFILE;
diff --git a/src/lib/onboard/experimental/portable-retired-subnet-recovery.test.ts b/src/lib/onboard/experimental/portable-retired-subnet-recovery.test.ts
new file mode 100644
index 0000000000..bb8f375df3
--- /dev/null
+++ b/src/lib/onboard/experimental/portable-retired-subnet-recovery.test.ts
@@ -0,0 +1,650 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import { BlockList } from "node:net";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ preparePortableExperimentalHost,
+ type PortableHostPreparationDeps,
+} from "./portable-host-preparation";
+import {
+ PORTABLE_DOCKER_NETWORK_SUBNET,
+ PORTABLE_HOST_GATEWAY_IP,
+ PORTABLE_REGISTRY_IP,
+} from "./portable-profile";
+
+type SpawnResult = ReturnType;
+type HostCommandMock = ReturnType<
+ typeof vi.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>
+>;
+type SocketAuthorityMock = ReturnType<
+ typeof vi.fn>
+>;
+
+const RETIRED_SUBNET = "169.254.1.0/24";
+const RETIRED_REGISTRY_IP = "169.254.1.3";
+const NETWORK_ID = "a".repeat(64);
+const REGISTRY_ID = "b".repeat(64);
+const OTHER_ID = "c".repeat(64);
+const SOCKET = "unix:///run/user/1001/podman/podman.sock";
+const OWNER_LABEL = "com.nvidia.nemoclaw.portable";
+const NO_RETIRED_GATEWAY_EVIDENCE = JSON.stringify([
+ { ifname: "lo", addr_info: [{ family: "inet", local: "127.0.0.1", prefixlen: 8 }] },
+]);
+
+type RecoveryScenario = {
+ readonly networkInspection?: SpawnResult;
+ readonly attachmentInspection?: SpawnResult;
+ readonly registryInspection?: SpawnResult;
+ readonly socketFailureAt?: number;
+};
+
+function result(status: number | null = 0, stdout = "", error?: Error): SpawnResult {
+ return {
+ status,
+ stdout,
+ stderr: "",
+ ...(error ? { error } : {}),
+ } as SpawnResult;
+}
+
+function networkRecord(overrides: Record = {}): Record {
+ return {
+ name: "openshell-docker",
+ id: NETWORK_ID,
+ driver: "bridge",
+ internal: false,
+ ipv6_enabled: false,
+ dns_enabled: true,
+ network_interface: "podman1",
+ subnets: [{ subnet: RETIRED_SUBNET, gateway: "169.254.1.1" }],
+ ipam_options: { driver: "host-local" },
+ labels: {},
+ options: {},
+ routes: [],
+ ...overrides,
+ };
+}
+
+function networkSnapshot(overrides: Record = {}): string {
+ return JSON.stringify([networkRecord(overrides)]);
+}
+
+function registryRecord(
+ overrides: {
+ readonly id?: string;
+ readonly name?: string;
+ readonly labels?: Record;
+ readonly running?: unknown;
+ readonly networks?: Record;
+ } = {},
+): Record {
+ return {
+ Id: overrides.id ?? REGISTRY_ID,
+ Name: overrides.name ?? "nemoclaw-portable-registry",
+ Config: { Labels: overrides.labels ?? { [OWNER_LABEL]: "1" } },
+ State: { Running: overrides.running ?? true },
+ NetworkSettings: {
+ Networks: overrides.networks ?? {
+ "openshell-docker": {
+ IPAddress: RETIRED_REGISTRY_IP,
+ NetworkID: NETWORK_ID,
+ },
+ },
+ },
+ };
+}
+
+function registrySnapshot(overrides: Parameters[0] = {}): string {
+ return JSON.stringify([registryRecord(overrides)]);
+}
+
+function recoveryPodman(scenario: RecoveryScenario, events: string[] = []) {
+ return vi.fn((args: readonly string[], _env: NodeJS.ProcessEnv): SpawnResult => {
+ expect(args.slice(0, 2)).toEqual(["--url", SOCKET]);
+ const command = args.slice(2);
+ events.push(`podman:${command.join(" ")}`);
+ switch (`${command[0]}:${command[1]}`) {
+ case "network:inspect":
+ return scenario.networkInspection ?? result(0, networkSnapshot());
+ case "ps:--all":
+ expect(command).toContain(`network=${NETWORK_ID}`);
+ return scenario.attachmentInspection ?? result();
+ case "container:inspect":
+ return scenario.registryInspection ?? result(0, registrySnapshot());
+ default:
+ throw new Error(`unexpected Podman command: ${command.join(" ")}`);
+ }
+ });
+}
+
+function preparationDeps(
+ home: string,
+ podman: ReturnType,
+ docker: HostCommandMock,
+ sudo: HostCommandMock,
+ assertSocketAuthority: SocketAuthorityMock,
+) {
+ return {
+ platform: "linux" as const,
+ home,
+ uid: 1001,
+ systemctl: () => result(),
+ podman,
+ docker,
+ hardenSocketDirectory: vi.fn(),
+ validateConfigAuthority: vi.fn(),
+ sudo,
+ ip: (args: readonly string[]) =>
+ args[0] === "-j"
+ ? result(0, NO_RETIRED_GATEWAY_EVIDENCE)
+ : result(0, `1: lo inet ${PORTABLE_HOST_GATEWAY_IP}/32 scope global lo\n`),
+ cpuDelegationPreflight: () => ({ ok: true as const, detail: "stubbed in tests" }),
+ runtimeReadiness: {
+ uid: 1001,
+ home,
+ hardenSocketDirectory: vi.fn(),
+ captureSocketAuthority: (socketPath: string) => ({
+ directoryChain: [],
+ device: "1",
+ inode: "2",
+ mode: String(0o140660),
+ ownerUid: "1001",
+ socketPath,
+ }),
+ assertSocketAuthority,
+ podmanCapture: () => ({
+ status: 0,
+ stdout: JSON.stringify({ Server: { Version: "5.7.0" } }),
+ stderr: "",
+ }),
+ },
+ };
+}
+
+const tempDirs: string[] = [];
+
+function runRecovery(scenario: RecoveryScenario = {}) {
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-"));
+ tempDirs.push(home);
+ const events: string[] = [];
+ let recoveryStarted = false;
+ let recoveryAssertionCount = 0;
+ const podman = recoveryPodman(scenario, events);
+ const docker = vi.fn((args: readonly string[], _env: NodeJS.ProcessEnv): SpawnResult => {
+ const isVersionProbe = args[0] === "--version";
+ recoveryStarted ||= !isVersionProbe;
+ return isVersionProbe ? result() : result(0, JSON.stringify([{ Subnet: RETIRED_SUBNET }]));
+ });
+ const sudo = vi.fn((_args: readonly string[], _env: NodeJS.ProcessEnv) => result());
+ const assertSocketAuthority = vi.fn<
+ NonNullable
+ >(() => {
+ events.push("assert");
+ recoveryAssertionCount += Number(recoveryStarted);
+ return recoveryStarted && scenario.socketFailureAt === recoveryAssertionCount
+ ? (() => {
+ throw new Error("socket authority changed");
+ })()
+ : undefined;
+ });
+ let message = "";
+ try {
+ preparePortableExperimentalHost(
+ { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
+ preparationDeps(home, podman, docker, sudo, assertSocketAuthority),
+ );
+ } catch (error) {
+ message = (error as Error).message;
+ }
+ expect(message).not.toBe("");
+
+ const issuedDocker = docker.mock.calls.map(([args]) => args.join(" "));
+ const issuedPodman = podman.mock.calls.map(([args]) => args.slice(2).join(" "));
+ expect(
+ issuedDocker.some((command) => /\b(?:rm|stop)\b|--force|network create/u.test(command)),
+ ).toBe(false);
+ expect(issuedPodman.some((command) => /\b(?:rm|stop)\b|--force/u.test(command))).toBe(false);
+ expect(sudo).not.toHaveBeenCalled();
+ for (const [args, env] of podman.mock.calls) {
+ expect(args.slice(0, 2)).toEqual(["--url", SOCKET]);
+ expect(env.DOCKER_HOST).toBe(SOCKET);
+ }
+ for (const [index] of [...events.entries()].filter(([, event]) => event.startsWith("podman:"))) {
+ expect(events[index - 1]).toBe("assert");
+ expect(events[index + 1]).toBe("assert");
+ }
+
+ return { message, podman };
+}
+
+function ownedRegistryScenario(overrides: RecoveryScenario = {}): RecoveryScenario {
+ return {
+ attachmentInspection: result(0, REGISTRY_ID),
+ registryInspection: result(0, registrySnapshot()),
+ ...overrides,
+ };
+}
+
+function expectNoRemovalGuidance(scenario: RecoveryScenario): void {
+ const { message } = runRecovery(scenario);
+ expect(message).toContain("No removal commands were produced");
+ expect(message).not.toContain("podman --url");
+ expect(message).not.toContain("network rm");
+ expect(message).not.toContain("container rm");
+ expect(message).not.toContain("container stop");
+}
+
+describe("portable retired-subnet recovery (#9707)", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ for (const tempDir of tempDirs.splice(0)) {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ });
+
+ it("keeps the sandbox subnet outside the link-local block netavark refuses", () => {
+ const linkLocal = new BlockList();
+ linkLocal.addSubnet("169.254.0.0", 16, "ipv4");
+ const [networkAddress] = PORTABLE_DOCKER_NETWORK_SUBNET.split("/");
+
+ expect(linkLocal.check(networkAddress!, "ipv4")).toBe(false);
+ expect(linkLocal.check(PORTABLE_REGISTRY_IP, "ipv4")).toBe(false);
+ });
+
+ it("prints only the socket-bound immutable network command when the retired network is empty", () => {
+ const { message, podman } = runRecovery();
+ const networkCommand = `podman --url '${SOCKET}' network rm ${NETWORK_ID}`;
+
+ expect(podman).toHaveBeenCalledTimes(2);
+ expect(podman.mock.calls[0]?.[0]).toEqual([
+ "--url",
+ SOCKET,
+ "network",
+ "inspect",
+ "openshell-docker",
+ ]);
+ expect(podman.mock.calls[1]?.[0]).toEqual([
+ "--url",
+ SOCKET,
+ "ps",
+ "--all",
+ "--no-trunc",
+ "--filter",
+ `network=${NETWORK_ID}`,
+ "--format",
+ "{{.ID}}",
+ ]);
+ expect(message).toContain(networkCommand);
+ expect(message).not.toContain("container rm");
+ expect(message).not.toContain("--force");
+ });
+
+ it("prints ordered force-free immutable commands for the sole owned running registry", () => {
+ const { message, podman } = runRecovery(ownedRegistryScenario());
+ const stopCommand = `podman --url '${SOCKET}' container stop ${REGISTRY_ID}`;
+ const registryCommand = `podman --url '${SOCKET}' container rm ${REGISTRY_ID}`;
+ const networkCommand = `podman --url '${SOCKET}' network rm ${NETWORK_ID}`;
+
+ expect(podman).toHaveBeenCalledTimes(3);
+ expect(podman.mock.calls[2]?.[0]).toEqual([
+ "--url",
+ SOCKET,
+ "container",
+ "inspect",
+ REGISTRY_ID,
+ ]);
+ expect(message).toContain(stopCommand);
+ expect(message).toContain(registryCommand);
+ expect(message).toContain(networkCommand);
+ expect(message.indexOf(stopCommand)).toBeLessThan(message.indexOf(registryCommand));
+ expect(message.indexOf(registryCommand)).toBeLessThan(message.indexOf(networkCommand));
+ expect(message).not.toContain("--force");
+ });
+
+ it("omits the stop command for the sole owned stopped registry", () => {
+ const { message } = runRecovery(
+ ownedRegistryScenario({
+ registryInspection: result(0, registrySnapshot({ running: false })),
+ }),
+ );
+
+ expect(message).not.toContain("container stop");
+ expect(message).toContain(`podman --url '${SOCKET}' container rm ${REGISTRY_ID}`);
+ expect(message).toContain(`podman --url '${SOCKET}' network rm ${NETWORK_ID}`);
+ expect(message).not.toContain("--force");
+ });
+
+ it.each<[string, RecoveryScenario]>([
+ [
+ "the network inspection process fails",
+ { networkInspection: result(null, "", new Error("spawn failed")) },
+ ],
+ ["the network inspection status fails", { networkInspection: result(125) }],
+ ["the network JSON is malformed", { networkInspection: result(0, "{") }],
+ ["no network record is returned", { networkInspection: result(0, "[]") }],
+ [
+ "more than one network record is returned",
+ { networkInspection: result(0, JSON.stringify([networkRecord(), networkRecord()])) },
+ ],
+ [
+ "the network ID is truncated",
+ { networkInspection: result(0, networkSnapshot({ id: "abc" })) },
+ ],
+ [
+ "the network ID uses uppercase",
+ { networkInspection: result(0, networkSnapshot({ id: "A".repeat(64) })) },
+ ],
+ [
+ "the network name differs",
+ { networkInspection: result(0, networkSnapshot({ name: "other" })) },
+ ],
+ [
+ "the network driver differs",
+ { networkInspection: result(0, networkSnapshot({ driver: "macvlan" })) },
+ ],
+ [
+ "the network is internal",
+ { networkInspection: result(0, networkSnapshot({ internal: true })) },
+ ],
+ [
+ "the network enables IPv6",
+ { networkInspection: result(0, networkSnapshot({ ipv6_enabled: true })) },
+ ],
+ [
+ "the network DNS mode is missing",
+ { networkInspection: result(0, networkSnapshot({ dns_enabled: undefined })) },
+ ],
+ [
+ "the network DNS mode is malformed",
+ { networkInspection: result(0, networkSnapshot({ dns_enabled: "true" })) },
+ ],
+ [
+ "the network disables DNS",
+ { networkInspection: result(0, networkSnapshot({ dns_enabled: false })) },
+ ],
+ [
+ "the network bridge interface is missing",
+ { networkInspection: result(0, networkSnapshot({ network_interface: undefined })) },
+ ],
+ [
+ "the network bridge interface is malformed",
+ { networkInspection: result(0, networkSnapshot({ network_interface: 1 })) },
+ ],
+ [
+ "the network uses a custom bridge interface",
+ { networkInspection: result(0, networkSnapshot({ network_interface: "portable0" })) },
+ ],
+ [
+ "the network has custom DNS servers",
+ {
+ networkInspection: result(0, networkSnapshot({ network_dns_servers: ["192.0.2.53"] })),
+ },
+ ],
+ [
+ "the network DNS server evidence is malformed",
+ { networkInspection: result(0, networkSnapshot({ network_dns_servers: "192.0.2.53" })) },
+ ],
+ [
+ "the network DNS server evidence is ambiguous",
+ { networkInspection: result(0, networkSnapshot({ network_dns_servers: null })) },
+ ],
+ [
+ "the subnet record is missing",
+ { networkInspection: result(0, networkSnapshot({ subnets: [] })) },
+ ],
+ [
+ "the network has another subnet",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({ subnets: [{ subnet: "10.0.0.0/24", gateway: "10.0.0.1" }] }),
+ ),
+ },
+ ],
+ [
+ "the network has more than one subnet",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({
+ subnets: [
+ { subnet: RETIRED_SUBNET, gateway: "169.254.1.1" },
+ { subnet: "fd00::/64", gateway: "fd00::1" },
+ ],
+ }),
+ ),
+ },
+ ],
+ [
+ "the network gateway differs",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({ subnets: [{ subnet: RETIRED_SUBNET, gateway: "169.254.1.254" }] }),
+ ),
+ },
+ ],
+ [
+ "the subnet has a custom lease range",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({
+ subnets: [
+ {
+ subnet: RETIRED_SUBNET,
+ gateway: "169.254.1.1",
+ lease_range: { start_ip: "169.254.1.20", end_ip: "169.254.1.30" },
+ },
+ ],
+ }),
+ ),
+ },
+ ],
+ [
+ "the subnet lease range is malformed",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({
+ subnets: [
+ {
+ subnet: RETIRED_SUBNET,
+ gateway: "169.254.1.1",
+ lease_range: "169.254.1.20-169.254.1.30",
+ },
+ ],
+ }),
+ ),
+ },
+ ],
+ [
+ "the subnet lease range evidence is ambiguous",
+ {
+ networkInspection: result(
+ 0,
+ networkSnapshot({
+ subnets: [{ subnet: RETIRED_SUBNET, gateway: "169.254.1.1", lease_range: null }],
+ }),
+ ),
+ },
+ ],
+ [
+ "the network IPAM driver is missing",
+ { networkInspection: result(0, networkSnapshot({ ipam_options: {} })) },
+ ],
+ [
+ "the network IPAM driver differs",
+ { networkInspection: result(0, networkSnapshot({ ipam_options: { driver: "dhcp" } })) },
+ ],
+ [
+ "the network has an ownership label",
+ { networkInspection: result(0, networkSnapshot({ labels: { owner: "user" } })) },
+ ],
+ [
+ "the network has a custom option",
+ { networkInspection: result(0, networkSnapshot({ options: { mtu: "1400" } })) },
+ ],
+ [
+ "the network has a custom route",
+ {
+ networkInspection: result(0, networkSnapshot({ routes: [{ destination: "10.0.0.0/8" }] })),
+ },
+ ],
+ [
+ "the attachment query process fails",
+ { attachmentInspection: result(null, "", new Error("spawn failed")) },
+ ],
+ ["the attachment query status fails", { attachmentInspection: result(125) }],
+ ["an attachment ID is truncated", { attachmentInspection: result(0, "abc") }],
+ ["an attachment ID uses uppercase", { attachmentInspection: result(0, "C".repeat(64)) }],
+ ["an attachment ID has whitespace", { attachmentInspection: result(0, ` ${OTHER_ID}`) }],
+ ["the attachment list has a blank row", { attachmentInspection: result(0, `${OTHER_ID}\n\n`) }],
+ [
+ "an attachment row has an extra field",
+ { attachmentInspection: result(0, `${OTHER_ID}|foreign`) },
+ ],
+ [
+ "an attachment row contains shell text",
+ { attachmentInspection: result(0, `${OTHER_ID};exit`) },
+ ],
+ [
+ "an attachment ID is duplicated",
+ { attachmentInspection: result(0, `${OTHER_ID}\n${OTHER_ID}`) },
+ ],
+ [
+ "more than one container is attached",
+ { attachmentInspection: result(0, `${REGISTRY_ID}\n${OTHER_ID}`) },
+ ],
+ ["the sole attachment is foreign", { attachmentInspection: result(0, OTHER_ID) }],
+ [
+ "the registry inspection process fails",
+ ownedRegistryScenario({ registryInspection: result(null, "", new Error("spawn failed")) }),
+ ],
+ [
+ "the registry inspection status fails",
+ ownedRegistryScenario({ registryInspection: result(125) }),
+ ],
+ [
+ "the registry JSON is malformed",
+ ownedRegistryScenario({ registryInspection: result(0, "{") }),
+ ],
+ [
+ "no registry record is returned",
+ ownedRegistryScenario({ registryInspection: result(0, "[]") }),
+ ],
+ [
+ "more than one registry record is returned",
+ ownedRegistryScenario({
+ registryInspection: result(0, JSON.stringify([registryRecord(), registryRecord()])),
+ }),
+ ],
+ [
+ "the registry ID differs from the holder inventory",
+ ownedRegistryScenario({ registryInspection: result(0, registrySnapshot({ id: OTHER_ID })) }),
+ ],
+ [
+ "the registry name differs",
+ ownedRegistryScenario({
+ registryInspection: result(0, registrySnapshot({ name: "replacement" })),
+ }),
+ ],
+ [
+ "the registry ownership label is missing",
+ ownedRegistryScenario({ registryInspection: result(0, registrySnapshot({ labels: {} })) }),
+ ],
+ [
+ "the registry ownership label differs",
+ ownedRegistryScenario({
+ registryInspection: result(0, registrySnapshot({ labels: { [OWNER_LABEL]: "0" } })),
+ }),
+ ],
+ [
+ "the registry running state is malformed",
+ ownedRegistryScenario({
+ registryInspection: result(0, registrySnapshot({ running: "true" })),
+ }),
+ ],
+ [
+ "the registry uses another retired-network IP",
+ ownedRegistryScenario({
+ registryInspection: result(
+ 0,
+ registrySnapshot({
+ networks: {
+ "openshell-docker": { IPAddress: "169.254.1.4", NetworkID: NETWORK_ID },
+ },
+ }),
+ ),
+ }),
+ ],
+ [
+ "the registry NetworkID differs",
+ ownedRegistryScenario({
+ registryInspection: result(
+ 0,
+ registrySnapshot({
+ networks: {
+ "openshell-docker": { IPAddress: RETIRED_REGISTRY_IP, NetworkID: OTHER_ID },
+ },
+ }),
+ ),
+ }),
+ ],
+ [
+ "the registry NetworkID is missing",
+ ownedRegistryScenario({
+ registryInspection: result(
+ 0,
+ registrySnapshot({
+ networks: {
+ "openshell-docker": { IPAddress: RETIRED_REGISTRY_IP },
+ },
+ }),
+ ),
+ }),
+ ],
+ [
+ "the registry is attached to another network",
+ ownedRegistryScenario({
+ registryInspection: result(
+ 0,
+ registrySnapshot({
+ networks: {
+ "openshell-docker": { IPAddress: RETIRED_REGISTRY_IP, NetworkID: NETWORK_ID },
+ other: { IPAddress: "10.0.0.2", NetworkID: OTHER_ID },
+ },
+ }),
+ ),
+ }),
+ ],
+ [
+ "the registry is attached only to another network",
+ ownedRegistryScenario({
+ registryInspection: result(
+ 0,
+ registrySnapshot({
+ networks: {
+ other: { IPAddress: "10.0.0.2", NetworkID: OTHER_ID },
+ },
+ }),
+ ),
+ }),
+ ],
+ ])("stops without deletion guidance when %s", (_label, scenario) => {
+ expectNoRemovalGuidance(scenario);
+ });
+
+ it.each([1, 2, 3, 4, 5, 6])(
+ "stops without deletion guidance when socket authority assertion %i fails",
+ (socketFailureAt) => {
+ expectNoRemovalGuidance(ownedRegistryScenario({ socketFailureAt }));
+ },
+ );
+});
diff --git a/src/lib/onboard/gateway-sandbox-reachability.test.ts b/src/lib/onboard/gateway-sandbox-reachability.test.ts
index 24c1654662..38e9cc82fe 100644
--- a/src/lib/onboard/gateway-sandbox-reachability.test.ts
+++ b/src/lib/onboard/gateway-sandbox-reachability.test.ts
@@ -126,7 +126,7 @@ describe("isSandboxBridgeGatewayReachable", () => {
const result = await isSandboxBridgeGatewayReachable({
inspectNetworkImpl: (networkName) => {
expect(networkName).toBe(PORTABLE_DOCKER_NETWORK_NAME);
- return { subnet: PORTABLE_DOCKER_NETWORK_SUBNET, gatewayIp: "169.254.1.1" };
+ return { subnet: PORTABLE_DOCKER_NETWORK_SUBNET, gatewayIp: "10.87.0.1" };
},
usesHostGatewayRouteImpl: () => false,
runImpl: (args) => {
@@ -147,7 +147,7 @@ describe("isSandboxBridgeGatewayReachable", () => {
const addHostIndex = seen.args.indexOf("--add-host");
expect(seen.args[networkIndex + 1]).toBe(PORTABLE_DOCKER_NETWORK_NAME);
expect(seen.args[addHostIndex + 1]).toBe(`host.openshell.internal:${PORTABLE_HOST_GATEWAY_IP}`);
- expect(seen.args).not.toContain("host.openshell.internal:169.254.1.1");
+ expect(seen.args).not.toContain("host.openshell.internal:10.87.0.1");
});
it("does not call a missing Docker network a firewall failure", async () => {
diff --git a/test/e2e/live/portable-cpu-delegation-proof.test.ts b/test/e2e/live/portable-cpu-delegation-proof.test.ts
index 3a13751667..d1def16720 100644
--- a/test/e2e/live/portable-cpu-delegation-proof.test.ts
+++ b/test/e2e/live/portable-cpu-delegation-proof.test.ts
@@ -142,16 +142,16 @@ function proveAdmission(
DOCKER_NETWORK_IPAM_INSPECT_FORMAT,
networkName,
]),
- commandResult(0, JSON.stringify([{ Subnet: "169.254.1.0/24" }])),
+ commandResult(0, JSON.stringify([{ Subnet: "10.87.0.0/24" }])),
],
[
JSON.stringify([
"inspect",
"--format",
- `{{ index .Config.Labels "com.nvidia.nemoclaw.portable" }} {{.State.Running}} {{with index .NetworkSettings.Networks ${JSON.stringify(networkName)}}}{{.IPAddress}}{{end}}`,
+ `{{ index .Config.Labels "com.nvidia.nemoclaw.portable" }}|{{.State.Running}}|{{with index .NetworkSettings.Networks ${JSON.stringify(networkName)}}}{{.IPAddress}}{{end}}`,
registryContainer,
]),
- commandResult(0, `1 true ${PORTABLE_REGISTRY_IP}`),
+ commandResult(0, `1|true|${PORTABLE_REGISTRY_IP}`),
],
]);
const ipResults: ReadonlyMap = new Map([
diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts
index 6bff7fcfda..dc2b0c20ff 100644
--- a/test/e2e/live/portable-profile-rootless-linux.test.ts
+++ b/test/e2e/live/portable-profile-rootless-linux.test.ts
@@ -72,6 +72,7 @@ const BASE_IMAGE =
const PORTABLE_PROFILE_E2E_PHASES = [
"select the Podman-reported runtime socket",
"prepare the rootless container runtime",
+ "verify immutable non-force network removal",
"build and publish the sandbox image",
"start the pinned Podman gateway",
"verify distinct same-network routes",
@@ -94,6 +95,15 @@ function run(command: string, args: readonly string[]): string {
return String(result.stdout).trim();
}
+function parseOnePodmanRecord(raw: string, label: string): Record {
+ const parsed: unknown = JSON.parse(raw);
+ assert.ok(Array.isArray(parsed), `${label} must be a JSON array`);
+ assert.equal(parsed.length, 1, `${label} must contain one record`);
+ const record = parsed[0];
+ assert.ok(record && typeof record === "object" && !Array.isArray(record), `${label} is invalid`);
+ return record as Record;
+}
+
async function waitForRegistry(attempt = 0): Promise {
assert.ok(attempt < 60, "The managed local registry did not become ready.");
const ready = await new Promise((resolve) => {
@@ -128,15 +138,25 @@ function selectInstallerPodmanRuntime(repoRoot: string): string {
async function main(progress: TestProgress): Promise {
assert.equal(process.platform, "linux", "portable profile E2E requires Linux");
assert.notEqual(process.getuid?.(), 0, "portable profile E2E must run without root privileges");
-
- const root = fs.mkdtempSync(
- path.join(os.userInfo().homedir, ".nemoclaw-portable-e2e-"),
+ const sourceRevision = process.env.E2E_SOURCE_REVISION;
+ assert.match(
+ sourceRevision ?? "",
+ /^[a-f0-9]{40}$/u,
+ "E2E_SOURCE_REVISION must identify the exact candidate commit",
);
+ assert.equal(run("git", ["rev-parse", "HEAD"]), sourceRevision);
+
+ const root = fs.mkdtempSync(path.join(os.userInfo().homedir, ".nemoclaw-portable-e2e-"));
const home = path.join(root, "home");
const binDir = path.join(root, "bin");
const stateDir = path.join(root, "gateway-state");
const configHome = path.join(home, ".config");
const runtimeDir = `/run/user/${String(process.getuid?.())}`;
+ const disposableNetworkName = `nemoclaw-portable-id-proof-${String(process.pid)}`;
+ let disposableNetworkCreated = false;
+ let disposableNetworkId: string | null = null;
+ let disposableNetworkSubnet: string | null = null;
+ let disposableNetworkInterface: string | null = null;
const gatewayAliasPresentBefore = run("ip", ["-o", "-4", "address", "show", "dev", "lo"])
.split("\n")
.some((line) => line.includes(`inet ${PORTABLE_HOST_GATEWAY_IP}/32`));
@@ -186,6 +206,61 @@ async function main(progress: TestProgress): Promise {
run("docker", ["version"]);
await waitForRegistry();
+ progress.phase("verify immutable non-force network removal");
+ const verifiedPodmanUrl = String(process.env.DOCKER_HOST);
+ assert.equal(verifiedPodmanUrl, `unix://${runtimeDir}/podman/podman.sock`);
+ // Netavark rejects the retired link-local subnet before this pinned runtime can create it.
+ // Deterministic tests own that state; this live boundary proves the emitted full-ID form.
+ run("podman", ["--url", verifiedPodmanUrl, "network", "create", disposableNetworkName]);
+ disposableNetworkCreated = true;
+ const disposableNetwork = parseOnePodmanRecord(
+ run("podman", ["--url", verifiedPodmanUrl, "network", "inspect", disposableNetworkName]),
+ "disposable network inspection",
+ );
+ assert.equal(disposableNetwork.name, disposableNetworkName);
+ assert.equal(disposableNetwork.driver, "bridge");
+ assert.equal(disposableNetwork.dns_enabled, true);
+ assert.match(String(disposableNetwork.network_interface), /^podman(?:0|[1-9][0-9]{0,8})$/u);
+ assert.equal(Object.hasOwn(disposableNetwork, "network_dns_servers"), false);
+ assert.match(String(disposableNetwork.id), /^[a-f0-9]{64}$/u);
+ assert.ok(Array.isArray(disposableNetwork.subnets));
+ assert.equal(disposableNetwork.subnets.length, 1);
+ const disposableSubnet = disposableNetwork.subnets[0] as Record;
+ assert.equal(typeof disposableSubnet.subnet, "string");
+ assert.equal(Object.hasOwn(disposableSubnet, "lease_range"), false);
+ assert.notEqual(disposableSubnet.subnet, "169.254.1.0/24");
+ disposableNetworkId = String(disposableNetwork.id);
+ disposableNetworkSubnet = String(disposableSubnet.subnet);
+ disposableNetworkInterface = String(disposableNetwork.network_interface);
+ run("podman", ["--url", verifiedPodmanUrl, "network", "rm", disposableNetworkId]);
+ disposableNetworkCreated = false;
+ const absentInspection = spawnSync(
+ "podman",
+ ["--url", verifiedPodmanUrl, "network", "inspect", disposableNetworkId],
+ {
+ encoding: "utf-8",
+ env: process.env,
+ killSignal: "SIGKILL",
+ stdio: ["ignore", "pipe", "pipe"],
+ timeout: 15_000,
+ },
+ );
+ assert.equal(absentInspection.error, undefined);
+ assert.notEqual(absentInspection.status, 0);
+ const remainingNetworkIds = run("podman", [
+ "--url",
+ verifiedPodmanUrl,
+ "network",
+ "ls",
+ "--no-trunc",
+ "--format",
+ "{{.ID}}",
+ ])
+ .split("\n")
+ .filter(Boolean);
+ assert.ok(remainingNetworkIds.every((id) => /^[a-f0-9]{64}$/u.test(id)));
+ assert.ok(!remainingNetworkIds.includes(disposableNetworkId));
+
progress.phase("build and publish the sandbox image");
const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), SANDBOX_BUILD_CONTEXT_PREFIX));
fs.chmodSync(buildCtx, 0o700);
@@ -235,6 +310,29 @@ async function main(progress: TestProgress): Promise {
]),
PORTABLE_REGISTRY_IP,
);
+ const currentNetwork = parseOnePodmanRecord(
+ run("podman", [
+ "--url",
+ verifiedPodmanUrl,
+ "network",
+ "inspect",
+ PORTABLE_DOCKER_NETWORK_NAME,
+ ]),
+ "portable network inspection",
+ );
+ assert.match(String(currentNetwork.id), /^[a-f0-9]{64}$/u);
+ const currentRegistry = parseOnePodmanRecord(
+ run("podman", [
+ "--url",
+ verifiedPodmanUrl,
+ "container",
+ "inspect",
+ "nemoclaw-portable-registry",
+ ]),
+ "portable registry inspection",
+ );
+ assert.match(String(currentRegistry.Id), /^[a-f0-9]{64}$/u);
+ assert.equal(currentRegistry.Name, "nemoclaw-portable-registry");
const gatewayBin = run("bash", ["-lc", "command -v openshell-gateway"]);
const sandboxBin = run("bash", ["-lc", "command -v openshell-sandbox"]);
@@ -312,9 +410,67 @@ async function main(progress: TestProgress): Promise {
);
});
+ const artifactDir = process.env.E2E_ARTIFACT_DIR;
+ assert.ok(artifactDir, "E2E_ARTIFACT_DIR is required for the rootless receipt");
+ fs.mkdirSync(artifactDir, { recursive: true, mode: 0o700 });
+ fs.writeFileSync(
+ path.join(artifactDir, "portable-profile-rootless-receipt.json"),
+ `${JSON.stringify(
+ {
+ sourceRevision,
+ rootless: true,
+ podmanPackageVersion: process.env.PODMAN_APT_VERSION ?? null,
+ podmanVersion: run("podman", ["--version"]),
+ podmanUrl: verifiedPodmanUrl,
+ immutableNetworkRemoval: {
+ networkId: disposableNetworkId,
+ subnet: disposableNetworkSubnet,
+ networkForce: false,
+ absentAfterRemoval: true,
+ retiredUpgradeEndToEnd: false,
+ inspectedShape: {
+ dnsEnabled: true,
+ networkInterface: disposableNetworkInterface,
+ networkDnsServersPresent: false,
+ leaseRangePresent: false,
+ },
+ },
+ portableNetwork: {
+ id: currentNetwork.id,
+ subnet: PORTABLE_DOCKER_NETWORK_SUBNET,
+ hostGateway: `${PORTABLE_HOST_GATEWAY_IP}/32`,
+ },
+ registry: { id: currentRegistry.Id, ip: PORTABLE_REGISTRY_IP },
+ authenticatedGatewayRoute: true,
+ registryRoute: true,
+ },
+ null,
+ 2,
+ )}\n`,
+ { encoding: "utf-8", mode: 0o600 },
+ );
+
progress.phase("record portable environment completion");
console.log("Portable profile rootless environment E2E passed.");
} finally {
+ void (disposableNetworkCreated
+ ? spawnSync(
+ "podman",
+ [
+ "--url",
+ `unix://${runtimeDir}/podman/podman.sock`,
+ "network",
+ "rm",
+ disposableNetworkId ?? disposableNetworkName,
+ ],
+ {
+ env: process.env,
+ killSignal: "SIGKILL",
+ stdio: "ignore",
+ timeout: 15_000,
+ },
+ )
+ : undefined);
spawnSync("podman", ["rm", "--force", "nemoclaw-portable-registry"], {
env: process.env,
killSignal: "SIGKILL",
diff --git a/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts b/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts
index ca35b209ce..56f4f55406 100644
--- a/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts
+++ b/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts
@@ -5,7 +5,13 @@ import fs from "node:fs";
import { describe, expect, it } from "vitest";
-import { readYaml, type Workflow } from "../../helpers/e2e-workflow-contract";
+import { readRepoText, readYaml, type Workflow } from "../../helpers/e2e-workflow-contract";
+
+type PortableProfileWorkflow = Workflow & {
+ on: {
+ pull_request: { paths: string[]; types: string[] };
+ };
+};
describe("portable profile rootless runtime workflow", () => {
// source-shape-contract: compatibility -- The workflow and live fixture must keep the accepted OS, Podman, AppArmor, and HTTP local-registry authorities aligned before live E2E
@@ -69,9 +75,7 @@ describe("portable profile rootless runtime workflow", () => {
expect(policy).toContain('if ! grep -Eq "$signal_rule" "$pasta_profile"; then');
expect(policy).toContain('test "$(grep -Ec "$signal_rule" "$pasta_profile")" -eq 1');
expect(policy).toContain('apparmor_parser -r "$pasta_profile"');
- expect(liveTest).toContain(
- 'path.join(os.userInfo().homedir, ".nemoclaw-portable-e2e-")',
- );
+ expect(liveTest).toContain('path.join(os.userInfo().homedir, ".nemoclaw-portable-e2e-")');
expect(liveTest).not.toMatch(
/mkdtempSync\(\s*path\.join\(os\.tmpdir\(\),\s*["']nemoclaw-portable-e2e-/,
);
@@ -82,4 +86,41 @@ describe("portable profile rootless runtime workflow", () => {
expect(liveTest).toContain("parseDockerNetworkIpamEntries(");
expect(liveTest).not.toContain("{{range .Subnets}}");
});
+
+ // source-shape-contract: security -- topology changes must select an exact-commit rootless proof, and the live receipt must distinguish ordinary full-ID removal from the netavark-rejected retired state
+ it("selects exact-commit rootless evidence for Portable recovery changes (#9707)", () => {
+ const workflow = readYaml(
+ ".github/workflows/portable-profile-e2e.yaml",
+ );
+ const job = workflow.jobs["rootless-linux"];
+ const checkout = job?.steps?.find((step) => step.name === "Checkout");
+ const upload = job?.steps?.find(
+ (step) => step.name === "Upload portable profile E2E artifacts",
+ );
+ const liveSource = readRepoText("test/e2e/live/portable-profile-rootless-linux.test.ts");
+ const revisionExpression = "${{ github.event.pull_request.head.sha || github.sha }}";
+
+ expect(workflow.on.pull_request.types).toEqual(["opened", "synchronize", "reopened"]);
+ expect(workflow.on.pull_request.paths).toEqual(
+ expect.arrayContaining([
+ "src/lib/onboard/experimental/portable-host-preparation.ts",
+ "src/lib/onboard/experimental/portable-profile.ts",
+ "src/lib/onboard/experimental/portable-retired-subnet-recovery.test.ts",
+ "test/e2e/live/portable-profile-rootless-linux.test.ts",
+ "test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts",
+ ]),
+ );
+ expect(job?.env?.E2E_SOURCE_REVISION).toBe(revisionExpression);
+ expect(checkout?.with?.["persist-credentials"]).toBe(false);
+ expect(checkout?.with?.ref).toBe(revisionExpression);
+ expect(upload?.if).toBe("always()");
+ expect(upload?.with?.name).toContain(revisionExpression);
+ expect(workflow.jobs["portable-launch"]?.if).toBe("${{ github.ref == 'refs/heads/main' }}");
+ expect(liveSource).toContain('run("git", ["rev-parse", "HEAD"])');
+ expect(liveSource).toContain('"network", "rm", disposableNetworkId');
+ expect(liveSource).not.toContain('"network", "rm", "--force"');
+ expect(liveSource).toContain("retiredUpgradeEndToEnd: false");
+ expect(liveSource).toContain("networkDnsServersPresent: false");
+ expect(liveSource).toContain("leaseRangePresent: false");
+ });
});
diff --git a/test/helpers/hermes-portable-ollama-test-harness.ts b/test/helpers/hermes-portable-ollama-test-harness.ts
index ef57f71f17..4179821811 100644
--- a/test/helpers/hermes-portable-ollama-test-harness.ts
+++ b/test/helpers/hermes-portable-ollama-test-harness.ts
@@ -89,7 +89,7 @@ export function createPortablePodmanCapture(
ipv6_enabled: false,
dns_enabled: true,
network_interface: "podman9",
- subnets: [{ subnet: "169.254.1.0/24", gateway: "169.254.1.1" }],
+ subnets: [{ subnet: "10.87.0.0/24", gateway: "10.87.0.1" }],
labels: authorityState.networkLabels ?? {},
ipam_options: {},
options: {},
@@ -114,7 +114,7 @@ export function createPortablePodmanCapture(
Networks: {
"openshell-docker": {
NetworkID: authorityState.registryNetworkId ?? authorityState.networkId,
- IPAddress: "169.254.1.3",
+ IPAddress: "10.87.0.3",
},
},
},