Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
55b994c
fix(hermes): accept GPU-supervised startup owner
ericksoa Jul 1, 2026
91f3c44
test(hermes): add GPU startup live proof
ericksoa Jul 1, 2026
e0c7388
test(hermes): keep live GPU proof linear
ericksoa Jul 1, 2026
14c9a82
test(hermes): isolate GPU startup proof
ericksoa Jul 1, 2026
97584bd
test(hermes): capture GPU restart topology
ericksoa Jul 1, 2026
3351571
fix(test): replace if with ternary in hermes-gpu-startup to satisfy g…
prekshivyas Jul 1, 2026
17a2e38
fix(gpu): preserve OpenShell command boundary
ericksoa Jul 1, 2026
722c5e2
test(hermes): prove GPU startup authority boundary
ericksoa Jul 1, 2026
4e52301
Merge branch 'main' into fix/hermes-gpu-startup-6110
cv Jul 1, 2026
8c083f7
test(gpu): close supervisor boundary proof gaps
ericksoa Jul 1, 2026
97e3e7e
fix(gpu): capture failed clone before rollback
ericksoa Jul 1, 2026
351061d
fix(gpu): prefer native OpenShell injection
ericksoa Jul 2, 2026
362a172
Merge remote-tracking branch 'origin/main' into fix/hermes-gpu-startu…
ericksoa Jul 2, 2026
1918ffe
refactor(gpu): keep onboarding entrypoint neutral
ericksoa Jul 2, 2026
7cb219d
test(gpu): keep diagnostic fixtures linear
ericksoa Jul 2, 2026
5a200ae
fix(gpu): preserve gateway routing invariant
ericksoa Jul 2, 2026
9a4b54c
fix(gpu): prime diagnostics redaction before capture
ericksoa Jul 2, 2026
9e971b5
test(gpu): normalize OpenShell readiness output
ericksoa Jul 2, 2026
22de6ea
test(gpu): expose forced legacy live route
ericksoa Jul 2, 2026
a1fc52c
test(e2e): detect forbidden inference markers
ericksoa Jul 2, 2026
54cf259
fix(hermes): accept OpenShell TLS key path
ericksoa Jul 2, 2026
970803a
fix(hermes): update validator integrity hash
ericksoa Jul 2, 2026
7335903
test(gpu): use canonical placeholder transport
ericksoa Jul 2, 2026
c5a67c4
docs(gpu): document native OpenShell routing
ericksoa Jul 2, 2026
a04a70a
test(gpu): verify managed startup trust boundary
ericksoa Jul 2, 2026
6ac4ebc
test(gpu): run gateway cleanup through built CLI
ericksoa Jul 2, 2026
65b06d6
test(gpu): tolerate clean preinstall teardown
ericksoa Jul 2, 2026
7b75355
fix(gpu): centralize diagnostic redaction
ericksoa Jul 2, 2026
5c87782
test(gpu): harden live proof isolation
ericksoa Jul 2, 2026
e077aa6
fix(gpu): redact generic key diagnostics
ericksoa Jul 2, 2026
d76f164
merge: sync current main into gpu fix
ericksoa Jul 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 49 additions & 2 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ on:
default: ""
type: string
jobs:
description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, jetson-nvmap-gpu, and sandbox-rlimits-connect are skipped unless selected."
description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs hermes-gpu-startup, openshell-gateway-auth-contract, jetson-nvmap-gpu, and sandbox-rlimits-connect are skipped unless selected."
required: false
default: ""
type: string
Expand Down Expand Up @@ -1578,6 +1578,52 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

hermes-gpu-startup:
needs: generate-matrix
if: ${{ contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,') }}
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
timeout-minutes: 75
env:
E2E_JOB: "1"
E2E_DEFAULT_ENABLED: "0"
E2E_TARGET_ID: "hermes-gpu-startup"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: hermes
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_GPU: "1"
NEMOCLAW_DOCKER_GPU_PATCH: "1"
NEMOCLAW_SANDBOX_NAME: e2e-hermes-gpu-startup
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28

- name: Run Hermes GPU startup live Vitest test
run: |
set -euo pipefail
npx vitest run --project e2e-live \
test/e2e/live/hermes-gpu-startup.test.ts \
--silent=false --reporter=default

- name: Upload Hermes GPU startup artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

hermes-dashboard:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }}
Expand Down Expand Up @@ -4426,6 +4472,7 @@ jobs:
sessions-agents-cli,
runtime-overrides,
hermes-e2e,
hermes-gpu-startup,
hermes-dashboard,
hermes-slack,
hermes-discord,
Expand Down Expand Up @@ -4639,7 +4686,7 @@ jobs:
? '**Requested jobs:** _(selector rejected by workflow validation)_'
: requestedJobs
? `**Requested jobs:** \`${requestedJobs}\``
: '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `jetson-nvmap-gpu`, and `sandbox-rlimits-connect` are skipped unless selected)_',
: '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `hermes-gpu-startup`, `openshell-gateway-auth-contract`, `jetson-nvmap-gpu`, and `sandbox-rlimits-connect` are skipped unless selected)_',
`**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`,
'',
'| Job | Result |',
Expand Down
16 changes: 15 additions & 1 deletion agents/hermes/runtime-config-guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,21 @@ def _pid1_is_nemoclaw_start() -> bool:
os.close(proc_pid_fd)
if proc_root_fd >= 0:
os.close(proc_root_fd)
return _cmdline_is_nemoclaw_start(cmdline)
# SOURCE_OF_TRUTH_REVIEW (#6110): OpenShell owns PID 1 in Docker-driver
# sandboxes and starts the workload as its direct non-root child. NemoClaw's
# GPU recreate used to append that workload to the supervisor argv; the
# source fix is buildDockerGpuCloneRunArgs() in docker-gpu-patch.ts, which
# now preserves an empty Config.Cmd. An attacker controlling an image or
# recreate argv could otherwise append `nemoclaw-start`, impersonate direct
# PID 1 authority, and bypass the startup identity check. Keep this
# exclusion as defense in depth; live proof is in
# test/e2e/live/hermes-gpu-startup-proof.ts. The dual-mode authorization
# branch can be removed only after direct-PID1 images are no longer
# supported; this supervisor exclusion itself remains a security invariant.
return (
not _cmdline_is_openshell_supervisor(cmdline)
and _cmdline_is_nemoclaw_start(cmdline)
)


def _pinned_process_matches_startup_identity(
Expand Down
4 changes: 2 additions & 2 deletions agents/hermes/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1681,8 +1681,8 @@ ensure_hermes_runtime_api_server_key() {
rm -f "$result_file"
return 1
}
# Keep the guard as PID 1's direct child: --startup-owner is authenticated by
# exact parent identity. The guard's own alarm bounds this startup-only call;
# Keep the guard as the startup owner's direct child: --startup-owner is
# authenticated by exact parent identity. Its own alarm bounds this call;
# wrapping it in `timeout` would interpose a different parent process.
if "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" ensure-api-key \
--hermes-dir "$HERMES_DIR" \
Expand Down
67 changes: 57 additions & 10 deletions src/lib/onboard/docker-gpu-patch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ describe("docker-gpu-patch", () => {
"env",
"CHAT_UI_URL=http://127.0.0.1:8642",
"NEMOCLAW_DASHBOARD_PORT=8642",
"HTTP_PROXY=http://proxy.example:8080",
"NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=TELEGRAM_BOT_TOKEN_AGENT_A,SLACK_BOT_TOKEN_AGENT_B",
"nemoclaw-start",
];

Expand All @@ -164,16 +166,13 @@ describe("docker-gpu-patch", () => {
expect(args).toEqual(
expect.arrayContaining([
"--env",
"OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 NEMOCLAW_DASHBOARD_PORT=8642 nemoclaw-start",
"OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 NEMOCLAW_DASHBOARD_PORT=8642 HTTP_PROXY=http://proxy.example:8080 NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=TELEGRAM_BOT_TOKEN_AGENT_A,SLACK_BOT_TOKEN_AGENT_B nemoclaw-start",
]),
);
expect(args).not.toEqual(
expect.arrayContaining(["--env", "OPENSHELL_SANDBOX_COMMAND=sleep infinity"]),
);
expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([
"openshell/sandbox:abc",
...sandboxCommand,
]);
expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual(["openshell/sandbox:abc"]);
});

it("adds OpenShell's sandbox command env when the inspected container lacks one", () => {
Expand Down Expand Up @@ -590,7 +589,12 @@ describe("docker-gpu-patch", () => {
if (args[0] === "info") return "";
return "";
});
const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" }));
const dockerRunDetached = vi.fn(
(_args: readonly string[], _opts?: Record<string, unknown>) => ({
status: 0,
stdout: "new-container-id\n",
}),
);
const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
const runOpenshell = vi.fn(() => ({ status: 1, stderr: "phase: Provisioning" }));

Expand Down Expand Up @@ -625,18 +629,61 @@ describe("docker-gpu-patch", () => {
expect(
dockerRm.mock.calls.some((call) => String(call[0]).includes("nemoclaw-gpu-backup")),
).toBe(false);
expect(dockerRunDetached).toHaveBeenCalledWith(
const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? [];
expect(cloneArgs).toEqual(
expect.arrayContaining([
"--env",
"OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start",
"openshell/sandbox:abc",
"env",
"CHAT_UI_URL=http://127.0.0.1:8642",
"nemoclaw-start",
]),
);
expect(cloneArgs.slice(cloneArgs.indexOf("openshell/sandbox:abc"))).toEqual([
"openshell/sandbox:abc",
]);
expect(dockerRunDetached).toHaveBeenCalledWith(
cloneArgs,
expect.objectContaining({ ignoreError: true }),
);
});

it("rejects whitespace-bearing startup values before touching the original container", () => {
const dockerCapture = vi.fn((args: readonly string[]) =>
args[0] === "ps"
? "old-container-id\n"
: args[0] === "inspect"
? JSON.stringify([inspectFixture()])
: "",
);
const dockerStop = vi.fn(() => ({ status: 0 }));
const dockerRename = vi.fn(() => ({ status: 0 }));
const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" }));

expect(() =>
recreateOpenShellDockerSandboxWithGpu(
{
sandboxName: "alpha",
timeoutSecs: 1,
openshellSandboxCommand: [
"env",
"HTTP_PROXY=http://proxy.example/path with space",
"nemoclaw-start",
],
},
{
dockerCapture,
dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })),
dockerRunDetached,
dockerRename,
dockerStop,
readDir: vi.fn(() => null),
readFile: vi.fn(() => null),
},
),
).toThrow("OpenShell sandbox startup command tokens cannot be empty or contain whitespace");
expect(dockerStop).not.toHaveBeenCalled();
expect(dockerRename).not.toHaveBeenCalled();
expect(dockerRunDetached).not.toHaveBeenCalled();
});
});

describe("docker-gpu-patch sandbox DNS fallback (#3579)", () => {
Expand Down
60 changes: 38 additions & 22 deletions src/lib/onboard/docker-gpu-patch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -412,8 +412,14 @@ function replaceEnvValue(entry: string, key: string, value: string | null | unde
function openshellSandboxCommandEnvValue(
command: readonly string[] | null | undefined,
): string | null {
const parts = (command || []).map((part) => String(part)).filter((part) => part.length > 0);
return parts.length > 0 ? parts.join(" ") : null;
const parts = (command || []).map((part) => String(part));
if (parts.length === 0) return null;
if (parts.some((part) => part.length === 0 || /[\s\u0085]/u.test(part))) {
throw new Error(
"OpenShell sandbox startup command tokens cannot be empty or contain whitespace.",
);
}
return parts.join(" ");
}

function dockerGpuHostEndpointFromOpenShellEndpoint(endpoint: string): string | null {
Expand Down Expand Up @@ -772,10 +778,16 @@ export function buildDockerGpuCloneRunArgs(

const entrypoint = stringArray(config.Entrypoint);
if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]);
const commandArgs =
options.openshellSandboxCommand && options.openshellSandboxCommand.length > 0
? [...options.openshellSandboxCommand]
: [...entrypoint.slice(1), ...stringArray(config.Cmd)];
// OpenShell 0.0.71's Docker driver deliberately clears Config.Cmd and passes
// the workload only through OPENSHELL_SANDBOX_COMMAND. Keep that contract
// when recreating the container: appending the same workload after the image
// puts `nemoclaw-start` in the supervisor's own argv. The serializer above
// rejects whitespace-bearing tokens because OpenShell uses split_whitespace()
// when reading this environment value (#6110). Remove this compatibility
// rewrite only after the Docker container-swap GPU patch itself is retired.
const commandArgs = openshellSandboxCommandEnv
? []
: [...entrypoint.slice(1), ...stringArray(config.Cmd)];
args.push(image, ...commandArgs);
return args;
}
Expand Down Expand Up @@ -1081,22 +1093,6 @@ export function recreateOpenShellDockerSandboxWithGpu(
const backupContainerName = buildBackupContainerName(originalName, d.now());
context.backupContainerName = backupContainerName;

d.dockerStop(oldContainerId, {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
});
const renameResult = d.dockerRename(oldContainerId, backupContainerName, {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
});
if (!isZeroStatus(renameResult)) {
throw new Error(
`Could not move original sandbox container aside: ${resultText(renameResult)}`,
);
}

const cloneOptions = buildDockerGpuCloneRunOptions(inspect);
cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null;
const sandboxFallbackDns = d.detectSandboxFallbackDns();
Expand All @@ -1121,7 +1117,27 @@ export function recreateOpenShellDockerSandboxWithGpu(
);
}
}
// Validate and build the replacement command before touching the original
// container. A malformed startup envelope must fail without stopping or
// renaming the user's working sandbox.
const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions);

d.dockerStop(oldContainerId, {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
});
const renameResult = d.dockerRename(oldContainerId, backupContainerName, {
ignoreError: true,
suppressOutput: true,
timeout: DOCKER_GPU_PATCH_TIMEOUT_MS,
});
if (!isZeroStatus(renameResult)) {
throw new Error(
`Could not move original sandbox container aside: ${resultText(renameResult)}`,
);
}

const runResult = d.dockerRunDetached(cloneArgs, {
ignoreError: true,
suppressOutput: true,
Expand Down
4 changes: 2 additions & 2 deletions src/lib/onboard/extra-placeholder-keys.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -290,15 +290,15 @@ describe("appendExtraPlaceholderKeysEnvArg", () => {
return `${key}=${value}`;
}

it("appends one whitespace-joined env arg containing only the key names, never their token values", () => {
it("appends one comma-joined env arg containing only the key names, never their token values", () => {
const envArgs: string[] = [];
appendExtraPlaceholderKeysEnvArg(
envArgs,
["TELEGRAM_BOT_TOKEN_AGENT_A", "SLACK_BOT_TOKEN_AGENT_B"],
formatEnvAssignment,
);
expect(envArgs).toEqual([
`${EXTRA_PLACEHOLDER_KEYS_ENV}=TELEGRAM_BOT_TOKEN_AGENT_A SLACK_BOT_TOKEN_AGENT_B`,
`${EXTRA_PLACEHOLDER_KEYS_ENV}=TELEGRAM_BOT_TOKEN_AGENT_A,SLACK_BOT_TOKEN_AGENT_B`,
]);
// The emitted env arg holds only the key list, not the resolved token
// value. Operators who set the credential see openshell:resolve:env:<KEY>
Expand Down
8 changes: 6 additions & 2 deletions src/lib/onboard/extra-placeholder-keys.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
// SPDX-License-Identifier: Apache-2.0

import { getCredential, normalizeCredentialValue } from "../credentials/store";
import { getChannelTokenKeys, listChannels } from "../sandbox/channels";
import * as webSearch from "../inference/web-search";
import { getChannelTokenKeys, listChannels } from "../sandbox/channels";

interface MessagingTokenDefShape {
name: string;
Expand Down Expand Up @@ -122,5 +122,9 @@ export function appendExtraPlaceholderKeysEnvArg(
formatEnvAssignment: (key: string, value: string) => string,
): void {
if (extraKeys.length === 0) return;
envArgs.push(formatEnvAssignment(EXTRA_PLACEHOLDER_KEYS_ENV, extraKeys.join(" ")));
// OpenShell's Docker supervisor deserializes OPENSHELL_SANDBOX_COMMAND with
// split_whitespace(). Commas preserve this list as one env assignment when
// the Docker GPU compatibility path transports the command through that
// variable; both host and sandbox parsers accept comma separators.
envArgs.push(formatEnvAssignment(EXTRA_PLACEHOLDER_KEYS_ENV, extraKeys.join(",")));
}
Loading
Loading