Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 104 additions & 32 deletions test/e2e/live/hermes-gpu-startup-proof.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import {
type ManagedWorkloadAuthority,
readManagedWorkloadAuthority,
} from "../../../src/lib/onboard/workload/authority.ts";
import { load as loadSandboxRegistry } from "../../../src/lib/state/registry/persistence.ts";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import {
type HostCliClient,
Expand Down Expand Up @@ -35,56 +40,99 @@ interface HermesGpuStartupProofOptions {
status: Pick<ShellProbeResult, "stdout" | "stderr">;
}

export async function assertHermesGpuStartupProof({
env,
gpuRoute,
host,
install,
sandbox,
sandboxName,
status,
}: HermesGpuStartupProofOptions): Promise<void> {
const installText = resultText(install);
const IMMUTABLE_IMAGE_REFERENCE = /^[^@\s]+@sha256:[a-f0-9]{64}$/u;

export function assertHermesGpuStartupOutputContract(
gpuRoute: HermesGpuStartupProofOptions["gpuRoute"],
installText: string,
): void {
expect(installText).toContain("Starting OpenShell Docker-driver gateway...");
expect(installText).toContain("Docker-driver gateway is healthy");
expect(installText).not.toContain("Reusing healthy NemoClaw gateway.");
expect(installText).not.toContain("Reusing existing Docker-driver gateway");
expect(installText).not.toContain("[reuse] Skipping gateway (running)");
if (gpuRoute === "compatibility-only") {
expect(installText).toContain("Docker container mode selected:");
for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) {
expect(installText).not.toContain(fragment);
}
} else if (gpuRoute === "compatibility-fallback") {
if (gpuRoute === "compatibility-fallback") {
expect(installText).toContain(
"Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry.",
);
for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) {
expect(installText).toContain(fragment);
}
expect(installText).toContain("Docker container mode selected:");
} else {
expect(installText).toContain(
"Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.",
);
expect(installText).toContain(
"Docker container mode selected: persistent sandbox startup command",
);
for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) {
expect(installText).not.toContain(fragment);
}
}
}

export function assertHermesManagedWorkloadAuthority(
sandboxName: string,
registryImageTag: string | null | undefined,
authority: ManagedWorkloadAuthority | null,
): string {
if (!authority) {
throw new Error(
`Hermes GPU sandbox '${sandboxName}' has no managed workload authority`,
);
}
if (
typeof registryImageTag !== "string" ||
typeof authority.receipt.reference !== "string" ||
!IMMUTABLE_IMAGE_REFERENCE.test(registryImageTag) ||
!IMMUTABLE_IMAGE_REFERENCE.test(authority.receipt.reference)
) {
throw new Error(
`Hermes GPU sandbox '${sandboxName}' has no immutable image reference`,
);
}
const authorityReference = authority.receipt.reference;
expect(authority).toMatchObject({
agent: "hermes",
contract: {
agent: "hermes",
reference: authorityReference,
},
profile: { agent: "hermes" },
receipt: {
kind: "managed-image",
reference: registryImageTag,
},
});
return authorityReference;
}

export function assertHermesContainerImageAuthority(
containerImage: unknown,
authorityReference: string,
): void {
expect(containerImage).toBe(authorityReference);
}

export async function assertHermesGpuStartupProof({
env,
gpuRoute,
host,
install,
sandbox,
sandboxName,
status,
}: HermesGpuStartupProofOptions): Promise<void> {
const installText = resultText(install);
assertHermesGpuStartupOutputContract(gpuRoute, installText);
const plainStatus = stripAnsi(resultText(status));
expect(plainStatus).toMatch(/Phase:\s*Ready/i);
expect(plainStatus).toContain("Sandbox GPU: enabled");
expect(plainStatus).toContain("CUDA verified");
expect(plainStatus).not.toMatch(/last CUDA proof failed|CUDA unverified/i);

const openshellState = await sandbox.openshell(["sandbox", "get", sandboxName], {
artifactName: "phase-4-openshell-sandbox-ready-gpu-startup",
env,
timeoutMs: 30_000,
});
const openshellState = await sandbox.openshell(
["sandbox", "get", sandboxName],
{
artifactName: "phase-4-openshell-sandbox-ready-gpu-startup",
env,
timeoutMs: 30_000,
},
);
expect(openshellState.exitCode, resultText(openshellState)).toBe(0);
expect(stripAnsi(resultText(openshellState))).toMatch(/Phase:\s*Ready/i);

Expand Down Expand Up @@ -132,6 +180,19 @@ export async function assertHermesGpuStartupProof({
const [containerId = ""] = containerRows[0].split(/\s+/, 1);
expect(containerId).not.toBe("");

const registryEntry = loadSandboxRegistry().sandboxes[sandboxName];
if (!registryEntry) {
throw new Error(
`Hermes GPU sandbox '${sandboxName}' is missing from the registry`,
);
}
const managedAuthority = readManagedWorkloadAuthority(registryEntry);
const managedImageReference = assertHermesManagedWorkloadAuthority(
sandboxName,
registryEntry.imageTag,
managedAuthority,
);

const expectedExtraPlaceholderAssignment = `NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=${HERMES_GPU_EXTRA_PLACEHOLDER_KEYS.join(",")}`;
const extraPlaceholderEnv = await host.command(
"docker",
Expand Down Expand Up @@ -168,7 +229,9 @@ raise SystemExit(1)`,
},
);
expect(extraPlaceholderEnv.exitCode, resultText(extraPlaceholderEnv)).toBe(0);
expect(extraPlaceholderEnv.stdout.trim()).toBe(expectedExtraPlaceholderAssignment);
expect(extraPlaceholderEnv.stdout.trim()).toBe(
expectedExtraPlaceholderAssignment,
);

const guardWithoutStartupOwner = await sandbox.execShell(
sandboxName,
Expand Down Expand Up @@ -218,7 +281,7 @@ raise SystemExit(1)`,
"bash",
[
"-lc",
String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`,
String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "image": config.get("Image"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`,
"hermes-gpu-command-boundary",
containerId,
],
Expand All @@ -228,13 +291,20 @@ raise SystemExit(1)`,
timeoutMs: 30_000,
},
);
expect(dockerCommandBoundary.exitCode, resultText(dockerCommandBoundary)).toBe(0);
expect(
dockerCommandBoundary.exitCode,
resultText(dockerCommandBoundary),
).toBe(0);
const commandBoundary = JSON.parse(dockerCommandBoundary.stdout);
expect(commandBoundary).toMatchObject({
cmd: ["--workdir", "/sandbox"],
entrypoint: ["/opt/openshell/bin/openshell-sandbox"],
has_openshell_sandbox_command: true,
});
assertHermesContainerImageAuthority(
commandBoundary.image,
managedImageReference,
);
expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(true);
expect(commandBoundary.command_is_sleep_infinity).toBe(false);

Expand Down Expand Up @@ -272,5 +342,7 @@ raise SystemExit(1)`,
.map((line) => line.trim())
.filter(Boolean);
expect(allContainerNames).toHaveLength(1);
expect(allContainerNames.filter((name) => name.includes("-nemoclaw-gpu-backup-"))).toEqual([]);
expect(
allContainerNames.filter((name) => name.includes("-nemoclaw-gpu-backup-")),
).toEqual([]);
}
1 change: 1 addition & 0 deletions test/e2e/live/hermes-gpu-startup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,7 @@ test(
: { compatibilityOnlyRouteVerified: true }),
openshellReady: true,
sandboxCudaVerified: true,
managedWorkloadAuthorityVerified: true,
extraPlaceholderCommandRoundTripValid: true,
stableSingleContainer: true,
startupConfigHashesValid: true,
Expand Down
1 change: 1 addition & 0 deletions test/e2e/mock-parity.json
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
"fast": [
"test/e2e/support/hermes-gpu-startup-fallback.test.ts",
"test/e2e/support/hermes-gpu-startup-integrity.test.ts",
"test/e2e/support/hermes-gpu-startup-proof.test.ts",
"test/e2e/support/hermes-workflow-boundary.test.ts",
"test/e2e/support/e2e-cleanup-resources.test.ts",
"test/e2e/support/e2e-clients.test.ts"
Expand Down
Loading
Loading