diff --git a/test/e2e/live/hermes-gpu-startup-proof.ts b/test/e2e/live/hermes-gpu-startup-proof.ts index 3eb19f2f525..9e4867a29ee 100644 --- a/test/e2e/live/hermes-gpu-startup-proof.ts +++ b/test/e2e/live/hermes-gpu-startup-proof.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { + type ManagedWorkloadAuthority, + readManagedWorkloadAuthority, +} from "../../../src/lib/onboard/workload/authority.ts"; +import { load as loadSandboxRegistry } from "../../../src/lib/state/registry/persistence.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { type HostCliClient, @@ -35,56 +40,99 @@ interface HermesGpuStartupProofOptions { status: Pick; } -export async function assertHermesGpuStartupProof({ - env, - gpuRoute, - host, - install, - sandbox, - sandboxName, - status, -}: HermesGpuStartupProofOptions): Promise { - const installText = resultText(install); +const IMMUTABLE_IMAGE_REFERENCE = /^[^@\s]+@sha256:[a-f0-9]{64}$/u; + +export function assertHermesGpuStartupOutputContract( + gpuRoute: HermesGpuStartupProofOptions["gpuRoute"], + installText: string, +): void { expect(installText).toContain("Starting OpenShell Docker-driver gateway..."); expect(installText).toContain("Docker-driver gateway is healthy"); expect(installText).not.toContain("Reusing healthy NemoClaw gateway."); expect(installText).not.toContain("Reusing existing Docker-driver gateway"); expect(installText).not.toContain("[reuse] Skipping gateway (running)"); - if (gpuRoute === "compatibility-only") { - expect(installText).toContain("Docker container mode selected:"); - for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { - expect(installText).not.toContain(fragment); - } - } else if (gpuRoute === "compatibility-fallback") { + if (gpuRoute === "compatibility-fallback") { expect(installText).toContain( "Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry.", ); for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { expect(installText).toContain(fragment); } - expect(installText).toContain("Docker container mode selected:"); } else { - expect(installText).toContain( - "Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.", - ); - expect(installText).toContain( - "Docker container mode selected: persistent sandbox startup command", - ); for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { expect(installText).not.toContain(fragment); } } +} + +export function assertHermesManagedWorkloadAuthority( + sandboxName: string, + registryImageTag: string | null | undefined, + authority: ManagedWorkloadAuthority | null, +): string { + if (!authority) { + throw new Error( + `Hermes GPU sandbox '${sandboxName}' has no managed workload authority`, + ); + } + if ( + typeof registryImageTag !== "string" || + typeof authority.receipt.reference !== "string" || + !IMMUTABLE_IMAGE_REFERENCE.test(registryImageTag) || + !IMMUTABLE_IMAGE_REFERENCE.test(authority.receipt.reference) + ) { + throw new Error( + `Hermes GPU sandbox '${sandboxName}' has no immutable image reference`, + ); + } + const authorityReference = authority.receipt.reference; + expect(authority).toMatchObject({ + agent: "hermes", + contract: { + agent: "hermes", + reference: authorityReference, + }, + profile: { agent: "hermes" }, + receipt: { + kind: "managed-image", + reference: registryImageTag, + }, + }); + return authorityReference; +} + +export function assertHermesContainerImageAuthority( + containerImage: unknown, + authorityReference: string, +): void { + expect(containerImage).toBe(authorityReference); +} + +export async function assertHermesGpuStartupProof({ + env, + gpuRoute, + host, + install, + sandbox, + sandboxName, + status, +}: HermesGpuStartupProofOptions): Promise { + const installText = resultText(install); + assertHermesGpuStartupOutputContract(gpuRoute, installText); const plainStatus = stripAnsi(resultText(status)); expect(plainStatus).toMatch(/Phase:\s*Ready/i); expect(plainStatus).toContain("Sandbox GPU: enabled"); expect(plainStatus).toContain("CUDA verified"); expect(plainStatus).not.toMatch(/last CUDA proof failed|CUDA unverified/i); - const openshellState = await sandbox.openshell(["sandbox", "get", sandboxName], { - artifactName: "phase-4-openshell-sandbox-ready-gpu-startup", - env, - timeoutMs: 30_000, - }); + const openshellState = await sandbox.openshell( + ["sandbox", "get", sandboxName], + { + artifactName: "phase-4-openshell-sandbox-ready-gpu-startup", + env, + timeoutMs: 30_000, + }, + ); expect(openshellState.exitCode, resultText(openshellState)).toBe(0); expect(stripAnsi(resultText(openshellState))).toMatch(/Phase:\s*Ready/i); @@ -132,6 +180,19 @@ export async function assertHermesGpuStartupProof({ const [containerId = ""] = containerRows[0].split(/\s+/, 1); expect(containerId).not.toBe(""); + const registryEntry = loadSandboxRegistry().sandboxes[sandboxName]; + if (!registryEntry) { + throw new Error( + `Hermes GPU sandbox '${sandboxName}' is missing from the registry`, + ); + } + const managedAuthority = readManagedWorkloadAuthority(registryEntry); + const managedImageReference = assertHermesManagedWorkloadAuthority( + sandboxName, + registryEntry.imageTag, + managedAuthority, + ); + const expectedExtraPlaceholderAssignment = `NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=${HERMES_GPU_EXTRA_PLACEHOLDER_KEYS.join(",")}`; const extraPlaceholderEnv = await host.command( "docker", @@ -168,7 +229,9 @@ raise SystemExit(1)`, }, ); expect(extraPlaceholderEnv.exitCode, resultText(extraPlaceholderEnv)).toBe(0); - expect(extraPlaceholderEnv.stdout.trim()).toBe(expectedExtraPlaceholderAssignment); + expect(extraPlaceholderEnv.stdout.trim()).toBe( + expectedExtraPlaceholderAssignment, + ); const guardWithoutStartupOwner = await sandbox.execShell( sandboxName, @@ -218,7 +281,7 @@ raise SystemExit(1)`, "bash", [ "-lc", - String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`, + String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "image": config.get("Image"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`, "hermes-gpu-command-boundary", containerId, ], @@ -228,13 +291,20 @@ raise SystemExit(1)`, timeoutMs: 30_000, }, ); - expect(dockerCommandBoundary.exitCode, resultText(dockerCommandBoundary)).toBe(0); + expect( + dockerCommandBoundary.exitCode, + resultText(dockerCommandBoundary), + ).toBe(0); const commandBoundary = JSON.parse(dockerCommandBoundary.stdout); expect(commandBoundary).toMatchObject({ cmd: ["--workdir", "/sandbox"], entrypoint: ["/opt/openshell/bin/openshell-sandbox"], has_openshell_sandbox_command: true, }); + assertHermesContainerImageAuthority( + commandBoundary.image, + managedImageReference, + ); expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(true); expect(commandBoundary.command_is_sleep_infinity).toBe(false); @@ -272,5 +342,7 @@ raise SystemExit(1)`, .map((line) => line.trim()) .filter(Boolean); expect(allContainerNames).toHaveLength(1); - expect(allContainerNames.filter((name) => name.includes("-nemoclaw-gpu-backup-"))).toEqual([]); + expect( + allContainerNames.filter((name) => name.includes("-nemoclaw-gpu-backup-")), + ).toEqual([]); } diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index 81041bfe685..d7ad88ec214 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -548,6 +548,7 @@ test( : { compatibilityOnlyRouteVerified: true }), openshellReady: true, sandboxCudaVerified: true, + managedWorkloadAuthorityVerified: true, extraPlaceholderCommandRoundTripValid: true, stableSingleContainer: true, startupConfigHashesValid: true, diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 21537d87501..31c40e8971a 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -101,6 +101,7 @@ "fast": [ "test/e2e/support/hermes-gpu-startup-fallback.test.ts", "test/e2e/support/hermes-gpu-startup-integrity.test.ts", + "test/e2e/support/hermes-gpu-startup-proof.test.ts", "test/e2e/support/hermes-workflow-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" diff --git a/test/e2e/support/hermes-gpu-startup-proof.test.ts b/test/e2e/support/hermes-gpu-startup-proof.test.ts new file mode 100644 index 00000000000..2062ba9bb18 --- /dev/null +++ b/test/e2e/support/hermes-gpu-startup-proof.test.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { ManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts"; +import { + assertHermesContainerImageAuthority, + assertHermesGpuStartupOutputContract, + assertHermesManagedWorkloadAuthority, + HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS, +} from "../live/hermes-gpu-startup-proof.ts"; + +const HEALTHY_NEW_GATEWAY = [ + "Starting OpenShell Docker-driver gateway...", + "Docker-driver gateway is healthy", +].join("\n"); +const NON_FALLBACK_DISCLOSURE_CASES = [ + ["native-success", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[0]], + ["native-success", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[1]], + ["native-success", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[2]], + ["native-success", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[3]], + ["native-success", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[4]], + ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[0]], + ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[1]], + ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[2]], + ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[3]], + ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[4]], +] as const; +const MANAGED_IMAGE_REFERENCE = `ghcr.io/nvidia/test@sha256:${"a".repeat(64)}`; +const OTHER_MANAGED_IMAGE_REFERENCE = `ghcr.io/nvidia/test@sha256:${"b".repeat(64)}`; +const VALID_MANAGED_AUTHORITY = { + agent: "hermes", + contract: { agent: "hermes", reference: MANAGED_IMAGE_REFERENCE }, + profile: { agent: "hermes" }, + receipt: { kind: "managed-image", reference: MANAGED_IMAGE_REFERENCE }, +} as unknown as ManagedWorkloadAuthority; + +describe("Hermes GPU startup output contract", () => { + it.each(["native-success", "compatibility-only"] as const)( + "accepts %s output without legacy Docker container progress text (#9362)", + (route) => { + expect(() => assertHermesGpuStartupOutputContract(route, HEALTHY_NEW_GATEWAY)).not.toThrow(); + }, + ); + + it("accepts fallback output only with the complete operator disclosure (#9362)", () => { + const output = [ + HEALTHY_NEW_GATEWAY, + "Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry.", + ...HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS, + ].join("\n"); + + expect(() => + assertHermesGpuStartupOutputContract("compatibility-fallback", output), + ).not.toThrow(); + }); + + it.each(HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS)( + "rejects fallback output that omits %s (#9362)", + (missingFragment) => { + const output = [ + HEALTHY_NEW_GATEWAY, + "Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry.", + ...HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS.filter( + (fragment) => fragment !== missingFragment, + ), + ].join("\n"); + + expect(() => + assertHermesGpuStartupOutputContract("compatibility-fallback", output), + ).toThrow(); + }, + ); + + it.each(NON_FALLBACK_DISCLOSURE_CASES)( + "rejects fallback disclosure in %s output: %s (#9362)", + (route, fragment) => { + expect(() => + assertHermesGpuStartupOutputContract(route, `${HEALTHY_NEW_GATEWAY}\n${fragment}`), + ).toThrow(); + }, + ); +}); + +describe("Hermes GPU managed-image authority proof", () => { + it("accepts one immutable authority shared by the registry, contract, and receipt (#9362)", () => { + expect( + assertHermesManagedWorkloadAuthority( + "hermes-gpu", + MANAGED_IMAGE_REFERENCE, + VALID_MANAGED_AUTHORITY, + ), + ).toBe(MANAGED_IMAGE_REFERENCE); + }); + + it("rejects a missing managed workload authority (#9362)", () => { + expect(() => + assertHermesManagedWorkloadAuthority("hermes-gpu", MANAGED_IMAGE_REFERENCE, null), + ).toThrow("has no managed workload authority"); + }); + + it.each([ + ["agent", { ...VALID_MANAGED_AUTHORITY, agent: "openclaw" }], + [ + "contract agent", + { + ...VALID_MANAGED_AUTHORITY, + contract: { ...VALID_MANAGED_AUTHORITY.contract, agent: "pi" }, + }, + ], + [ + "contract reference", + { + ...VALID_MANAGED_AUTHORITY, + contract: { + ...VALID_MANAGED_AUTHORITY.contract, + reference: "different-reference", + }, + }, + ], + ["profile agent", { ...VALID_MANAGED_AUTHORITY, profile: { agent: "openclaw" } }], + [ + "receipt kind", + { + ...VALID_MANAGED_AUTHORITY, + receipt: { ...VALID_MANAGED_AUTHORITY.receipt, kind: "custom" }, + }, + ], + ] as const)("rejects managed authority drift in %s (#9362)", (_label, authority) => { + expect(() => + assertHermesManagedWorkloadAuthority( + "hermes-gpu", + MANAGED_IMAGE_REFERENCE, + authority as unknown as ManagedWorkloadAuthority, + ), + ).toThrow(); + }); + + it("rejects registry-to-receipt image drift (#9362)", () => { + expect(() => + assertHermesManagedWorkloadAuthority( + "hermes-gpu", + OTHER_MANAGED_IMAGE_REFERENCE, + VALID_MANAGED_AUTHORITY, + ), + ).toThrow(); + }); + + it.each([ + "ghcr.io/nvidia/test:latest", + "ghcr.io/nvidia/test@sha256:different", + `ghcr.io/nvidia/test@sha256:${"A".repeat(64)}`, + ])("rejects matching mutable or malformed image authority: %s (#9362)", (reference) => { + const authority = { + ...VALID_MANAGED_AUTHORITY, + contract: { ...VALID_MANAGED_AUTHORITY.contract, reference }, + receipt: { ...VALID_MANAGED_AUTHORITY.receipt, reference }, + } as unknown as ManagedWorkloadAuthority; + + expect(() => assertHermesManagedWorkloadAuthority("hermes-gpu", reference, authority)).toThrow( + "has no immutable image reference", + ); + }); + + it("accepts the running container's exact digest-backed authority (#9362)", () => { + expect(() => + assertHermesContainerImageAuthority(MANAGED_IMAGE_REFERENCE, MANAGED_IMAGE_REFERENCE), + ).not.toThrow(); + }); + + it("rejects a running container outside the recorded authority (#9362)", () => { + expect(() => + assertHermesContainerImageAuthority("ghcr.io/nvidia/test:latest", MANAGED_IMAGE_REFERENCE), + ).toThrow(); + }); +});