From 7d0f7ee3ff982037d729e1cf5b4869e307a4f811 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 7 Jul 2026 10:30:20 -0400 Subject: [PATCH 1/3] test(e2e): reproduce Hermes shields cycle regression Signed-off-by: Julie Yaunches --- .github/workflows/e2e.yaml | 44 +++ test/e2e/live/hermes-shields-config.test.ts | 253 ++++++++++++++++++ ...ad-e2e-artifacts-workflow-boundary.test.ts | 4 +- ...upload-e2e-artifacts-workflow-boundary.mts | 4 +- 4 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 test/e2e/live/hermes-shields-config.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8d65390489..926e44ba42 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2043,6 +2043,49 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + hermes-shields-config: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-shields-config,') || contains(format(',{0},', inputs.targets), ',hermes-shields-config,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "hermes-shields-config" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-shields-config + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_AGENT: hermes + NEMOCLAW_SANDBOX_NAME: e2e-hermes-shields + OPENSHELL_GATEWAY: nemoclaw + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Run Hermes shields-config live regression + # This hermetic regression uses a local OpenAI-compatible endpoint and + # proves two complete down/up cycles on a fresh non-root Hermes sandbox. + run: | + set -euo pipefail + npx vitest run --project e2e-live \ + test/e2e/live/hermes-shields-config.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes shields-config artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + rebuild-openclaw: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-openclaw,') || contains(format(',{0},', inputs.targets), ',rebuild-openclaw,') }} @@ -4594,6 +4637,7 @@ jobs: network-policy, common-egress-agent, shields-config, + hermes-shields-config, rebuild-openclaw, rebuild-hermes, rebuild-hermes-stale-base, diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts new file mode 100644 index 0000000000..22cfba02e3 --- /dev/null +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-shields"; +const GATEWAY_NAME = process.env.OPENSHELL_GATEWAY ?? "nemoclaw"; +const COMPATIBLE_API_KEY = "hermes-shields-e2e-key"; +const COMPATIBLE_MODEL = "hermes-shields-e2e-model"; +const CONFIG_PATH = "/sandbox/.hermes/config.yaml"; +const HERMES_DIR = "/sandbox/.hermes"; +const TEST_TIMEOUT_MS = 45 * 60_000; +const COMMAND_TIMEOUT_MS = 120_000; + +validateSandboxName(SANDBOX_NAME); + +function commandEnv(endpointUrl?: string): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY, + NVIDIA_INFERENCE_API_KEY: COMPATIBLE_API_KEY, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_AGENT: "hermes", + NEMOCLAW_COMPAT_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_ENDPOINT_URL: endpointUrl ?? "", + NEMOCLAW_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60", + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_GPU: "0", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: GATEWAY_NAME, + }; +} + +async function cleanup(host: HostCliClient, label: string): Promise { + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: `${label}-destroy-sandbox`, + env: commandEnv(), + timeoutMs: 15 * 60_000, + }); + await host + .cleanupGatewayRegistration(GATEWAY_NAME, { + artifactName: `${label}-destroy-gateway`, + env: commandEnv(), + timeoutMs: 60_000, + }) + .catch(() => undefined); +} + +async function sandboxShell( + sandbox: SandboxClient, + script: string, + artifactName: string, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(script), { + artifactName, + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: COMMAND_TIMEOUT_MS, + }); +} + +async function runShields( + host: HostCliClient, + args: string[], + artifactName: string, +): Promise { + return await host.command("nemohermes", [SANDBOX_NAME, "shields", ...args], { + artifactName, + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: COMMAND_TIMEOUT_MS, + }); +} + +async function expectShieldsStatus( + host: HostCliClient, + expected: "DOWN" | "UP", + artifactName: string, +): Promise { + const status = await runShields(host, ["status"], artifactName); + assertExitZero(status, `read Hermes shields ${expected} status`); + expect(resultText(status)).toContain(`Shields: ${expected}`); +} + +async function expectMutablePosture(sandbox: SandboxClient, cycle: number): Promise { + const result = await sandboxShell( + sandbox, + `stat -c '%a %U:%G %n' /sandbox ${HERMES_DIR} ${CONFIG_PATH} ${HERMES_DIR}/.env ${HERMES_DIR}/.config-hash`, + `cycle-${cycle}-mutable-posture`, + ); + assertExitZero(result, `inspect Hermes mutable posture after cycle ${cycle}`); + expect(result.stdout).toContain("755 sandbox:sandbox /sandbox"); + expect(result.stdout).toContain(`3770 sandbox:sandbox ${HERMES_DIR}`); + expect(result.stdout).toContain(`640 sandbox:sandbox ${CONFIG_PATH}`); + expect(result.stdout).toContain(`640 sandbox:sandbox ${HERMES_DIR}/.env`); + expect(result.stdout).toContain(`640 sandbox:sandbox ${HERMES_DIR}/.config-hash`); +} + +async function expectLockedPosture(sandbox: SandboxClient, cycle: number): Promise { + const result = await sandboxShell( + sandbox, + `stat -c '%a %U:%G %n' /sandbox ${HERMES_DIR} ${CONFIG_PATH} ${HERMES_DIR}/.env ${HERMES_DIR}/.config-hash`, + `cycle-${cycle}-locked-posture`, + ); + assertExitZero(result, `inspect Hermes locked posture after cycle ${cycle}`); + expect(result.stdout).toContain("1775 root:sandbox /sandbox"); + expect(result.stdout).toContain(`755 root:root ${HERMES_DIR}`); + expect(result.stdout).toContain(`444 root:root ${CONFIG_PATH}`); + expect(result.stdout).toContain(`444 root:root ${HERMES_DIR}/.env`); + expect(result.stdout).toContain(`444 root:root ${HERMES_DIR}/.config-hash`); +} + +async function completeShieldsCycle( + host: HostCliClient, + sandbox: SandboxClient, + cycle: number, +): Promise { + const down = await runShields( + host, + ["down", "--timeout", "15m", "--reason", `Hermes live E2E cycle ${cycle}`], + `cycle-${cycle}-shields-down`, + ); + assertExitZero(down, `unlock fresh Hermes config in cycle ${cycle}`); + await expectShieldsStatus(host, "DOWN", `cycle-${cycle}-status-down`); + await expectMutablePosture(sandbox, cycle); + + const up = await runShields(host, ["up"], `cycle-${cycle}-shields-up`); + assertExitZero(up, `lock fresh Hermes config in cycle ${cycle}`); + await expectShieldsStatus(host, "UP", `cycle-${cycle}-status-up`); + await expectLockedPosture(sandbox, cycle); +} + +test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields cycles (#6381)", { + timeout: TEST_TIMEOUT_MS, +}, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox }) => { + await artifacts.target.declare({ + id: "hermes-shields-config", + boundary: "fresh CPU-only Hermes onboard plus two real shields down/up transitions", + contracts: [ + "fresh OpenShell-managed non-root Hermes startup mints its API key", + "the first shields-down reconciles the startup hash anchor", + "shields-up establishes the root-owned locked posture", + "a second down/up cycle completes without corrupting config state", + ], + issue: "#6381", + sandboxName: SANDBOX_NAME, + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertExitZero(docker, "Docker prerequisite for Hermes shields E2E"); + + const fake = await startFakeOpenAiCompatibleServer({ + apiKey: COMPATIBLE_API_KEY, + host: "0.0.0.0", + model: COMPATIBLE_MODEL, + publicHost: "host.openshell.internal", + requireAuth: true, + }); + cleanupRegistry.add("close Hermes shields fake inference endpoint", async () => { + await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); + await fake.close(); + }); + cleanupRegistry.add(`destroy Hermes shields sandbox ${SANDBOX_NAME}`, async () => { + await cleanup(host, "cleanup"); + }); + await cleanup(host, "pre-cleanup"); + + const env = commandEnv(fake.baseUrl); + const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: "fresh-hermes-onboard", + cwd: REPO_ROOT, + env, + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 30 * 60_000, + }); + assertExitZero(install, "fresh CPU-only Hermes onboard"); + + const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { + artifactName: "fresh-hermes-status", + env, + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(status, "read fresh Hermes status"); + expect(resultText(status)).toContain("Phase: Ready"); + + const trigger = await sandboxShell( + sandbox, + [ + "set -eu", + "test ! -e /run/nemoclaw/hermes-root-lifecycle", + `grep -Eq '^API_SERVER_KEY=[0-9a-fA-F]{64}$' ${HERMES_DIR}/.env`, + `stat -c '%a %U:%G' ${HERMES_DIR}`, + `sha256sum ${CONFIG_PATH} | awk '{print $1}'`, + ].join("\n"), + "fresh-nonroot-trigger", + ); + assertExitZero(trigger, "prove fresh non-root Hermes startup trigger"); + const triggerLines = trigger.stdout.trim().split(/\r?\n/); + expect(triggerLines[0]).toMatch(/^(700|3770) sandbox:sandbox$/); + const configHashBefore = triggerLines.at(-1) ?? ""; + expect(configHashBefore).toMatch(/^[0-9a-f]{64}$/); + + await completeShieldsCycle(host, sandbox, 1); + await completeShieldsCycle(host, sandbox, 2); + + const configHashAfter = await sandboxShell( + sandbox, + `sha256sum ${CONFIG_PATH} | awk '{print $1}'`, + "config-hash-after-two-cycles", + ); + assertExitZero(configHashAfter, "read Hermes config hash after two shields cycles"); + expect(configHashAfter.stdout.trim()).toBe(configHashBefore); + + const finalStatus = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { + artifactName: "final-hermes-status", + env, + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(finalStatus, "read Hermes status after two shields cycles"); + expect(resultText(finalStatus)).toContain("Phase: Ready"); + + await artifacts.target.complete({ + id: "hermes-shields-config", + sandboxName: SANDBOX_NAME, + assertions: { + configPreserved: true, + freshNonrootTrigger: true, + firstCycle: true, + secondCycle: true, + }, + }); +}); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 3c67b3d878..5cde721b7d 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -177,8 +177,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 73 live and E2E_JOB execution jobs", - "upload-e2e-artifacts must keep exactly 62 default callers", + "upload-e2e-artifacts must cover exactly 74 live and E2E_JOB execution jobs", + "upload-e2e-artifacts must keep exactly 63 default callers", ]), ); }); diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index ef42515dda..0b3914c63a 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -34,8 +34,8 @@ const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 73; -const EXPECTED_DEFAULT_CALLER_COUNT = 62; +const EXPECTED_UPLOAD_JOB_COUNT = 74; +const EXPECTED_DEFAULT_CALLER_COUNT = 63; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { From bf970485147b21d23e5aa475bd7ca76dce58dacb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 7 Jul 2026 10:39:06 -0400 Subject: [PATCH 2/3] test(e2e): normalize Hermes status output Signed-off-by: Julie Yaunches --- test/e2e/live/hermes-shields-config.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index 22cfba02e3..ab275811d5 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -13,6 +13,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { stripAnsi } from "./json-envelope.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-shields"; const GATEWAY_NAME = process.env.OPENSHELL_GATEWAY ?? "nemoclaw"; @@ -201,7 +202,7 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields timeoutMs: COMMAND_TIMEOUT_MS, }); assertExitZero(status, "read fresh Hermes status"); - expect(resultText(status)).toContain("Phase: Ready"); + expect(stripAnsi(resultText(status))).toMatch(/Phase:\s*Ready/i); const trigger = await sandboxShell( sandbox, @@ -238,7 +239,7 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields timeoutMs: COMMAND_TIMEOUT_MS, }); assertExitZero(finalStatus, "read Hermes status after two shields cycles"); - expect(resultText(finalStatus)).toContain("Phase: Ready"); + expect(stripAnsi(resultText(finalStatus))).toMatch(/Phase:\s*Ready/i); await artifacts.target.complete({ id: "hermes-shields-config", From 0076b5d86eb73ab092f552c208936fdd871197de Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 7 Jul 2026 10:47:55 -0400 Subject: [PATCH 3/3] test(e2e): avoid stale workflow count wording Signed-off-by: Julie Yaunches --- test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 5cde721b7d..9985da8cf1 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -77,7 +77,7 @@ function validateActionMutation(mutate: (action: MutableAction) => void): string } describe("upload-e2e-artifacts workflow boundary", () => { - it("binds one canonical uploader to all 73 E2E execution jobs", () => { + it("binds one canonical uploader to every E2E execution job", () => { expect(validateUploadE2eArtifactsAction()).toEqual([]); expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); });