diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 88e2dab4022..c00cc455ba6 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -160,28 +160,18 @@ jobs: build-hermes-sandbox-image: runs-on: ubuntu-latest - # Preserve the former 30-minute image-build, 60-minute secret-boundary, - # and 45-minute root-entrypoint budgets, plus orchestration/cleanup time. - timeout-minutes: 150 + timeout-minutes: 30 steps: - *checkout - *dockerhub-auth - - &setup-node - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22.19.0 - cache: npm - - - &install-root-dependencies - name: Install root dependencies - run: npm ci --ignore-scripts - - name: Resolve Hermes base image uses: ./.github/actions/resolve-hermes-base-image + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - &hermes-export-swap name: Add swap for Hermes image export shell: bash @@ -200,14 +190,33 @@ jobs: df -h / /mnt docker system df - - name: Build Hermes production image + - name: Validate Hermes production build args env: HERMES_BASE_IMAGE: ${{ env.HERMES_BASE_IMAGE }} run: | set -euo pipefail build_args=(-f agents/hermes/Dockerfile --build-arg "BASE_IMAGE=${HERMES_BASE_IMAGE}") scripts/check-production-build-args.sh "${build_args[@]}" - docker build "${build_args[@]}" -t nemoclaw-hermes-production . + + - name: Record resources before Hermes image build + shell: bash + run: | + set -u + free -h || true + df -h || true + docker system df || true + + - name: Build Hermes production image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: agents/hermes/Dockerfile + load: true + push: false + tags: nemoclaw-hermes-production + build-args: BASE_IMAGE=${{ env.HERMES_BASE_IMAGE }} + cache-from: type=gha,scope=hermes-production-${{ runner.os }}-${{ runner.arch }} + cache-to: type=gha,mode=max,scope=hermes-production-${{ runner.os }}-${{ runner.arch }} - name: Scan completed Hermes image for node-tar id: node-tar-scan @@ -254,9 +263,66 @@ jobs: docker run --rm --user sandbox nemoclaw-hermes-production \ test -x /usr/local/bin/nemoclaw-start + - name: Save Hermes production image + run: | + set -euo pipefail + docker save nemoclaw-hermes-production | gzip > /tmp/hermes-isolation-image.tar.gz + gzip -t /tmp/hermes-isolation-image.tar.gz + + - name: Upload Hermes isolation image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hermes-isolation-image + path: /tmp/hermes-isolation-image.tar.gz + retention-days: 1 + + - name: Record resources after Hermes image build + if: always() + shell: bash + run: | + set -u + free -h || true + df -h || true + docker system df || true + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + test-hermes-sandbox-image: + runs-on: ubuntu-latest + needs: build-hermes-sandbox-image + timeout-minutes: 90 + steps: + - *checkout + + - &setup-node + name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + cache: npm + + - &install-root-dependencies + name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Download Hermes production image + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: hermes-isolation-image + path: /tmp + + - name: Load Hermes production image + run: | + set -euo pipefail + gunzip -c /tmp/hermes-isolation-image.tar.gz | docker load + docker image inspect nemoclaw-hermes-production >/dev/null + - name: Run Hermes sandbox secret boundary test id: hermes-secret-boundary - timeout-minutes: 60 + timeout-minutes: 45 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-sandbox-secret-boundary NEMOCLAW_HERMES_TEST_IMAGE: nemoclaw-hermes-production @@ -277,7 +343,7 @@ jobs: - name: Run Hermes root entrypoint smoke Vitest test if: ${{ !cancelled() && (steps.hermes-secret-boundary.outcome == 'success' || steps.hermes-secret-boundary.outcome == 'failure') }} - timeout-minutes: 45 + timeout-minutes: 30 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-root-entrypoint-smoke NEMOCLAW_HERMES_TEST_IMAGE: nemoclaw-hermes-production @@ -298,23 +364,14 @@ jobs: if-no-files-found: ignore retention-days: 14 - - name: Save Hermes production image - run: | - set -euo pipefail - docker save nemoclaw-hermes-production | gzip > /tmp/hermes-isolation-image.tar.gz - gzip -t /tmp/hermes-isolation-image.tar.gz - - - name: Upload Hermes isolation image - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: hermes-isolation-image - path: /tmp/hermes-isolation-image.tar.gz - retention-days: 1 - - - name: Clean up Docker auth + - name: Record resources after Hermes image tests if: always() shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh + run: | + set -u + free -h || true + df -h || true + docker system df || true messaging-plan-image-boundary: runs-on: ubuntu-latest diff --git a/test/e2e/README.md b/test/e2e/README.md index 527992f12ea..96cdfe002b1 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -20,6 +20,22 @@ before those targets run; local runners must provide it themselves. call their target E2E tests directly. The Ollama auth proxy target is selected through `.github/workflows/e2e.yaml`. +## CI execution shape + +The sandbox image workflow builds the Hermes production image in the dedicated +30-minute `build-hermes-sandbox-image` job. It uses full-SHA-pinned Buildx +actions and a GitHub Actions cache scoped to the runner OS and architecture. +The producer adds a bounded 32 GiB swap file and validates the guarded +production build arguments before the build. It loads the image locally with +registry writes disabled. After the build, it scans the completed image for +node-tar and verifies the sandbox-readable installed files. It then uploads the +compressed image as the one-day `hermes-isolation-image` artifact. + +The 90-minute `test-hermes-sandbox-image` job and the +`state-dir-guard-metadata` job download and load that artifact instead of +rebuilding the image. Within the Hermes test job, the secret-boundary and +root-entrypoint steps have 45- and 30-minute budgets respectively. + The former top-level `test/e2e/test-*.sh` suite has been removed. Keep real shell, installer, process, Docker, OpenShell, `/proc`, and sandbox boundaries in E2E tests when those boundaries are the behavior under test. diff --git a/test/e2e/support/hermes-secret-boundary-workflow.test.ts b/test/e2e/support/hermes-secret-boundary-workflow.test.ts index f965bdc6c4a..db1cdfcb1ee 100644 --- a/test/e2e/support/hermes-secret-boundary-workflow.test.ts +++ b/test/e2e/support/hermes-secret-boundary-workflow.test.ts @@ -23,12 +23,15 @@ describe("Hermes image workflow secret boundary", () => { it("rejects broad Hermes job and test-step secret scope", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); - const job = imageWorkflow.jobs["build-hermes-sandbox-image"]; - job.env = { - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + const producer = imageWorkflow.jobs["build-hermes-sandbox-image"]; + const job = imageWorkflow.jobs["test-hermes-sandbox-image"]; + producer.env = { DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", }; + job.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }; const secretBoundary = job.steps?.find( (step) => step.name === "Run Hermes sandbox secret boundary test", ); @@ -41,11 +44,11 @@ describe("Hermes image workflow secret boundary", () => { expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( expect.arrayContaining([ - "build-hermes-sandbox-image must not expose NVIDIA_INFERENCE_API_KEY at job scope", "build-hermes-sandbox-image must not expose DOCKERHUB_USERNAME at job scope", "build-hermes-sandbox-image must not expose DOCKERHUB_TOKEN at job scope", - "build-hermes-sandbox-image step 'Run Hermes sandbox secret boundary test' must not receive NVIDIA_INFERENCE_API_KEY", - "build-hermes-sandbox-image step 'Run Hermes sandbox secret boundary test' must not receive DOCKERHUB_TOKEN", + "test-hermes-sandbox-image must not expose NVIDIA_INFERENCE_API_KEY at job scope", + "test-hermes-sandbox-image step 'Run Hermes sandbox secret boundary test' must not receive NVIDIA_INFERENCE_API_KEY", + "test-hermes-sandbox-image step 'Run Hermes sandbox secret boundary test' must not receive DOCKERHUB_TOKEN", ]), ); }); diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 5ad606eb77f..eafc96720ba 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -67,9 +67,8 @@ describe("sandbox image workflow boundary", () => { }, { jobName: "build-hermes-sandbox-image", - stepName: "Build Hermes production image", - error: - "Hermes production image must use the guarded build_args shape under nemoclaw-hermes-production", + stepName: "Validate Hermes production build args", + error: "Hermes production image must validate the guarded build_args shape", }, { jobName: "build-sandbox-images-arm64", @@ -98,11 +97,6 @@ describe("sandbox image workflow boundary", () => { stepName: "Build production image", error: "OpenClaw production image must have exactly one source build", }, - { - jobName: "build-hermes-sandbox-image", - stepName: "Build Hermes production image", - error: "Hermes production image must have exactly one source build", - }, { jobName: "build-sandbox-images-arm64", stepName: "Build production image on arm64", @@ -119,6 +113,88 @@ describe("sandbox image workflow boundary", () => { } }); + it("rejects a Hermes Buildx action that can publish or bypass the shared cache", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const build = imageWorkflow.jobs["build-hermes-sandbox-image"].steps!.find( + (step) => step.name === "Build Hermes production image", + )!; + build.with!.push = true; + build.with!["cache-to"] = "type=registry,ref=registry.example.invalid/cache"; + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( + expect.arrayContaining([ + "build-hermes-sandbox-image step 'Build Hermes production image' must not write images to a registry", + "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + ]), + ); + }); + + it("rejects non-canonical Hermes Buildx action pins", () => { + for (const stepName of ["Set up Docker Buildx", "Build Hermes production image"]) { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const step = imageWorkflow.jobs["build-hermes-sandbox-image"].steps!.find( + (candidate) => candidate.name === stepName, + )!; + step.uses = step.uses!.replace(/@[0-9a-f]{40}$/u, `@${"0".repeat(40)}`); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + stepName === "Set up Docker Buildx" + ? "Hermes producer must use the canonical Docker Buildx setup action exactly once" + : "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + ); + } + }); + + it("rejects a non-canonical Hermes artifact download pin", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const download = imageWorkflow.jobs["test-hermes-sandbox-image"].steps!.find( + (step) => step.name === "Download Hermes production image", + )!; + download.uses = `actions/download-artifact@${"0".repeat(40)}`; + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "Hermes image tests must download and load the producer artifact exactly once with the canonical action", + ); + }); + + it("rejects Docker Hub authentication in the Hermes image consumer", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const producer = imageWorkflow.jobs["build-hermes-sandbox-image"]; + const consumer = imageWorkflow.jobs["test-hermes-sandbox-image"]; + consumer.steps!.push({ + ...producer.steps!.find((step) => step.name === "Authenticate to Docker Hub")!, + }); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "Hermes image test consumer must not authenticate to Docker Hub", + ); + }); + + it("rejects a duplicate Hermes production-image build", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const producer = imageWorkflow.jobs["build-hermes-sandbox-image"]; + producer.steps!.push({ + ...producer.steps!.find((step) => step.name === "Build Hermes production image")!, + }); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + ); + }); + + it("rejects a Hermes cache scope without runner OS and architecture", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const build = imageWorkflow.jobs["build-hermes-sandbox-image"].steps!.find( + (step) => step.name === "Build Hermes production image", + )!; + build.with!["cache-from"] = "type=gha,scope=hermes-production-"; + build.with!["cache-to"] = "type=gha,mode=max,scope=hermes-production-"; + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + ); + }); + it("requires hardened completed-image node-tar scans and retained evidence", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); const openclaw = imageWorkflow.jobs["build-sandbox-images"]; @@ -278,10 +354,13 @@ describe("sandbox image workflow boundary", () => { it("rejects duplicate setup, rebuilding, or failing to reuse the Hermes image", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); - const hermes = imageWorkflow.jobs["build-hermes-sandbox-image"]; - hermes["timeout-minutes"] = 30; + const producer = imageWorkflow.jobs["build-hermes-sandbox-image"]; + const hermes = imageWorkflow.jobs["test-hermes-sandbox-image"]; + producer["timeout-minutes"] = 45; + hermes["timeout-minutes"] = 75; for (const stepName of ["Set up Node", "Install root dependencies"]) { hermes.steps!.push({ ...hermes.steps!.find((step) => step.name === stepName)! }); + producer.steps!.push({ ...hermes.steps!.find((step) => step.name === stepName)! }); } const rootEntrypoint = hermes.steps!.find( (step) => step.name === "Run Hermes root entrypoint smoke Vitest test", @@ -291,10 +370,11 @@ describe("sandbox image workflow boundary", () => { expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( expect.arrayContaining([ - "Hermes image job timeout must cover both inherited probe budgets", - "build-hermes-sandbox-image must run 'Set up Node' exactly once", - "build-hermes-sandbox-image must run 'Install root dependencies' exactly once", - "Hermes production image must have exactly one source build", + "Hermes image producer must retain its 30-minute budget", + "Hermes image test consumer must retain its 90-minute budget", + "build-hermes-sandbox-image must not install Node dependencies", + "test-hermes-sandbox-image must run 'Set up Node' exactly once", + "test-hermes-sandbox-image must run 'Install root dependencies' exactly once", "Hermes root entrypoint must consume the prebuilt Hermes production image", "Hermes root entrypoint step must not rebuild the prebuilt image", ]), @@ -303,24 +383,24 @@ describe("sandbox image workflow boundary", () => { it("keeps Hermes probes failure-isolated with their inherited budgets", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); - const hermes = imageWorkflow.jobs["build-hermes-sandbox-image"]; + const hermes = imageWorkflow.jobs["test-hermes-sandbox-image"]; const secretBoundary = hermes.steps!.find( (step) => step.name === "Run Hermes sandbox secret boundary test", )!; delete secretBoundary.id; - secretBoundary["timeout-minutes"] = 45; + secretBoundary["timeout-minutes"] = 44; const rootEntrypoint = hermes.steps!.find( (step) => step.name === "Run Hermes root entrypoint smoke Vitest test", )!; delete rootEntrypoint.if; - rootEntrypoint["timeout-minutes"] = 30; + rootEntrypoint["timeout-minutes"] = 29; expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( expect.arrayContaining([ "Hermes secret boundary step must expose its outcome to the next probe", - "Hermes secret boundary must retain its 60-minute probe budget", + "Hermes secret boundary must retain its 45-minute probe budget", "Hermes root entrypoint must run after either secret-boundary outcome", - "Hermes root entrypoint must retain its 45-minute probe budget", + "Hermes root entrypoint must retain its 30-minute probe budget", ]), ); }); diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index f2482c82cd6..4cd0462b25f 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -23,6 +23,15 @@ const HERMES_SECRET_BOUNDARY_STEP_ID = "hermes-secret-boundary"; const HERMES_ROOT_AFTER_SECRET_CONDITION = "${{ !cancelled() && (steps.hermes-secret-boundary.outcome == 'success' || steps.hermes-secret-boundary.outcome == 'failure') }}"; const HERMES_EXPORT_SWAP_STEP_NAME = "Add swap for Hermes image export"; +const HERMES_SETUP_BUILDX_ACTION = + "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c"; +const HERMES_BUILD_PUSH_ACTION = + "docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a"; +const HERMES_DOWNLOAD_ARTIFACT_ACTION = + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; +const HERMES_CACHE_FROM = "type=gha,scope=hermes-production-${{ runner.os }}-${{ runner.arch }}"; +const HERMES_CACHE_TO = + "type=gha,mode=max,scope=hermes-production-${{ runner.os }}-${{ runner.arch }}"; const MESSAGING_PLAN_IMAGE_BOUNDARY_JOB = "messaging-plan-image-boundary"; const IMAGE_BUILD_JOBS = [ "build-sandbox-images", @@ -80,7 +89,7 @@ const GUARDED_PRODUCTION_BUILD_CONTRACTS: readonly GuardedProductionBuildContrac envName: "HERMES_BASE_IMAGE", jobName: "build-hermes-sandbox-image", label: "Hermes production image", - stepName: "Build Hermes production image", + stepName: "Validate Hermes production build args", target: "nemoclaw-hermes-production", }, { @@ -352,10 +361,10 @@ function validateSecretScopeAndRegistryWrites( errors.push(`${label} must not authenticate to a registry`); } } - if ( - REGISTRY_WRITE.test(run) || - String(step.uses ?? "").includes("docker/build-push-action") - ) { + const buildActionWritesRegistry = + String(step.uses ?? "").startsWith("docker/build-push-action@") && + record(step.with).push !== false; + if (REGISTRY_WRITE.test(run) || buildActionWritesRegistry) { errors.push(`${label} must not write images to a registry`); } } @@ -389,6 +398,48 @@ function validateGuardedProductionBuild( [contract.envName]: `\${{ env.${contract.envName} }}`, }; + if (contract.jobName === "build-hermes-sandbox-image") { + const expectedValidationRun = [ + "set -euo pipefail", + `build_args=(${contract.args})`, + 'scripts/check-production-build-args.sh "${build_args[@]}"', + "", + ].join("\n"); + if (!isDeepStrictEqual(record(build.env), expectedEnv) || build.run !== expectedValidationRun) { + errors.push(`${contract.label} must validate the guarded build_args shape`); + } + const setupBuildx = requireStep(errors, contract.jobName, job, "Set up Docker Buildx"); + if ( + steps(job).filter((step) => step.name === "Set up Docker Buildx").length !== 1 || + setupBuildx.uses !== HERMES_SETUP_BUILDX_ACTION + ) { + errors.push("Hermes producer must use the canonical Docker Buildx setup action exactly once"); + } + const action = requireStep(errors, contract.jobName, job, "Build Hermes production image"); + const actionWith = record(action.with); + const buildActions = steps(job).filter((step) => + String(step.uses ?? "").startsWith("docker/build-push-action@"), + ); + if ( + buildActions.length !== 1 || + dockerBuildLines(job).length !== 0 || + action.uses !== HERMES_BUILD_PUSH_ACTION || + actionWith.context !== "." || + actionWith.file !== "agents/hermes/Dockerfile" || + actionWith.load !== true || + actionWith.push !== false || + actionWith.tags !== contract.target || + actionWith["build-args"] !== "BASE_IMAGE=${{ env.HERMES_BASE_IMAGE }}" || + actionWith["cache-from"] !== HERMES_CACHE_FROM || + actionWith["cache-to"] !== HERMES_CACHE_TO + ) { + errors.push( + "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + ); + } + return; + } + if (!isDeepStrictEqual(record(build.env), expectedEnv) || build.run !== expectedRun) { errors.push(`${contract.label} must use the guarded build_args shape under ${contract.target}`); } @@ -775,39 +826,61 @@ function validateRuntimeImageReuse(errors: string[], workflow: SandboxImagesWork } function validateHermesImageReuse(errors: string[], workflow: SandboxImagesWorkflow): void { - const jobName = "build-hermes-sandbox-image"; - const job = workflow.jobs[jobName] ?? {}; - if (job["timeout-minutes"] !== 150) { - errors.push("Hermes image job timeout must cover both inherited probe budgets"); + const producerName = "build-hermes-sandbox-image"; + const producer = workflow.jobs[producerName] ?? {}; + const testJobName = "test-hermes-sandbox-image"; + const testJob = workflow.jobs[testJobName] ?? {}; + if (producer["timeout-minutes"] !== 30) { + errors.push("Hermes image producer must retain its 30-minute budget"); + } + if (testJob["timeout-minutes"] !== 90) { + errors.push("Hermes image test consumer must retain its 90-minute budget"); + } + if (testJob.needs !== producerName) { + errors.push("Hermes image tests must depend on the Hermes image producer"); + } + if (steps(testJob).some((step) => step.name === AUTH_STEP_NAME)) { + errors.push("Hermes image test consumer must not authenticate to Docker Hub"); + } + const consumerBuilds = steps(testJob).filter( + (step) => + String(step.uses ?? "").startsWith("docker/build-push-action@") || + /\bdocker\s+(?:build|buildx\s+build)\b/u.test(step.run ?? ""), + ); + if (consumerBuilds.length !== 0) { + errors.push("Hermes image test consumer must not rebuild the prebuilt image"); } for (const stepName of ["Set up Node", "Install root dependencies"]) { - if (steps(job).filter((step) => step.name === stepName).length !== 1) { - errors.push(`${jobName} must run '${stepName}' exactly once`); + if (steps(producer).some((step) => step.name === stepName)) { + errors.push(`${producerName} must not install Node dependencies`); + } + if (steps(testJob).filter((step) => step.name === stepName).length !== 1) { + errors.push(`${testJobName} must run '${stepName}' exactly once`); } } const secretBoundary = requireStep( errors, - jobName, - job, + testJobName, + testJob, "Run Hermes sandbox secret boundary test", ); const rootEntrypoint = requireStep( errors, - jobName, - job, + testJobName, + testJob, "Run Hermes root entrypoint smoke Vitest test", ); if (secretBoundary.id !== HERMES_SECRET_BOUNDARY_STEP_ID) { errors.push("Hermes secret boundary step must expose its outcome to the next probe"); } - if (secretBoundary["timeout-minutes"] !== 60) { - errors.push("Hermes secret boundary must retain its 60-minute probe budget"); + if (secretBoundary["timeout-minutes"] !== 45) { + errors.push("Hermes secret boundary must retain its 45-minute probe budget"); } if (rootEntrypoint.if !== HERMES_ROOT_AFTER_SECRET_CONDITION) { errors.push("Hermes root entrypoint must run after either secret-boundary outcome"); } - if (rootEntrypoint["timeout-minutes"] !== 45) { - errors.push("Hermes root entrypoint must retain its 45-minute probe budget"); + if (rootEntrypoint["timeout-minutes"] !== 30) { + errors.push("Hermes root entrypoint must retain its 30-minute probe budget"); } for (const [label, step, target, artifactDirectory] of [ [ @@ -839,14 +912,33 @@ function validateHermesImageReuse(errors: string[], workflow: SandboxImagesWorkf if (/\bdocker\s+build\b/u.test(step.run ?? "")) { errors.push(`${label} step must not rebuild the prebuilt image`); } - if (stepIndex(job, "Build Hermes production image") >= stepIndex(job, step.name ?? "")) { - errors.push(`${label} must run after the Hermes production image build`); + if (stepIndex(testJob, "Load Hermes production image") >= stepIndex(testJob, step.name ?? "")) { + errors.push(`${label} must run after loading the Hermes production image`); } } - const save = requireStep(errors, jobName, job, "Save Hermes production image"); + const download = requireStep(errors, testJobName, testJob, "Download Hermes production image"); + const load = requireStep(errors, testJobName, testJob, "Load Hermes production image"); + if ( + steps(testJob).filter((step) => step.name === "Download Hermes production image").length !== + 1 || + download.uses !== HERMES_DOWNLOAD_ARTIFACT_ACTION || + !isDeepStrictEqual(record(download.with), { + name: "hermes-isolation-image", + path: "/tmp", + }) || + steps(testJob).filter((step) => step.name === "Load Hermes production image").length !== 1 || + !(load.run ?? "").includes("/tmp/hermes-isolation-image.tar.gz | docker load") || + !(load.run ?? "").includes("docker image inspect nemoclaw-hermes-production") + ) { + errors.push( + "Hermes image tests must download and load the producer artifact exactly once with the canonical action", + ); + } + + const save = requireStep(errors, producerName, producer, "Save Hermes production image"); if ( - steps(job).filter((step) => step.name === "Save Hermes production image").length !== 1 || + steps(producer).filter((step) => step.name === "Save Hermes production image").length !== 1 || !(save.run ?? "").includes( "docker save nemoclaw-hermes-production | gzip > /tmp/hermes-isolation-image.tar.gz", ) || @@ -854,9 +946,9 @@ function validateHermesImageReuse(errors: string[], workflow: SandboxImagesWorkf ) { errors.push("Hermes producer must save and verify its production image exactly once"); } - const upload = requireStep(errors, jobName, job, "Upload Hermes isolation image"); + const upload = requireStep(errors, producerName, producer, "Upload Hermes isolation image"); if ( - steps(job).filter((step) => step.name === "Upload Hermes isolation image").length !== 1 || + steps(producer).filter((step) => step.name === "Upload Hermes isolation image").length !== 1 || !(upload.uses ?? "").startsWith("actions/upload-artifact@") || !FULL_SHA_ACTION.test(upload.uses ?? "") || !isDeepStrictEqual(record(upload.with), { @@ -864,8 +956,8 @@ function validateHermesImageReuse(errors: string[], workflow: SandboxImagesWorkf path: "/tmp/hermes-isolation-image.tar.gz", "retention-days": 1, }) || - stepIndex(job, save.name ?? "") >= stepIndex(job, upload.name ?? "") || - stepIndex(job, upload.name ?? "") >= stepIndex(job, CLEANUP_STEP_NAME) + stepIndex(producer, save.name ?? "") >= stepIndex(producer, upload.name ?? "") || + stepIndex(producer, upload.name ?? "") >= stepIndex(producer, CLEANUP_STEP_NAME) ) { errors.push("Hermes producer must upload the saved production image before auth cleanup"); }