From 36b15e158513983497b7f3beed351c2dc2f2594d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 22 Jul 2026 04:54:18 -0700 Subject: [PATCH] ci(hermes): add swap for image exports Signed-off-by: Aaron Erickson --- .github/workflows/sandbox-images-and-e2e.yaml | 20 +++++++++++ test/e2e/README.md | 15 ++++---- .../sandbox-images-workflow-boundary.test.ts | 16 +++++++++ .../e2e/sandbox-images-workflow-boundary.mts | 35 +++++++++++++++++++ 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 4bf87ea6ce1..f9801ba7a18 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -182,6 +182,24 @@ jobs: - name: Resolve Hermes base image uses: ./.github/actions/resolve-hermes-base-image + - &hermes-export-swap + name: Add swap for Hermes image export + shell: bash + run: | + set -euo pipefail + swap_file=/mnt/nemoclaw-hermes-image-export.swap + free -h + df -h / /mnt + swapon --show + sudo fallocate -l 32G "$swap_file" + sudo chmod 0600 "$swap_file" + sudo mkswap "$swap_file" + sudo swapon "$swap_file" + swapon --show + free -h + df -h / /mnt + docker system df + - name: Build Hermes production image env: HERMES_BASE_IMAGE: ${{ env.HERMES_BASE_IMAGE }} @@ -330,6 +348,8 @@ jobs: node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts verify \ nemoclaw-openclaw-plan-boundary openclaw + - *hermes-export-swap + - name: Build and verify Hermes messaging plan boundary env: HERMES_BASE_IMAGE: ${{ env.HERMES_BASE_IMAGE }} diff --git a/test/e2e/README.md b/test/e2e/README.md index 11bd333e9fb..b19e1f74311 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -410,13 +410,14 @@ base-build reason before the allowance applies. Published-image runs retain the normal limits, and output silence, first-turn, and all other phase requirements remain unchanged. -The two Hermes rebuild jobs add a bounded 32 GiB swap file on their ephemeral -hosted runners before invoking the live fixture. The fixture verifies that -floor and provisions the same swap file on GitHub Actions when a trusted -control-plane run uses the workflow definition from `main`. Those jobs build -both old and current Hermes image layers and can otherwise exhaust the runner's -default memory and swap during Docker layer export. Other E2E jobs keep the -standard runner memory configuration. +The two Hermes rebuild jobs and both reusable-workflow Hermes image exporters +add a bounded 32 GiB swap file on their ephemeral hosted runners before the +memory-heavy image build. The rebuild fixture verifies that floor and +provisions the same swap file on GitHub Actions when a trusted control-plane +run uses the workflow definition from `main`. Those paths build large Hermes +image layers and can otherwise exhaust the runner's default memory and swap +during Docker layer export. Other E2E jobs keep the standard runner memory +configuration. These assertions run inside the existing `full-e2e` lifecycle instead of a second standalone onboarding run. This keeps the measurement on the job's first diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 0707263e401..92b79cb85c2 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -197,6 +197,22 @@ describe("sandbox image workflow boundary", () => { ); }); + it("requires bounded swap before every hosted Hermes image export", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + for (const jobName of ["build-hermes-sandbox-image", "messaging-plan-image-boundary"]) { + const job = imageWorkflow.jobs[jobName]; + const swap = job.steps!.find((step) => step.name === "Add swap for Hermes image export")!; + swap.run = swap.run!.replace('sudo swapon "$swap_file"', 'echo "swap omitted"'); + } + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( + expect.arrayContaining([ + 'build-hermes-sandbox-image Hermes export swap must include sudo swapon "$swap_file"', + 'messaging-plan-image-boundary Hermes export swap must include sudo swapon "$swap_file"', + ]), + ); + }); + it("rejects coupling, rebuilding, or failing to reuse the OpenClaw image artifact", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); const producer = imageWorkflow.jobs["build-sandbox-images"]; diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index 3f08d353c2b..7711cad733d 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -22,6 +22,7 @@ const CLEANUP_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; const HERMES_SECRET_BOUNDARY_STEP_ID = "hermes-secret-boundary"; const HERMES_ROOT_AFTER_SECRET_CONDITION = "${{ !cancelled() && (steps.hermes-secret-boundary.outcome == 'success' || steps.hermes-secret-boundary.outcome == 'failure') }}"; +const HERMES_EXPORT_SWAP_STEP_NAME = "Add swap for Hermes image export"; const MESSAGING_PLAN_IMAGE_BOUNDARY_JOB = "messaging-plan-image-boundary"; const IMAGE_BUILD_JOBS = [ "build-sandbox-images", @@ -610,6 +611,39 @@ function validateMessagingPlanImageBoundary( } } +function validateHermesExportSwap(errors: string[], workflow: SandboxImagesWorkflow): void { + for (const [jobName, buildStepName] of [ + ["build-hermes-sandbox-image", "Build Hermes production image"], + [MESSAGING_PLAN_IMAGE_BOUNDARY_JOB, "Build and verify Hermes messaging plan boundary"], + ] as const) { + const job = workflow.jobs[jobName] ?? {}; + const swapSteps = steps(job).filter((step) => step.name === HERMES_EXPORT_SWAP_STEP_NAME); + if (swapSteps.length !== 1) { + errors.push(`${jobName} must provision Hermes export swap exactly once`); + continue; + } + const run = swapSteps[0]?.run ?? ""; + for (const fragment of [ + "swap_file=/mnt/nemoclaw-hermes-image-export.swap", + 'sudo fallocate -l 32G "$swap_file"', + 'sudo chmod 0600 "$swap_file"', + 'sudo mkswap "$swap_file"', + 'sudo swapon "$swap_file"', + "swapon --show", + "free -h", + "df -h / /mnt", + "docker system df", + ]) { + if (!run.includes(fragment)) { + errors.push(`${jobName} Hermes export swap must include ${fragment}`); + } + } + if (stepIndex(job, HERMES_EXPORT_SWAP_STEP_NAME) >= stepIndex(job, buildStepName)) { + errors.push(`${jobName} must provision swap before the Hermes image build`); + } + } +} + function validateRuntimeImageReuse(errors: string[], workflow: SandboxImagesWorkflow): void { const producerName = "build-sandbox-images"; const producer = workflow.jobs[producerName] ?? {}; @@ -969,6 +1003,7 @@ export function validateSandboxImagesWorkflow( validateSecretScopeAndRegistryWrites(errors, workflow); validateGuardedProductionBuildContracts(errors, workflow); validateNodeTarImageScans(errors, workflow); + validateHermesExportSwap(errors, workflow); validateMessagingPlanImageBoundary(errors, workflow); validateRuntimeImageReuse(errors, workflow); validateHermesImageReuse(errors, workflow);