diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index af7eab402b5..070730f9bcc 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -27,8 +27,12 @@ on: - "agents/langchain-deepagents-code/Dockerfile.base" - "agents/langchain-deepagents-code/manifest.yaml" - "agents/langchain-deepagents-code/requirements.lock" + - "agents/openclaw/mcporter-runtime/package.json" + - "agents/openclaw/mcporter-runtime/package-lock.json" # Dockerfile.base validates min_openclaw_version from this file at build time. - "nemoclaw-blueprint/blueprint.yaml" + - "scripts/lib/reviewed-npm-archive.mts" + - "scripts/lib/sandbox-rlimits.sh" workflow_dispatch: inputs: openclaw_version: @@ -123,8 +127,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max build-args: ${{ steps.production-build-args.outputs.openclaw_build_arg }} build-and-push-hermes: @@ -172,8 +176,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/nvidia/nemoclaw/hermes-sandbox-base:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/nvidia/nemoclaw/hermes-sandbox-base:buildcache,mode=max build-and-push-langchain-deepagents-code: if: github.repository == 'NVIDIA/NemoClaw' @@ -220,5 +224,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:buildcache,mode=max diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6c21643b344..9bef5c180bb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -296,6 +296,39 @@ jobs: exit 1 fi + - name: Set up DCode profile gate Buildx + id: dcode-profile-gate-buildx + if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }} + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: default-load=true + + - name: Route DCode profile gate Docker builds through Buildx + if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }} + env: + DCODE_PROFILE_GATE_BUILDER: ${{ steps.dcode-profile-gate-buildx.outputs.name }} + shell: bash + run: | + set -euo pipefail + test -n "${DCODE_PROFILE_GATE_BUILDER}" + printf 'BUILDX_BUILDER=%s\n' "${DCODE_PROFILE_GATE_BUILDER}" >> "${GITHUB_ENV}" + + - name: Validate DCode profile gate production Docker build args + if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }} + run: scripts/check-production-build-args.sh + + # Import the trusted publisher cache without allowing PR-controlled E2E + # builds to replace it. The negative gate still executes every reviewed + # Dockerfile and proves the expected production-image failure. + - name: Warm DCode profile gate base build cache + if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }} + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + builder: ${{ steps.dcode-profile-gate-buildx.outputs.name }} + context: . + file: agents/langchain-deepagents-code/Dockerfile.base + cache-from: type=registry,ref=ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:buildcache + - name: Configure live E2E trace directory env: TARGET_ID: ${{ matrix.id }} @@ -2335,6 +2368,36 @@ jobs: - *dockerhub-auth + - &rebuild-buildx + name: Set up rebuild Buildx + id: rebuild-buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: default-load=true + + - &route-rebuild-builds + name: Route rebuild Docker builds through Buildx + env: + REBUILD_BUILDER: ${{ steps.rebuild-buildx.outputs.name }} + shell: bash + run: | + set -euo pipefail + test -n "${REBUILD_BUILDER}" + printf 'BUILDX_BUILDER=%s\n' "${REBUILD_BUILDER}" >> "${GITHUB_ENV}" + + - name: Validate OpenClaw rebuild production Docker build args + run: scripts/check-production-build-args.sh + + # This job may build PR-controlled source, so it imports the trusted + # publisher cache read-only and keeps any new layers job-local. + - name: Warm current OpenClaw base build cache + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + builder: ${{ steps.rebuild-buildx.outputs.name }} + context: . + file: Dockerfile.base + cache-from: type=registry,ref=ghcr.io/nvidia/nemoclaw/sandbox-base:buildcache + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 @@ -2406,6 +2469,23 @@ jobs: - *dockerhub-auth + - *rebuild-buildx + + - *route-rebuild-builds + + - &validate-hermes-rebuild-build-args + name: Validate Hermes rebuild production Docker build args + run: scripts/check-production-build-args.sh + + - &warm-current-hermes-base + name: Warm current Hermes base build cache + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + builder: ${{ steps.rebuild-buildx.outputs.name }} + context: . + file: agents/hermes/Dockerfile.base + cache-from: type=registry,ref=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:buildcache + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 with: @@ -2462,6 +2542,14 @@ jobs: - *dockerhub-auth + - *rebuild-buildx + + - *route-rebuild-builds + + - *validate-hermes-rebuild-build-args + + - *warm-current-hermes-base + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 with: diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index b16616fb1de..da37f38b30a 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -49,6 +49,7 @@ describe("sandbox BuildKit prebuild", () => { vi.stubEnv("DOCKER_HOST", "unix:///var/run/docker.sock"); vi.stubEnv("DOCKER_CONFIG", "/home/user/.docker-ci"); vi.stubEnv("DOCKER_CONTEXT", "remote-builder"); + vi.stubEnv("BUILDX_BUILDER", "cache-builder"); vi.stubEnv("XDG_CONFIG_HOME", "/home/user/.config"); vi.stubEnv("HTTPS_PROXY", "http://proxy:8080"); vi.stubEnv("NVIDIA_INFERENCE_API_KEY", "secret"); @@ -68,6 +69,7 @@ describe("sandbox BuildKit prebuild", () => { DOCKER_HOST: "unix:///var/run/docker.sock", DOCKER_CONFIG: "/home/user/.docker-ci", DOCKER_CONTEXT: "remote-builder", + BUILDX_BUILDER: "cache-builder", XDG_CONFIG_HOME: "/home/user/.config", HTTPS_PROXY: "http://proxy:8080", }); diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 5ed957227e0..699336ac328 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -19,6 +19,7 @@ const TRUTHY_FLAG_VALUES = new Set(["1", "true", "yes", "on"]); const FALSY_FLAG_VALUES = new Set(["0", "false", "no", "off"]); const LOCAL_IMAGE_REPO = LOCAL_SANDBOX_IMAGE_REPO; const DOCKER_ENV_NAMES = [ + "BUILDX_BUILDER", "DOCKER_API_VERSION", "DOCKER_CERT_PATH", "DOCKER_CONFIG", diff --git a/src/lib/sandbox-base-image/resolution-key.test.ts b/src/lib/sandbox-base-image/resolution-key.test.ts index dcf264cc83a..a150c4785b1 100644 --- a/src/lib/sandbox-base-image/resolution-key.test.ts +++ b/src/lib/sandbox-base-image/resolution-key.test.ts @@ -64,6 +64,18 @@ describe("sandbox base-image resolution key", () => { expect(createSandboxBaseImageResolutionKey(options(root))).not.toBe(before); }); + it("changes when a Dockerfile-copied runtime helper changes", () => { + const root = fixture(); + const helper = path.join(root, "scripts", "lib", "sandbox-rlimits.sh"); + fs.mkdirSync(path.dirname(helper), { recursive: true }); + fs.writeFileSync(helper, "ulimit -n 1024\n"); + const before = createSandboxBaseImageResolutionKey(options(root)); + + fs.writeFileSync(helper, "ulimit -n 2048\n"); + + expect(createSandboxBaseImageResolutionKey(options(root))).not.toBe(before); + }); + it("changes when an agent-specific dependency lock changes (#6456)", () => { const root = fixture(); const lockfile = path.join(root, "agents", "langchain-deepagents-code", "requirements.lock"); diff --git a/src/lib/sandbox-base-image/source-identity.test.ts b/src/lib/sandbox-base-image/source-identity.test.ts index 7a3fe2b9b0a..b62d77a6a5d 100644 --- a/src/lib/sandbox-base-image/source-identity.test.ts +++ b/src/lib/sandbox-base-image/source-identity.test.ts @@ -135,7 +135,15 @@ describe("sandbox base-image source identity", () => { "Dockerfile.base", "../outside/Dockerfile.base", ]), - ).toEqual(["Dockerfile.base", "nemoclaw-blueprint/blueprint.yaml", agentDockerfile]); + ).toEqual([ + "Dockerfile.base", + "nemoclaw-blueprint/blueprint.yaml", + "scripts/lib/sandbox-rlimits.sh", + "agents/openclaw/mcporter-runtime/package.json", + "agents/openclaw/mcporter-runtime/package-lock.json", + "scripts/lib/reviewed-npm-archive.mts", + agentDockerfile, + ]); }); it("builds deterministic local tags from a source SHA and falls back without one", () => { diff --git a/src/lib/sandbox-base-image/source-identity.ts b/src/lib/sandbox-base-image/source-identity.ts index 91309745b43..97526770954 100644 --- a/src/lib/sandbox-base-image/source-identity.ts +++ b/src/lib/sandbox-base-image/source-identity.ts @@ -7,7 +7,14 @@ import path from "node:path"; import { ROOT } from "../runner"; -export const BASE_IMAGE_INPUT_PATHS = ["Dockerfile.base", "nemoclaw-blueprint/blueprint.yaml"]; +export const BASE_IMAGE_INPUT_PATHS = [ + "Dockerfile.base", + "nemoclaw-blueprint/blueprint.yaml", + "scripts/lib/sandbox-rlimits.sh", + "agents/openclaw/mcporter-runtime/package.json", + "agents/openclaw/mcporter-runtime/package-lock.json", + "scripts/lib/reviewed-npm-archive.mts", +]; export function normalizeBaseImageInputPaths(rootDir: string, paths: string[] = []): string[] { const absoluteRootDir = path.resolve(rootDir); diff --git a/test/dcode-base-image-workflow.test.ts b/test/dcode-base-image-workflow.test.ts index 94a1938e431..25db982f077 100644 --- a/test/dcode-base-image-workflow.test.ts +++ b/test/dcode-base-image-workflow.test.ts @@ -35,6 +35,11 @@ type Publisher = { dockerfile: string; }; +type RegistryCacheEntry = { + mode?: string; + ref?: string; +}; + const repoRoot = path.resolve(import.meta.dirname, ".."); const workflow = YAML.parse( fs.readFileSync(path.join(repoRoot, ".github", "workflows", "base-image.yaml"), "utf8"), @@ -57,17 +62,43 @@ function publisherJobs(candidate: Workflow): Publisher[] { }); } -function copiedLocks(dockerfile: string): string[] { +function copiedInputs(dockerfile: string): string[] { return [ ...fs .readFileSync(path.join(repoRoot, dockerfile), "utf8") - .matchAll(/^COPY\s+(\S+\.lock)\s+/gm), - ].map(([, lock]) => lock); + .matchAll(/^COPY\s+(?!--from=)(?:--\S+\s+)*(\S+)\s+\S+/gm), + ].map(([, input]) => input); +} + +function copiedLocks(dockerfile: string): string[] { + return copiedInputs(dockerfile).filter((input) => input.endsWith(".lock")); +} + +function registryCacheEntries(value: unknown): RegistryCacheEntry[] { + return String(value ?? "") + .split(/\r?\n/u) + .map((entry) => entry.trim()) + .filter((entry) => entry.split(",").includes("type=registry")) + .map((entry) => + Object.fromEntries( + entry + .split(",") + .filter((field) => field !== "type=registry") + .map((field) => field.split("=", 2) as [string, string]), + ), + ); } function validatePublishers(candidate: Workflow): string[] { const triggerPaths = candidate.on?.push?.paths ?? []; - return publisherJobs(candidate).flatMap(({ jobName, job, build, buildIndex, dockerfile }) => { + const publishers = publisherJobs(candidate); + const exportedCacheRefCounts = new Map(); + for (const { build } of publishers) { + const cacheRef = registryCacheEntries(build.with?.["cache-to"])[0]?.ref ?? ""; + exportedCacheRefCounts.set(cacheRef, (exportedCacheRefCounts.get(cacheRef) ?? 0) + 1); + } + + return publishers.flatMap(({ jobName, job, build, buildIndex, dockerfile }) => { const steps = job.steps ?? []; const metadata = steps.find((step) => step.id === "meta"); const guardIndex = steps.findIndex((step) => @@ -75,27 +106,31 @@ function validatePublishers(candidate: Workflow): string[] { ); const dockerfileExists = dockerfile.length > 0 && fs.existsSync(path.join(repoRoot, dockerfile)); - const copiedLockPaths = dockerfileExists ? copiedLocks(dockerfile) : []; + const copiedInputPaths = dockerfileExists ? copiedInputs(dockerfile) : []; const dockerActions = steps.filter((step) => step.uses?.startsWith("docker/")); const tags = String(metadata?.with?.tags ?? ""); + const metadataImage = String(metadata?.with?.images ?? ""); + const expectedCacheRef = `${metadataImage}:buildcache`; + const cacheFrom = registryCacheEntries(build.with?.["cache-from"]); + const cacheTo = registryCacheEntries(build.with?.["cache-to"]); + const importedCacheRef = cacheFrom[0]?.ref; + const exportedCacheRef = cacheTo[0]?.ref; return [ ...(!dockerfileExists ? [`${jobName} must publish from an existing Dockerfile`] : []), ...(!triggerPaths.includes(dockerfile) ? [`${jobName} Dockerfile must trigger the publisher workflow`] : []), - ...copiedLockPaths - .filter((lock) => !triggerPaths.includes(lock)) - .map((lock) => `${jobName} copied lockfile must trigger the publisher workflow: ${lock}`), + ...copiedInputPaths + .filter((input) => !triggerPaths.includes(input)) + .map((input) => `${jobName} copied input must trigger the publisher workflow: ${input}`), ...(guardIndex < 0 || guardIndex >= buildIndex ? [`${jobName} must validate production build args before publishing`] : []), ...(!metadata?.uses?.startsWith("docker/metadata-action@") ? [`${jobName} must derive publication metadata with docker/metadata-action`] : []), - ...(String(metadata?.with?.images ?? "").length === 0 - ? [`${jobName} must declare a publication image`] - : []), + ...(metadataImage.length === 0 ? [`${jobName} must declare a publication image`] : []), ...(!tags.includes("type=ref,event=tag") || !tags.includes("type=raw,value=latest") || !tags.includes("type=sha,prefix=,format=short") @@ -116,6 +151,24 @@ function validatePublishers(candidate: Workflow): string[] { build.with?.labels !== "${{ steps.meta.outputs.labels }}" ? [`${jobName} must publish the reviewed metadata outputs`] : []), + ...(cacheFrom.length !== 1 || !importedCacheRef + ? [`${jobName} cache-from must declare exactly one registry cache ref`] + : []), + ...(cacheTo.length !== 1 || !exportedCacheRef + ? [`${jobName} cache-to must declare exactly one registry cache ref`] + : []), + ...(importedCacheRef !== exportedCacheRef + ? [`${jobName} must import and export the same registry cache ref`] + : []), + ...(cacheTo[0]?.mode !== "max" + ? [`${jobName} must export its registry cache in max mode`] + : []), + ...(exportedCacheRef && exportedCacheRef !== expectedCacheRef + ? [`${jobName} registry cache must use its publication image buildcache tag`] + : []), + ...(exportedCacheRef && exportedCacheRefCounts.get(exportedCacheRef) !== 1 + ? [`${jobName} must use a publisher-unique registry cache ref`] + : []), ]; }); } @@ -128,7 +181,7 @@ function pinnedAptVersion(dockerfile: string, packageName: string): string { } describe("base-image publication behavior", () => { - // source-shape-contract: security -- Publisher mutations must preserve immutable actions and guarded production build arguments + // source-shape-contract: security -- Publisher mutations must preserve immutable actions, guarded arguments, and trusted registry cache ownership it("accepts every discovered publisher and rejects supply-chain mutations", () => { const publishers = publisherJobs(workflow); expect(publishers.length).toBeGreaterThan(0); @@ -137,11 +190,18 @@ describe("base-image publication behavior", () => { const mutated = structuredClone(workflow); const mutatedPublisher = publisherJobs(mutated)[0]; const mutatedSteps = mutatedPublisher.job.steps ?? []; + const otherPublisher = publisherJobs(mutated)[1]; + const otherCacheRef = registryCacheEntries(otherPublisher.build.with?.["cache-to"])[0]?.ref; const mutatedGuard = mutatedSteps.find((step) => (step.run ?? "").includes("scripts/check-production-build-args.sh"), ); mutatedPublisher.build.uses = "docker/build-push-action@v7"; - mutatedPublisher.build.with = { ...mutatedPublisher.build.with, push: false }; + mutatedPublisher.build.with = { + ...mutatedPublisher.build.with, + push: false, + "cache-from": "type=gha", + "cache-to": `type=registry,ref=${otherCacheRef}`, + }; mutatedGuard!.run = "true"; expect(validatePublishers(mutated)).toEqual( @@ -150,6 +210,11 @@ describe("base-image publication behavior", () => { `${mutatedPublisher.jobName} Docker action must use a full commit SHA: docker/build-push-action@v7`, `${mutatedPublisher.jobName} build-push action must use a full commit SHA`, `${mutatedPublisher.jobName} must push the built image`, + `${mutatedPublisher.jobName} cache-from must declare exactly one registry cache ref`, + `${mutatedPublisher.jobName} must import and export the same registry cache ref`, + `${mutatedPublisher.jobName} must export its registry cache in max mode`, + `${mutatedPublisher.jobName} registry cache must use its publication image buildcache tag`, + `${mutatedPublisher.jobName} must use a publisher-unique registry cache ref`, ]), ); }); diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index f3f83c1ba9c..53dcd3e7c41 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -10,6 +10,7 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_API_VERSION", + "BUILDX_BUILDER", "XDG_RUNTIME_DIR", "NEMOCLAW_OLLAMA_PULL_TIMEOUT", "NEMOCLAW_TRACE_DIR", diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index b52752cbf32..2697c11be90 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -36,6 +36,78 @@ describe("e2e workflow boundary", () => { expect(validateE2eWorkflowBoundary()).toEqual([]); }); + it("rejects rebuild and DCode cache wiring drift", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-build-cache-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { + steps: Array<{ + env?: Record; + name?: string; + uses?: string; + with?: Record; + }>; + } + >; + }; + type Step = (typeof workflow.jobs)[string]["steps"][number]; + const missingStep = (job: string, name: string): never => { + throw new Error(`Missing workflow step in ${job}: ${name}`); + }; + const cloneStep = (job: string, name: string): Step => { + const steps = workflow.jobs[job].steps; + const index = steps.findIndex((step) => step.name === name); + const clone = structuredClone(steps[index] ?? missingStep(job, name)); + steps[index] = clone; + return clone; + }; + + const openClawSetup = cloneStep("rebuild-openclaw", "Set up rebuild Buildx"); + const openClawRoute = cloneStep( + "rebuild-openclaw", + "Route rebuild Docker builds through Buildx", + ); + const openClawWarm = cloneStep("rebuild-openclaw", "Warm current OpenClaw base build cache"); + openClawSetup.with!["driver-opts"] = "network=host"; + openClawRoute.env!.REBUILD_BUILDER = "default"; + openClawWarm.with!["cache-from"] = "type=gha,scope=buildkit"; + openClawWarm.with!["cache-to"] = "type=registry,ref=example.invalid/cache"; + + const hermesWarm = cloneStep("rebuild-hermes", "Warm current Hermes base build cache"); + hermesWarm.with!["cache-from"] = "type=gha,scope=buildkit"; + hermesWarm.with!["cache-to"] = "type=registry,ref=example.invalid/cache"; + + const dcodeSetup = cloneStep("live", "Set up DCode profile gate Buildx"); + const dcodeRoute = cloneStep("live", "Route DCode profile gate Docker builds through Buildx"); + const dcodeWarm = cloneStep("live", "Warm DCode profile gate base build cache"); + dcodeSetup.with!["driver-opts"] = "network=host"; + dcodeRoute.env!.DCODE_PROFILE_GATE_BUILDER = "default"; + dcodeWarm.with!["cache-from"] = "type=gha,scope=buildkit"; + dcodeWarm.with!["cache-to"] = "type=registry,ref=example.invalid/cache"; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "rebuild-openclaw Buildx must enable default-load for the live Docker builds", + "rebuild-openclaw must route Docker builds to the configured Buildx builder", + "rebuild-openclaw base cache must import the trusted publisher cache", + "rebuild-openclaw must keep PR-controlled cache layers job-local", + "rebuild-hermes base cache must import the trusted publisher cache", + "rebuild-hermes must keep PR-controlled cache layers job-local", + "live DCode Buildx must enable default-load for the gate Docker builds", + "live DCode gate must route Docker builds to the configured Buildx builder", + "live DCode cache warm must import the trusted publisher cache", + "live DCode cache warm must keep PR-controlled cache layers job-local", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + // source-shape-contract: security -- Mutates the shipped workflow to prove PR-safe routing rejects credential-backed smokes it("rejects credential-backed provider smokes in the PR-safe inference-routing job", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inference-routing-workflow-")); diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index 522c755ab11..5462c891a2b 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -206,6 +206,20 @@ describe("hosted inference E2E config", () => { expect(env).not.toHaveProperty("RANDOM_NON_SECRET"); }); + it("preserves a selected Buildx builder without widening the fixture environment", () => { + const env = buildAvailabilityProbeEnv({ + HOME: "/tmp/home", + PATH: "/usr/bin", + BUILDX_BUILDER: "e2e-cache-builder", + GITHUB_TOKEN: "must-not-reach-child", + RANDOM_NON_SECRET: "not-allowlisted", + }); + + expect(env.BUILDX_BUILDER).toBe("e2e-cache-builder"); + expect(env).not.toHaveProperty("GITHUB_TOKEN"); + expect(env).not.toHaveProperty("RANDOM_NON_SECRET"); + }); + it("builds provider reachability probes only from trusted endpoints", async () => { const calls: Array<{ command: TrustedShellCommand; options?: ShellProbeRunOptions }> = []; const provider = providerClientWithCalls(calls); diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index eaaaba4ddc6..4a4e23777ff 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -1085,13 +1085,14 @@ exit 0`, expect(result.stdout).not.toContain("Installing OpenShell from release 'dev'"); }); - it("preserves the rebuild Hermes requested channel through the real installer boundary", () => { + it("preserves rebuild Hermes workflow inputs through the real installer boundary", () => { const childEnv = buildRebuildHermesChildEnv( { HOME: process.env.HOME, PATH: process.env.PATH, NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", NEMOCLAW_OPENSHELL_CHANNEL: "dev", + BUILDX_BUILDER: "hermes-cache-builder", NVIDIA_API_KEY: "must-not-reach-child", }, {}, @@ -1100,6 +1101,7 @@ exit 0`, expect(childEnv.NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL).toBe("1"); expect(childEnv.NEMOCLAW_OPENSHELL_CHANNEL).toBe("dev"); + expect(childEnv.BUILDX_BUILDER).toBe("hermes-cache-builder"); expect(childEnv.NVIDIA_API_KEY).toBeUndefined(); expect(result.status).not.toBe(0); expect(result.stdout).toContain("Installing OpenShell from release 'dev'"); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 9a5c4ee7436..3fbb23cb669 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -436,6 +436,84 @@ function requireJobStep( return step; } +function requireReadOnlyBuildCacheImports( + errors: string[], + steps: readonly WorkflowStep[], + boundary: string, +): void { + for (const step of steps.filter((candidate) => + stringValue(candidate.uses).startsWith("docker/build-push-action@"), + )) { + const inputs = asRecord(step.with); + if (Object.hasOwn(inputs, "cache-to") || inputs.push === true) { + errors.push(`${boundary} must keep PR-controlled cache layers job-local`); + } + } +} + +function validateRebuildBaseCache( + errors: string[], + jobName: string, + steps: readonly WorkflowStep[], + options: { agentName: "Hermes" | "OpenClaw"; cacheRef: string; dockerfile: string }, +): Array { + requireReadOnlyBuildCacheImports(errors, steps, jobName); + const setupBuildx = requireJobStep(errors, jobName, steps, "Set up rebuild Buildx"); + if (setupBuildx?.uses !== "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c") { + errors.push(`${jobName} must use the reviewed rebuild Buildx action`); + } + if (setupBuildx?.id !== "rebuild-buildx") { + errors.push(`${jobName} Buildx setup must expose the rebuild-buildx step id`); + } + if (asRecord(setupBuildx?.with)["driver-opts"] !== "default-load=true") { + errors.push(`${jobName} Buildx must enable default-load for the live Docker builds`); + } + + const routeBuilds = requireJobStep( + errors, + jobName, + steps, + "Route rebuild Docker builds through Buildx", + ); + if (asRecord(routeBuilds?.env).REBUILD_BUILDER !== "${{ steps.rebuild-buildx.outputs.name }}") { + errors.push(`${jobName} must route Docker builds to the configured Buildx builder`); + } + requireRunContains(errors, routeBuilds, "test -n"); + requireRunContains(errors, routeBuilds, "BUILDX_BUILDER=%s"); + requireRunContains(errors, routeBuilds, '>> "${GITHUB_ENV}"'); + + const validateBuildArgs = requireJobStep( + errors, + jobName, + steps, + `Validate ${options.agentName} rebuild production Docker build args`, + ); + requireRunContains(errors, validateBuildArgs, "scripts/check-production-build-args.sh"); + + const warmCurrentBase = requireJobStep( + errors, + jobName, + steps, + `Warm current ${options.agentName} base build cache`, + ); + if ( + warmCurrentBase?.uses !== "docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a" + ) { + errors.push(`${jobName} must warm the current base with the reviewed action`); + } + const warmInputs = asRecord(warmCurrentBase?.with); + if (warmInputs.builder !== "${{ steps.rebuild-buildx.outputs.name }}") { + errors.push(`${jobName} base cache must use the routed Buildx builder`); + } + if (warmInputs.context !== "." || warmInputs.file !== options.dockerfile) { + errors.push(`${jobName} base cache must build the reviewed ${options.agentName} Dockerfile`); + } + if (warmInputs["cache-from"] !== options.cacheRef) { + errors.push(`${jobName} base cache must import the trusted publisher cache`); + } + return [setupBuildx, routeBuilds, validateBuildArgs, warmCurrentBase]; +} + function requireRunContains( errors: string[], step: WorkflowStep | undefined, @@ -1279,6 +1357,12 @@ function validateRebuildOpenClawJob(errors: string[], jobs: WorkflowRecord): voi errors.push("rebuild-openclaw checkout step must set persist-credentials=false"); } + const cacheSteps = validateRebuildBaseCache(errors, jobName, steps, { + agentName: "OpenClaw", + cacheRef: "type=registry,ref=ghcr.io/nvidia/nemoclaw/sandbox-base:buildcache", + dockerfile: "Dockerfile.base", + }); + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); requireEnvDoesNotExposeSecret( errors, @@ -1301,6 +1385,22 @@ function validateRebuildOpenClawJob(errors: string[], jobs: WorkflowRecord): voi requireRunContains(errors, runVitest, "OPENSHELL_BIN"); requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); requireRunContains(errors, runVitest, "test/e2e/live/rebuild-openclaw.test.ts"); + + const prepareWorkspace = requireJobStep(errors, jobName, steps, "Prepare E2E workspace"); + const orderedSteps = [...cacheSteps, prepareWorkspace, installOpenShell, runVitest]; + if ( + orderedSteps.every((step) => step !== undefined) && + orderedSteps.some( + (step, index) => + index > 0 && + steps.indexOf(orderedSteps[index - 1] as WorkflowStep) >= + steps.indexOf(step as WorkflowStep), + ) + ) { + errors.push( + "rebuild-openclaw Buildx setup, cache warming, workspace prep, and test must stay in order", + ); + } } function validateRebuildHermesJob( @@ -1393,6 +1493,12 @@ function validateRebuildHermesJob( errors.push(`${jobName} checkout step must set persist-credentials=false`); } + const cacheSteps = validateRebuildBaseCache(errors, jobName, steps, { + agentName: "Hermes", + cacheRef: "type=registry,ref=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:buildcache", + dockerfile: "agents/hermes/Dockerfile.base", + }); + const runVitest = requireJobStep( errors, jobName, @@ -1405,6 +1511,22 @@ function validateRebuildHermesJob( } requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); requireRunContains(errors, runVitest, "test/e2e/live/rebuild-hermes.test.ts"); + + const prepareWorkspace = requireJobStep(errors, jobName, steps, "Prepare E2E workspace"); + const orderedSteps = [...cacheSteps, prepareWorkspace, runVitest]; + if ( + orderedSteps.every((step) => step !== undefined) && + orderedSteps.some( + (step, index) => + index > 0 && + steps.indexOf(orderedSteps[index - 1] as WorkflowStep) >= + steps.indexOf(step as WorkflowStep), + ) + ) { + errors.push( + `${jobName} Buildx setup, cache warming, workspace prep, and test must stay in order`, + ); + } } function validateSandboxRebuildJob(errors: string[], jobs: WorkflowRecord): void { @@ -3816,6 +3938,64 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push("checkout step must set persist-credentials=false"); } + const dcodeTargetIf = "${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}"; + requireReadOnlyBuildCacheImports(errors, steps, "live DCode cache warm"); + const dcodeBuildx = requireStep(errors, steps, "Set up DCode profile gate Buildx"); + if (dcodeBuildx?.uses !== "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c") { + errors.push("live DCode profile gate must use the reviewed Buildx action"); + } + if (dcodeBuildx?.id !== "dcode-profile-gate-buildx") { + errors.push("live DCode Buildx setup must expose the dcode-profile-gate-buildx step id"); + } + if (dcodeBuildx?.if !== dcodeTargetIf) { + errors.push("live DCode Buildx setup must be scoped to the typed DCode target"); + } + if (asRecord(dcodeBuildx?.with)["driver-opts"] !== "default-load=true") { + errors.push("live DCode Buildx must enable default-load for the gate Docker builds"); + } + + const routeDcodeBuilds = requireStep( + errors, + steps, + "Route DCode profile gate Docker builds through Buildx", + ); + if (routeDcodeBuilds?.if !== dcodeTargetIf) { + errors.push("live DCode builder routing must be scoped to the typed DCode target"); + } + if ( + asRecord(routeDcodeBuilds?.env).DCODE_PROFILE_GATE_BUILDER !== + "${{ steps.dcode-profile-gate-buildx.outputs.name }}" + ) { + errors.push("live DCode gate must route Docker builds to the configured Buildx builder"); + } + requireRunContains(errors, routeDcodeBuilds, "test -n"); + requireRunContains(errors, routeDcodeBuilds, "BUILDX_BUILDER=%s"); + requireRunContains(errors, routeDcodeBuilds, '>> "${GITHUB_ENV}"'); + + const warmDcodeBase = requireStep(errors, steps, "Warm DCode profile gate base build cache"); + if (warmDcodeBase?.if !== dcodeTargetIf) { + errors.push("live DCode cache warm must be scoped to the typed DCode target"); + } + if (warmDcodeBase?.uses !== "docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a") { + errors.push("live DCode cache warm must use the reviewed build action"); + } + const warmDcodeInputs = asRecord(warmDcodeBase?.with); + if (warmDcodeInputs.builder !== "${{ steps.dcode-profile-gate-buildx.outputs.name }}") { + errors.push("live DCode cache warm must use the routed Buildx builder"); + } + if ( + warmDcodeInputs.context !== "." || + warmDcodeInputs.file !== "agents/langchain-deepagents-code/Dockerfile.base" + ) { + errors.push("live DCode cache warm must build the reviewed base Dockerfile"); + } + if ( + warmDcodeInputs["cache-from"] !== + "type=registry,ref=ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:buildcache" + ) { + errors.push("live DCode cache warm must import the trusted publisher cache"); + } + const configureTrace = requireStep(errors, steps, "Configure live E2E trace directory"); const configureTraceEnv = asRecord(configureTrace?.env); if (configureTraceEnv.TARGET_ID !== "${{ matrix.id }}") { @@ -3843,10 +4023,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { "Install Deep Agents Code TUI host dependencies", ["expect"], ); - if ( - dcodeHostDependencies?.if !== - "${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}" - ) { + if (dcodeHostDependencies?.if !== dcodeTargetIf) { errors.push("live DCode TUI host dependencies must be scoped to the typed DCode target"); } @@ -3871,10 +4048,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { "live DCode profile import gate must build the reviewed repository base without an override", ); } - if ( - dcodeProfileImportGate?.["if"] !== - "${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}" - ) { + if (dcodeProfileImportGate?.["if"] !== dcodeTargetIf) { errors.push("live DCode profile import gate must be scoped to the typed DCode target"); } if (dcodeProfileImportGate?.shell !== "bash") { @@ -3902,6 +4076,24 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { ) { errors.push("live DCode profile import gate must run before live E2E tests"); } + const dcodeOrderedSteps = [ + dcodeBuildx, + routeDcodeBuilds, + warmDcodeBase, + prepareWorkspace, + dcodeProfileImportGate, + ]; + if ( + dcodeOrderedSteps.every((step) => step !== undefined) && + dcodeOrderedSteps.some( + (step, index) => + index > 0 && + steps.indexOf(dcodeOrderedSteps[index - 1] as WorkflowStep) >= + steps.indexOf(step as WorkflowStep), + ) + ) { + errors.push("live DCode Buildx setup, cache warm, workspace prep, and gate must stay in order"); + } const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.TARGET_ID !== "${{ matrix.id }}") { errors.push("live E2E step must pass matrix.id through TARGET_ID env");