diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 44f29301f0f..8c4b8a9b3c2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -84,7 +84,60 @@ env: NEMOCLAW_E2E_SHARD: default jobs: + base-image-publication: + runs-on: ubuntu-latest + timeout-minutes: 55 + permissions: + actions: read + contents: read + steps: + - id: publication_mode + name: Classify base-image publication requirement + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + EVENT_NAME: ${{ github.event_name }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in + NVIDIA/NemoClaw:refs/heads/main:schedule:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) + required=1 + ;; + NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:controller) + required=0 + ;; + *) + echo "::error::base-image publication mode is not trusted" >&2 + exit 1 + ;; + esac + printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}" + + - name: Check out trusted E2E workflow + if: ${{ steps.publication_mode.outputs.required == '1' }} + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node for publication verification + if: ${{ steps.publication_mode.outputs.required == '1' }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + + - name: Verify applicable base-image publication + if: ${{ steps.publication_mode.outputs.required == '1' }} + env: + EXPECTED_SHA: ${{ github.sha }} + GITHUB_TOKEN: ${{ github.token }} + run: node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30 + generate-matrix: + needs: base-image-publication runs-on: ubuntu-latest outputs: matrix: ${{ steps.controller_matrix.outputs.matrix || steps.matrix.outputs.matrix }} @@ -4932,6 +4985,7 @@ jobs: # skipped checks to the normal pull_request workflow. needs: &e2e-result-jobs [ + base-image-publication, generate-matrix, live, shared-e2e, diff --git a/test/e2e/support/base-image-publication-workflow-boundary.test.ts b/test/e2e/support/base-image-publication-workflow-boundary.test.ts new file mode 100644 index 00000000000..07a8b81e1a9 --- /dev/null +++ b/test/e2e/support/base-image-publication-workflow-boundary.test.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + type OperationsWorkflow, + validateBaseImagePublicationGate, +} from "../../../tools/e2e/operations-workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type MutableStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type MutableJob = Record & { + needs?: unknown; + permissions?: Record; + steps?: MutableStep[]; +}; + +type MutableWorkflow = { + jobs: Record; +}; + +function workflow(): MutableWorkflow { + return structuredClone(readWorkflow()) as MutableWorkflow; +} + +function validate(value: MutableWorkflow): string[] { + return validateBaseImagePublicationGate(value as unknown as OperationsWorkflow); +} + +function required(value: T | undefined, message: string): T { + return ( + value ?? + (() => { + throw new Error(message); + })() + ); +} + +function gateSteps(value: MutableWorkflow): MutableStep[] { + return required( + value.jobs["base-image-publication"]?.steps, + "base-image-publication test fixture is missing steps", + ); +} + +function runClassifier(environment: { + checkoutSha: string; + eventName: string; + ref: string; + repository: string; +}): { output: string; status: number | null } { + const source = required( + gateSteps(workflow())[0]?.run, + "publication classifier fixture is missing its script", + ); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-publication-mode-")); + const outputPath = path.join(directory, "github-output"); + try { + const result = spawnSync("/bin/bash", ["-c", source], { + encoding: "utf8", + env: { + CHECKOUT_SHA: environment.checkoutSha, + EVENT_NAME: environment.eventName, + GITHUB_OUTPUT: outputPath, + REF: environment.ref, + REPOSITORY: environment.repository, + }, + }); + return { + output: fs.existsSync(outputPath) ? fs.readFileSync(outputPath, "utf8") : "", + status: result.status, + }; + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +describe("base-image publication workflow boundary (#7372)", () => { + it.each([ + ["scheduled main", "schedule", "", "1"], + ["manual main", "workflow_dispatch", "", "1"], + ["controller-selected PR", "workflow_dispatch", "a".repeat(40), "0"], + ])("classifies %s without executing untrusted code (#7372)", (_case, eventName, checkoutSha, required) => { + expect( + runClassifier({ + checkoutSha, + eventName, + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }), + ).toEqual({ output: `required=${required}\n`, status: 0 }); + }); + + it.each([ + ["a fork", "schedule", "", "refs/heads/main", "attacker/NemoClaw"], + ["a non-main ref", "schedule", "", "refs/heads/release", "NVIDIA/NemoClaw"], + ["an unexpected event", "pull_request", "", "refs/heads/main", "NVIDIA/NemoClaw"], + [ + "a scheduled controller checkout", + "schedule", + "a".repeat(40), + "refs/heads/main", + "NVIDIA/NemoClaw", + ], + ])("rejects %s instead of skipping the gate (#7372)", (_case, eventName, checkoutSha, ref, repository) => { + expect(runClassifier({ checkoutSha, eventName, ref, repository }).status).not.toBe(0); + }); + + const mutations: Array<[string, (value: MutableWorkflow) => void]> = [ + ["runner size", (value) => (value.jobs["base-image-publication"]["runs-on"] = "self-hosted")], + ["timeout", (value) => (value.jobs["base-image-publication"]["timeout-minutes"] = 60)], + [ + "permissions", + (value) => { + value.jobs["base-image-publication"].permissions!.actions = "write"; + }, + ], + [ + "failure tolerance", + (value) => (value.jobs["base-image-publication"]["continue-on-error"] = true), + ], + [ + "classifier context", + (value) => { + gateSteps(value)[0].env!.REPOSITORY = "${{ github.actor }}"; + }, + ], + [ + "classifier outcome", + (value) => { + gateSteps(value)[0].run = gateSteps(value)[0].run!.replace("required=0", "required=1"); + }, + ], + ["checkout condition", (value) => (gateSteps(value)[1].if = "${{ always() }}")], + ["checkout pin", (value) => (gateSteps(value)[1].uses = "actions/checkout@v6")], + ["checkout ref", (value) => (gateSteps(value)[1].with!.ref = "${{ inputs.checkout_sha }}")], + ["checkout history", (value) => (gateSteps(value)[1].with!["fetch-depth"] = 1)], + ["checkout credentials", (value) => (gateSteps(value)[1].with!["persist-credentials"] = true)], + ["Node condition", (value) => (gateSteps(value)[2].if = "${{ always() }}")], + ["Node pin", (value) => (gateSteps(value)[2].uses = "actions/setup-node@v6")], + ["Node version", (value) => (gateSteps(value)[2].with!["node-version"] = 20)], + ["verifier condition", (value) => (gateSteps(value)[3].if = "${{ always() }}")], + ["verifier token", (value) => (gateSteps(value)[3].env!.GITHUB_TOKEN = "${{ secrets.TOKEN }}")], + [ + "verifier SHA", + (value) => (gateSteps(value)[3].env!.EXPECTED_SHA = "${{ inputs.checkout_sha }}"), + ], + [ + "verifier command", + (value) => { + gateSteps(value)[3].run = "node tools/e2e/base-image-publication.mts"; + }, + ], + ["step count", (value) => gateSteps(value).push({ name: "Unreviewed step", run: "true" })], + ["fanout dependency", (value) => (value.jobs["generate-matrix"].needs = [])], + ]; + + it.each(mutations)("rejects %s drift (#7372)", (_case, mutate) => { + const value = workflow(); + mutate(value); + expect(validate(value)).not.toEqual([]); + }); +}); diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts new file mode 100644 index 00000000000..f5e7a7dadf2 --- /dev/null +++ b/test/e2e/support/base-image-publication.test.ts @@ -0,0 +1,632 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + collectPaginated, + type FirstParentHistory, + githubRequest, + type PublicationRun, + parseBaseImagePushPaths, + resolveFirstParentHistory, + selectPublicationRun, + validateBoundRun, + validatePublisherJobs, + validateWorkflow, + waitForBaseImagePublication, +} from "../../../tools/e2e/base-image-publication.mts"; + +const EXPECTED_SHA = "a".repeat(40); +const DESCENDANT_SHA = "b".repeat(40); +const RELEVANT_SHA = "c".repeat(40); +const STALE_SHA = "d".repeat(40); +const RUN_ID = 29891942278; +const WORKFLOW_ID = 251475843; +const RUN_URL_ROOT = "https://github.com/NVIDIA/NemoClaw/actions/runs"; +const RUN_URL = `https://github.com/NVIDIA/NemoClaw/actions/runs/${RUN_ID}`; +const WORKFLOW_SOURCE = `on: + push: + branches: [main] + paths: + - ".github/workflows/base-image.yaml" + - "Dockerfile.base" + workflow_dispatch: +jobs: {} +`; + +function required(value: T | undefined, message: string): T { + return ( + value ?? + (() => { + throw new Error(message); + })() + ); +} + +function historyGitResponse(args: string[], relevantSha: string, firstParentShas: string): string { + const responses = new Map([ + ["rev-parse:--verify", EXPECTED_SHA], + ["rev-parse:--is-shallow-repository", "false"], + ["log:--first-parent", relevantSha], + ["rev-list:--first-parent", firstParentShas], + ]); + return required(responses.get(`${args[0]}:${args[1]}`), "unexpected git history request"); +} + +function nextFetchResponse(responses: Array): Promise { + const response = required(responses.shift(), "unexpected GitHub request"); + return response instanceof Error ? Promise.reject(response) : Promise.resolve(response); +} + +function history(): FirstParentHistory { + return { + expectedSha: EXPECTED_SHA, + relevantSha: RELEVANT_SHA, + relevantDistance: 2, + distanceBySha: new Map([ + [EXPECTED_SHA, 0], + [DESCENDANT_SHA, 1], + [RELEVANT_SHA, 2], + ]), + }; +} + +function workflowRun(overrides: Record = {}): Record { + return { + id: RUN_ID, + run_attempt: 1, + workflow_id: WORKFLOW_ID, + name: "Images / Base Images", + event: "push", + status: "completed", + conclusion: "success", + head_sha: RELEVANT_SHA, + head_branch: "main", + path: ".github/workflows/base-image.yaml", + repository: { full_name: "NVIDIA/NemoClaw" }, + head_repository: { full_name: "NVIDIA/NemoClaw" }, + html_url: RUN_URL, + ...overrides, + }; +} + +function workflowMetadata(overrides: Record = {}): Record { + return { + id: WORKFLOW_ID, + name: "Images / Base Images", + path: ".github/workflows/base-image.yaml", + state: "active", + html_url: "https://github.com/NVIDIA/NemoClaw/blob/main/.github/workflows/base-image.yaml", + url: `https://api.github.com/repos/NVIDIA/NemoClaw/actions/workflows/${WORKFLOW_ID}`, + ...overrides, + }; +} + +function runsPayload(runs: unknown[]): Record { + return { total_count: runs.length, workflow_runs: runs }; +} + +function selectedRun(overrides: Partial = {}): PublicationRun { + return { + id: RUN_ID, + attempt: 1, + workflowId: WORKFLOW_ID, + headSha: RELEVANT_SHA, + status: "completed", + conclusion: "success", + url: RUN_URL, + ...overrides, + }; +} + +function publisherJob( + name: string, + overrides: Record = {}, +): Record { + return { + id: 1000, + run_id: RUN_ID, + run_attempt: 1, + head_sha: RELEVANT_SHA, + name, + status: "completed", + conclusion: "success", + ...overrides, + }; +} + +function successfulJobs(overrides: { runAttempt?: number } = {}): Record[] { + const runAttempt = overrides.runAttempt ?? 1; + return [ + publisherJob("Build and push OpenClaw base image", { id: 1, run_attempt: runAttempt }), + publisherJob("Build and push Hermes base image", { id: 2, run_attempt: runAttempt }), + publisherJob("Build and push Deep Agents Code base image", { + id: 3, + run_attempt: runAttempt, + }), + ]; +} + +describe("base-image publication evidence", () => { + it("extracts the checked-in literal publisher paths without runtime dependencies (#7372)", () => { + const source = fs.readFileSync( + path.resolve(import.meta.dirname, "../../../.github/workflows/base-image.yaml"), + "utf8", + ); + + expect(parseBaseImagePushPaths(source)).toEqual( + expect.arrayContaining([ + ".github/workflows/base-image.yaml", + "Dockerfile.base", + "agents/hermes/Dockerfile.base", + "agents/langchain-deepagents-code/Dockerfile.base", + ]), + ); + }); + + it.each([ + [ + "a duplicate", + WORKFLOW_SOURCE.replace( + ' - "Dockerfile.base"', + ' - "Dockerfile.base"\n - "Dockerfile.base"', + ), + /must be unique/u, + ], + [ + "a glob", + WORKFLOW_SOURCE.replace("Dockerfile.base", "Dockerfile.*"), + /not a safe literal path/u, + ], + [ + "a parent traversal", + WORKFLOW_SOURCE.replace("Dockerfile.base", "../Dockerfile.base"), + /not a safe literal path/u, + ], + [ + "an unquoted scalar", + WORKFLOW_SOURCE.replace('"Dockerfile.base"', "Dockerfile.base"), + /must be one quoted scalar/u, + ], + [ + "a missing workflow path", + WORKFLOW_SOURCE.replace(' - ".github/workflows/base-image.yaml"\n', ""), + /must include/u, + ], + [ + "a flow list", + WORKFLOW_SOURCE.replace( + 'paths:\n - ".github/workflows/base-image.yaml"\n - "Dockerfile.base"', + 'paths: [".github/workflows/base-image.yaml", "Dockerfile.base"]', + ), + /non-empty on\.push\.paths/u, + ], + [ + "a non-main branch", + WORKFLOW_SOURCE.replace("branches: [main]", "branches: [release]"), + /non-empty on\.push\.paths/u, + ], + ])("rejects %s in publisher trigger paths (#7372)", (_case, source, expected) => { + expect(() => parseBaseImagePushPaths(source)).toThrow(expected); + }); + + it("binds the applicable commit to the checked-out first-parent chain (#7372)", () => { + const calls: string[][] = []; + const resolved = resolveFirstParentHistory(EXPECTED_SHA, ["Dockerfile.base"], (args) => { + calls.push(args); + return historyGitResponse( + args, + RELEVANT_SHA, + `${EXPECTED_SHA}\n${DESCENDANT_SHA}\n${RELEVANT_SHA}\n${STALE_SHA}`, + ); + }); + + expect(resolved.relevantSha).toBe(RELEVANT_SHA); + expect([...resolved.distanceBySha]).toEqual([ + [EXPECTED_SHA, 0], + [DESCENDANT_SHA, 1], + [RELEVANT_SHA, 2], + ]); + expect(calls[2]).toEqual([ + "log", + "--first-parent", + "-n", + "1", + "--format=%H", + EXPECTED_SHA, + "--", + "Dockerfile.base", + ]); + }); + + it("selects the merge commit instead of its side-branch source commit (#7372)", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-publication-history-")); + const git = (...args: string[]) => + execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); + const write = (file: string, contents: string) => + fs.writeFileSync(path.join(directory, file), contents); + const commit = (message: string) => { + git("add", "."); + git("commit", "-m", message); + return git("rev-parse", "HEAD"); + }; + + try { + git("init", "-b", "main"); + git("config", "user.name", "NemoClaw Test"); + git("config", "user.email", "test@example.com"); + write("Dockerfile.base", "base\n"); + commit("base"); + write("unrelated.txt", "main\n"); + const branchPoint = commit("main change"); + git("switch", "-c", "feature"); + write("Dockerfile.base", "feature\n"); + const sideBranchSha = commit("side change"); + git("switch", "main"); + write("main-only.txt", "main\n"); + commit("later main change"); + git("merge", "--no-ff", "feature", "-m", "merge feature"); + const mergeSha = git("rev-parse", "HEAD"); + + const resolved = resolveFirstParentHistory(mergeSha, ["Dockerfile.base"], (args) => + git(...args), + ); + + expect(branchPoint).not.toBe(sideBranchSha); + expect(resolved.relevantSha).toBe(mergeSha); + expect(resolved.distanceBySha.has(sideBranchSha)).toBe(false); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("rejects checkout and history identity drift (#7372)", () => { + expect(() => + resolveFirstParentHistory(EXPECTED_SHA, ["Dockerfile.base"], () => DESCENDANT_SHA), + ).toThrow(/checked-out commit/u); + expect(() => + resolveFirstParentHistory(EXPECTED_SHA, ["Dockerfile.base"], (args) => + historyGitResponse(args, STALE_SHA, `${EXPECTED_SHA}\n${RELEVANT_SHA}`), + ), + ).toThrow(/not on the first-parent history/u); + }); + + it("binds API evidence to the active checked-in workflow identity (#7372)", () => { + expect(validateWorkflow(workflowMetadata())).toBe(WORKFLOW_ID); + expect(() => validateWorkflow(workflowMetadata({ state: "disabled_manually" }))).toThrow( + /state must be active/u, + ); + expect(() => + selectPublicationRun( + runsPayload([workflowRun({ workflow_id: WORKFLOW_ID + 1 })]), + history(), + WORKFLOW_ID, + ), + ).toThrow(/workflow id does not match/u); + }); + + it("collects page-two evidence and rejects duplicate or truncated pagination (#7372)", async () => { + const entries = Array.from({ length: 101 }, (_, index) => ({ id: index + 1 })); + const pages = [ + { total_count: entries.length, workflow_runs: entries.slice(0, 100) }, + { total_count: entries.length, workflow_runs: entries.slice(100) }, + ]; + const requests: string[] = []; + + await expect( + collectPaginated( + async (requestPath) => { + requests.push(requestPath); + return pages.shift(); + }, + "/runs?per_page=100", + "workflow_runs", + ), + ).resolves.toMatchObject({ total_count: 101, workflow_runs: entries }); + expect(requests).toEqual(["/runs?per_page=100&page=1", "/runs?per_page=100&page=2"]); + + await expect( + collectPaginated( + async () => ({ total_count: 101, jobs: entries.slice(0, 100) }), + "/jobs?per_page=100", + "jobs", + 1, + ), + ).rejects.toThrow(/exceeded the 1-page safety cap/u); + await expect( + collectPaginated( + async () => ({ total_count: 2, jobs: [{ id: 1 }, { id: 1 }] }), + "/jobs?per_page=100", + "jobs", + ), + ).rejects.toThrow(/duplicate id/u); + }); + + it("accepts a batch-push tip that descends from the newest changed input (#7372)", () => { + const selection = selectPublicationRun( + runsPayload([workflowRun({ head_sha: EXPECTED_SHA })]), + history(), + WORKFLOW_ID, + ); + + expect(selection).toMatchObject({ state: "ready", run: { headSha: EXPECTED_SHA } }); + }); + + it("prefers the graph-newest trusted run without relying on API order (#7372)", () => { + const selection = selectPublicationRun( + runsPayload([ + workflowRun({ + id: 10, + head_sha: RELEVANT_SHA, + html_url: `${RUN_URL.replace(String(RUN_ID), "10")}`, + }), + workflowRun({ + id: 11, + head_sha: DESCENDANT_SHA, + html_url: `${RUN_URL.replace(String(RUN_ID), "11")}`, + }), + ]), + history(), + WORKFLOW_ID, + ); + + expect(selection).toMatchObject({ state: "ready", run: { id: 11, headSha: DESCENDANT_SHA } }); + }); + + it("waits for missing and in-progress publication evidence (#7372)", () => { + expect(selectPublicationRun(runsPayload([]), history(), WORKFLOW_ID)).toEqual({ + state: "missing", + }); + expect( + selectPublicationRun( + runsPayload([workflowRun({ status: "in_progress", conclusion: null })]), + history(), + WORKFLOW_ID, + ), + ).toMatchObject({ state: "pending", run: { status: "in_progress" } }); + }); + + it.each([ + "failure", + "cancelled", + ] as const)("fails closed when publication concludes %s (#7372)", (conclusion) => { + expect(() => + selectPublicationRun(runsPayload([workflowRun({ conclusion })]), history(), WORKFLOW_ID), + ).toThrow(`base-image workflow for ${RELEVANT_SHA} concluded ${conclusion}; ${RUN_URL}`); + }); + + it("fails closed on ambiguous or malformed runs (#7372)", () => { + expect(() => + selectPublicationRun( + runsPayload([ + workflowRun(), + workflowRun({ id: RUN_ID + 1, html_url: `${RUN_URL_ROOT}/${RUN_ID + 1}` }), + ]), + history(), + WORKFLOW_ID, + ), + ).toThrow(/multiple trusted/u); + expect(() => + selectPublicationRun( + runsPayload([workflowRun({ repository: { full_name: "attacker/fork" } })]), + history(), + WORKFLOW_ID, + ), + ).toThrow(/repository must be NVIDIA\/NemoClaw/u); + expect(() => + selectPublicationRun( + { total_count: 2, workflow_runs: [workflowRun()] }, + history(), + WORKFLOW_ID, + ), + ).toThrow(/incomplete/u); + }); + + it("requires every publisher latest attempt to complete successfully (#7372)", () => { + const run = selectedRun({ attempt: 2 }); + const jobs = [ + ...successfulJobs(), + publisherJob("Build and push Hermes base image", { + id: 4, + run_attempt: 1, + conclusion: "failure", + }), + publisherJob("Build and push Hermes base image", { id: 5, run_attempt: 2 }), + ].filter((job, index) => index !== 1); + + expect(() => validatePublisherJobs({ total_count: jobs.length, jobs }, run)).not.toThrow(); + }); + + it("reconfirms the selected successful run after reading job history (#7372)", () => { + expect(() => validateBoundRun(workflowRun(), selectedRun())).not.toThrow(); + expect(() => + validateBoundRun(workflowRun({ status: "in_progress", conclusion: null }), selectedRun()), + ).toThrow(/changed while evidence was verified/u); + }); + + it.each([ + ["missing", successfulJobs().slice(0, 2), /missing required/u], + [ + "duplicated", + [...successfulJobs(), publisherJob("Build and push Hermes base image", { id: 9 })], + /duplicated in attempt/u, + ], + [ + "failed latest attempt", + successfulJobs().map((job) => + job.name === "Build and push Hermes base image" ? { ...job, conclusion: "failure" } : job, + ), + /did not complete successfully/u, + ], + [ + "wrong run", + successfulJobs().map((job, index) => (index === 0 ? { ...job, run_id: 7 } : job)), + /provenance does not match/u, + ], + ])("rejects %s publisher evidence (#7372)", (_case, jobs, expected) => { + expect(() => validatePublisherJobs({ total_count: jobs.length, jobs }, selectedRun())).toThrow( + expected, + ); + }); + + it("polls from missing through completion and verifies jobs (#7372)", async () => { + const responses = [ + workflowMetadata(), + runsPayload([]), + runsPayload([workflowRun({ status: "queued", conclusion: null })]), + runsPayload([workflowRun()]), + { total_count: 3, jobs: successfulJobs() }, + workflowRun(), + ]; + const requests: string[] = []; + const notices: string[] = []; + let currentTime = 0; + + const run = await waitForBaseImagePublication({ + history: history(), + request: async (requestPath) => { + requests.push(requestPath); + return responses.shift(); + }, + waitMs: 100, + pollMs: 10, + now: () => currentTime, + sleep: async (milliseconds) => { + currentTime += milliseconds; + }, + notice: (message) => notices.push(message), + }); + + expect(run.id).toBe(RUN_ID); + expect(requests).toEqual([ + "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", + "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", + "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", + "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", + `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/jobs?filter=all&per_page=100&page=1`, + `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}`, + ]); + expect(notices).toHaveLength(2); + }); + + it("reports the selected publisher SHA and run URL for invalid job evidence (#7372)", async () => { + const jobs = successfulJobs().map((job, index) => + index === 0 ? { ...job, run_id: RUN_ID + 1 } : job, + ); + const responses = [ + workflowMetadata(), + runsPayload([workflowRun()]), + { total_count: jobs.length, jobs }, + ]; + + await expect( + waitForBaseImagePublication({ + history: history(), + request: async () => responses.shift(), + waitMs: 100, + pollMs: 10, + }), + ).rejects.toThrow(new RegExp(`provenance does not match.*${RELEVANT_SHA}.*${RUN_URL}`, "u")); + }); + + it("times out deterministically without sleeping past its budget (#7372)", async () => { + const responses = [workflowMetadata(), runsPayload([])]; + await expect( + waitForBaseImagePublication({ + history: history(), + request: async () => responses.shift(), + waitMs: 0, + pollMs: 10, + now: () => 10, + sleep: async () => { + throw new Error("must not sleep"); + }, + }), + ).rejects.toThrow(new RegExp(`timed out.*${RELEVANT_SHA}`, "u")); + }); + + it("retries bounded transient and rate-limited GitHub responses (#7372)", async () => { + const transientResponses: Array = [ + new Error("network unavailable"), + new Response("unavailable", { status: 503, headers: { "retry-after": "2" } }), + new Response(JSON.stringify({ ok: true }), { status: 200 }), + ]; + const transientSleeps: number[] = []; + + await expect( + githubRequest("/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", "token", { + fetchImpl: () => nextFetchResponse(transientResponses), + sleep: async (milliseconds) => { + transientSleeps.push(milliseconds); + }, + }), + ).resolves.toEqual({ ok: true }); + expect(transientSleeps).toEqual([1000, 2000]); + + const rateLimitResponses: Array = [ + new Response("limited", { + status: 403, + headers: { "retry-after": "7", "x-ratelimit-remaining": "0" }, + }), + new Response(JSON.stringify({ ok: true }), { status: 200 }), + ]; + const rateLimitSleeps: number[] = []; + await expect( + githubRequest("/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", "token", { + attempts: 2, + fetchImpl: () => nextFetchResponse(rateLimitResponses), + sleep: async (milliseconds) => { + rateLimitSleeps.push(milliseconds); + }, + }), + ).resolves.toEqual({ ok: true }); + expect(rateLimitSleeps).toEqual([7000]); + }); + + it("fails permanent and malformed GitHub responses without retrying (#7372)", async () => { + let requests = 0; + await expect( + githubRequest("/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", "token", { + fetchImpl: async () => { + requests += 1; + return new Response("not found", { status: 404 }); + }, + sleep: async () => { + throw new Error("must not retry"); + }, + }), + ).rejects.toThrow(/HTTP 404/u); + expect(requests).toBe(1); + + await expect( + githubRequest("/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", "token", { + fetchImpl: async () => new Response("{", { status: 200 }), + }), + ).rejects.toThrow(/not valid JSON/u); + }); + + it("loads directly with the Node strip-types runtime used by Actions (#7372)", () => { + const modulePath = path.resolve( + import.meta.dirname, + "../../../tools/e2e/base-image-publication.mts", + ); + expect(() => + execFileSync( + process.execPath, + [ + "--experimental-strip-types", + "--no-warnings", + "--eval", + `import(${JSON.stringify(modulePath)})`, + ], + { encoding: "utf8" }, + ), + ).not.toThrow(); + }); +}); diff --git a/test/pr-e2e-gate-fork-skip.test.ts b/test/pr-e2e-gate-fork-skip.test.ts index 6f9db2a41b1..def30d55a08 100644 --- a/test/pr-e2e-gate-fork-skip.test.ts +++ b/test/pr-e2e-gate-fork-skip.test.ts @@ -203,7 +203,7 @@ function startControlPlaneCommand(workDir: string) { function approvalWorkflowRun(overrides: Record = {}) { return { id: APPROVAL_RUN_ID, - name: "E2E / PR Gate Controller", + name: `E2E Gate workflow_run ${APPROVAL_RUN_ID}`, path: ".github/workflows/pr-e2e-gate.yaml", event: "workflow_run", head_sha: WORKFLOW_SHA, @@ -698,7 +698,6 @@ describe("PR E2E controller fork credentialed E2E skip approval safety", () => { it.each([ { name: "wrong run id", overrides: { id: APPROVAL_RUN_ID + 1 } }, - { name: "wrong workflow name", overrides: { name: "Other workflow" } }, { name: "wrong event", overrides: { event: "workflow_dispatch" } }, { name: "untrusted workflow path suffix", diff --git a/test/pr-e2e-gate-internal-approval.test.ts b/test/pr-e2e-gate-internal-approval.test.ts index eea3165bd41..150a27cfb8e 100644 --- a/test/pr-e2e-gate-internal-approval.test.ts +++ b/test/pr-e2e-gate-internal-approval.test.ts @@ -87,7 +87,7 @@ function approvedControlPlaneCommand(workDir: string) { function approvalWorkflowRun() { return { id: APPROVAL_RUN_ID, - name: "E2E / PR Gate Controller", + name: `E2E Gate workflow_run ${APPROVAL_RUN_ID}`, path: ".github/workflows/pr-e2e-gate.yaml", event: "workflow_run", head_sha: WORKFLOW_SHA, diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts new file mode 100644 index 00000000000..0d59abd2957 --- /dev/null +++ b/tools/e2e/base-image-publication.mts @@ -0,0 +1,720 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; + +const REPOSITORY = "NVIDIA/NemoClaw"; +const MAIN_BRANCH = "main"; +const WORKFLOW_PATH = ".github/workflows/base-image.yaml"; +const WORKFLOW_FILE = "base-image.yaml"; +const WORKFLOW_NAME = "Images / Base Images"; +const API_ROOT = "https://api.github.com"; +const RUN_URL_ROOT = `https://github.com/${REPOSITORY}/actions/runs`; +const WORKFLOW_URL = `https://github.com/${REPOSITORY}/blob/${MAIN_BRANCH}/${WORKFLOW_PATH}`; +const PAGE_SIZE = 100; +const MAX_API_PAGES = 10; +const REQUEST_ATTEMPTS = 3; +const REQUEST_TIMEOUT_MS = 20_000; +const MAX_RETRY_DELAY_MS = 10_000; +const SHA_PATTERN = /^[0-9a-f]{40}$/u; +const SAFE_PATH_PATTERN = /^[A-Za-z0-9._/-]+$/u; +const PENDING_RUN_STATUSES = new Set(["requested", "waiting", "pending", "queued", "in_progress"]); +const COMPLETED_CONCLUSIONS = new Set([ + "action_required", + "cancelled", + "failure", + "neutral", + "skipped", + "stale", + "startup_failure", + "success", + "timed_out", +]); + +export const REQUIRED_PUBLISHER_JOBS = [ + "Build and push OpenClaw base image", + "Build and push Hermes base image", + "Build and push Deep Agents Code base image", +] as const; + +type JsonRecord = Record; + +export interface FirstParentHistory { + expectedSha: string; + relevantSha: string; + relevantDistance: number; + distanceBySha: ReadonlyMap; +} + +export interface PublicationRun { + id: number; + attempt: number; + workflowId: number; + headSha: string; + status: string; + conclusion: string | null; + url: string; +} + +export type PublicationSelection = + | { state: "missing" } + | { state: "pending"; run: PublicationRun } + | { state: "ready"; run: PublicationRun }; + +export interface PublicationWaitOptions { + history: FirstParentHistory; + request: (path: string) => Promise; + waitMs: number; + pollMs: number; + now?: () => number; + sleep?: (milliseconds: number) => Promise; + notice?: (message: string) => void; +} + +export interface GithubRequestOptions { + fetchImpl?: (input: string, init: RequestInit) => Promise; + sleep?: (milliseconds: number) => Promise; + now?: () => number; + attempts?: number; + timeoutMs?: number; +} + +function asRecord(value: unknown): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("GitHub response must contain JSON objects"); + } + return value as JsonRecord; +} + +function positiveSafeInteger(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error(`${label} must be a positive safe integer`); + } + return Number(value); +} + +function exactString(value: unknown, expected: string, label: string): string { + if (value !== expected) { + throw new Error(`${label} must be ${expected}`); + } + return expected; +} + +function sha(value: unknown, label: string): string { + if (typeof value !== "string" || !SHA_PATTERN.test(value)) { + throw new Error(`${label} must be a lowercase 40-character SHA`); + } + return value; +} + +function parseQuotedPath(raw: string, lineNumber: number): string { + let value: unknown; + try { + if (raw.startsWith('"') && raw.endsWith('"')) { + value = JSON.parse(raw); + } else if (raw.startsWith("'") && raw.endsWith("'")) { + value = raw.slice(1, -1).replaceAll("''", "'"); + } else { + throw new Error("not quoted"); + } + } catch { + throw new Error(`base-image push path on line ${lineNumber} must be one quoted scalar`); + } + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { + throw new Error(`base-image push path on line ${lineNumber} must be a non-empty exact path`); + } + if ( + !SAFE_PATH_PATTERN.test(value) || + value.startsWith("/") || + value.startsWith("-") || + value.startsWith(":") || + value.includes("//") || + value.split("/").some((segment) => segment === "" || segment === "." || segment === "..") + ) { + throw new Error(`base-image push path on line ${lineNumber} is not a safe literal path`); + } + return value; +} + +/** + * Read the literal path list that controls the publisher without requiring a + * dependency install in the preflight job. Deliberately reject YAML features + * such as flow lists, aliases, globs, and folded scalars instead of guessing. + */ +export function parseBaseImagePushPaths(source: string): string[] { + const lines = source.split(/\r?\n/u); + let inOn = false; + let inPush = false; + let inPaths = false; + let sawOn = false; + let sawPush = false; + let sawPaths = false; + let sawMainBranch = false; + const paths: string[] = []; + + for (const [index, line] of lines.entries()) { + const lineNumber = index + 1; + if (line.trim().length === 0 || line.trimStart().startsWith("#")) continue; + const indent = line.length - line.trimStart().length; + const trimmed = line.trim(); + + if (indent === 0) { + inOn = trimmed === "on:"; + inPush = false; + inPaths = false; + if (inOn) { + if (sawOn) throw new Error("base-image workflow must declare exactly one on block"); + sawOn = true; + } + continue; + } + if (!inOn) continue; + + if (indent === 2) { + inPush = trimmed === "push:"; + inPaths = false; + if (inPush) { + if (sawPush) throw new Error("base-image workflow must declare exactly one push trigger"); + sawPush = true; + } + continue; + } + if (!inPush) continue; + + if (indent === 4) { + if (trimmed === "branches: [main]") sawMainBranch = true; + inPaths = trimmed === "paths:"; + if (inPaths) { + if (sawPaths) + throw new Error("base-image push trigger must declare exactly one paths list"); + sawPaths = true; + } + continue; + } + if (!inPaths) continue; + + const match = line.match(/^ {6}- (.+)$/u); + if (!match) { + throw new Error( + `base-image push paths must be a six-space-indented scalar list (line ${lineNumber})`, + ); + } + paths.push(parseQuotedPath(match[1], lineNumber)); + } + + if (!sawOn || !sawPush || !sawMainBranch || !sawPaths || paths.length === 0) { + throw new Error("base-image workflow must declare a non-empty on.push.paths list"); + } + if (new Set(paths).size !== paths.length) { + throw new Error("base-image push paths must be unique"); + } + if (!paths.includes(WORKFLOW_PATH)) { + throw new Error(`base-image push paths must include ${WORKFLOW_PATH}`); + } + return paths; +} + +function defaultGit(args: string[]): string { + return execFileSync("git", args, { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} + +export function resolveFirstParentHistory( + expectedSha: string, + paths: readonly string[], + runGit: (args: string[]) => string = defaultGit, +): FirstParentHistory { + sha(expectedSha, "expected SHA"); + if (paths.length === 0) throw new Error("at least one base-image path is required"); + + const checkedOutSha = runGit(["rev-parse", "--verify", "HEAD^{commit}"]); + if (checkedOutSha !== expectedSha) { + throw new Error( + `checked-out commit ${checkedOutSha || "missing"} does not match ${expectedSha}`, + ); + } + if (runGit(["rev-parse", "--is-shallow-repository"]) !== "false") { + throw new Error("base-image publication gate requires a complete Git history"); + } + + const relevantSha = runGit([ + "log", + "--first-parent", + "-n", + "1", + "--format=%H", + expectedSha, + "--", + ...paths, + ]); + sha(relevantSha, "latest applicable base-image commit"); + + const firstParentShas = runGit(["rev-list", "--first-parent", expectedSha]) + .split(/\r?\n/u) + .filter(Boolean); + if (firstParentShas.length === 0 || firstParentShas[0] !== expectedSha) { + throw new Error("first-parent history must begin at the expected SHA"); + } + if (new Set(firstParentShas).size !== firstParentShas.length) { + throw new Error("first-parent history must not contain duplicate commits"); + } + for (const [index, value] of firstParentShas.entries()) + sha(value, `first-parent commit ${index}`); + + const relevantDistance = firstParentShas.indexOf(relevantSha); + if (relevantDistance < 0) { + throw new Error("latest applicable base-image commit is not on the first-parent history"); + } + const eligibleShas = firstParentShas.slice(0, relevantDistance + 1); + return { + expectedSha, + relevantSha, + relevantDistance, + distanceBySha: new Map(eligibleShas.map((value, index) => [value, index])), + }; +} + +export function validateWorkflow(payload: unknown): number { + const workflow = asRecord(payload); + const workflowId = positiveSafeInteger(workflow.id, "base-image workflow id"); + exactString(workflow.name, WORKFLOW_NAME, "base-image workflow name"); + exactString(workflow.path, WORKFLOW_PATH, "base-image workflow path"); + exactString(workflow.state, "active", "base-image workflow state"); + exactString(workflow.html_url, WORKFLOW_URL, "base-image workflow URL"); + exactString( + workflow.url, + `${API_ROOT}/repos/${REPOSITORY}/actions/workflows/${workflowId}`, + "base-image workflow API URL", + ); + return workflowId; +} + +function validateRun(value: unknown, index: number, expectedWorkflowId: number): PublicationRun { + const run = asRecord(value); + const id = positiveSafeInteger(run.id, `workflow run ${index} id`); + const attempt = positiveSafeInteger(run.run_attempt, `workflow run ${index} attempt`); + if ( + positiveSafeInteger(run.workflow_id, `workflow run ${index} workflow id`) !== expectedWorkflowId + ) { + throw new Error(`workflow run ${index} workflow id does not match the base-image workflow`); + } + const headSha = sha(run.head_sha, `workflow run ${index} head SHA`); + exactString(run.event, "push", `workflow run ${index} event`); + exactString(run.head_branch, MAIN_BRANCH, `workflow run ${index} branch`); + exactString(run.path, WORKFLOW_PATH, `workflow run ${index} path`); + exactString(run.name, WORKFLOW_NAME, `workflow run ${index} name`); + exactString(asRecord(run.repository).full_name, REPOSITORY, `workflow run ${index} repository`); + exactString( + asRecord(run.head_repository).full_name, + REPOSITORY, + `workflow run ${index} head repository`, + ); + const url = `${RUN_URL_ROOT}/${id}`; + exactString(run.html_url, url, `workflow run ${index} URL`); + + if (typeof run.status !== "string") throw new Error(`workflow run ${index} status is invalid`); + const status = run.status; + let conclusion: string | null = null; + if (status === "completed") { + if (typeof run.conclusion !== "string" || !COMPLETED_CONCLUSIONS.has(run.conclusion)) { + throw new Error(`workflow run ${index} completed conclusion is invalid`); + } + conclusion = run.conclusion; + } else { + if (!PENDING_RUN_STATUSES.has(status) || run.conclusion !== null) { + throw new Error(`workflow run ${index} pending state is invalid`); + } + } + return { id, attempt, workflowId: expectedWorkflowId, headSha, status, conclusion, url }; +} + +export function selectPublicationRun( + payload: unknown, + history: FirstParentHistory, + workflowId: number, +): PublicationSelection { + positiveSafeInteger(workflowId, "base-image workflow id"); + const response = asRecord(payload); + const totalCount = Number(response.total_count); + if (!Number.isSafeInteger(totalCount) || totalCount < 0) { + throw new Error("workflow run total_count is invalid"); + } + if (!Array.isArray(response.workflow_runs) || response.workflow_runs.length !== totalCount) { + throw new Error("workflow run listing is incomplete"); + } + + const runs = response.workflow_runs.map((run, index) => validateRun(run, index, workflowId)); + if (new Set(runs.map((run) => run.id)).size !== runs.length) { + throw new Error("workflow run listing contains duplicate run ids"); + } + const eligible = runs.flatMap((run) => { + const distance = history.distanceBySha.get(run.headSha); + return distance === undefined ? [] : [{ run, distance }]; + }); + if (eligible.length === 0) return { state: "missing" }; + + const nearestDistance = Math.min(...eligible.map(({ distance }) => distance)); + const nearest = eligible.filter(({ distance }) => distance === nearestDistance); + if (nearest.length !== 1) { + throw new Error( + `multiple trusted base-image workflow runs match ${nearest[0]?.run.headSha ?? history.relevantSha}`, + ); + } + const run = nearest[0].run; + if (run.status !== "completed") return { state: "pending", run }; + if (run.conclusion !== "success") { + throw new Error( + `base-image workflow for ${run.headSha} concluded ${run.conclusion}; ${run.url}`, + ); + } + return { state: "ready", run }; +} + +export function validatePublisherJobs(payload: unknown, run: PublicationRun): void { + const response = asRecord(payload); + const totalCount = Number(response.total_count); + if (!Number.isSafeInteger(totalCount) || totalCount < 0) { + throw new Error("publisher job total_count is invalid"); + } + if (!Array.isArray(response.jobs) || response.jobs.length !== totalCount) { + throw new Error("publisher job listing is incomplete"); + } + + const jobsByName = new Map< + string, + Array<{ attempt: number; status: string; conclusion: string }> + >(); + for (const [index, value] of response.jobs.entries()) { + const job = asRecord(value); + positiveSafeInteger(job.id, `publisher job ${index} id`); + const attempt = positiveSafeInteger(job.run_attempt, `publisher job ${index} attempt`); + if (job.run_id !== run.id || attempt > run.attempt || job.head_sha !== run.headSha) { + throw new Error(`publisher job ${index} provenance does not match the selected run`); + } + if (typeof job.name !== "string" || job.name.length === 0) { + throw new Error(`publisher job ${index} name is invalid`); + } + if ( + job.status !== "completed" || + typeof job.conclusion !== "string" || + !COMPLETED_CONCLUSIONS.has(job.conclusion) + ) { + throw new Error(`publisher job ${job.name} completion evidence is invalid; ${run.url}`); + } + const occurrences = jobsByName.get(job.name) ?? []; + if (occurrences.some((occurrence) => occurrence.attempt === attempt)) { + throw new Error(`publisher job ${job.name} is duplicated in attempt ${attempt}; ${run.url}`); + } + occurrences.push({ attempt, status: job.status, conclusion: job.conclusion }); + jobsByName.set(job.name, occurrences); + } + + for (const requiredName of REQUIRED_PUBLISHER_JOBS) { + const occurrences = jobsByName.get(requiredName) ?? []; + if (occurrences.length === 0) { + throw new Error(`missing required ${requiredName} job; ${run.url}`); + } + const latestAttempt = Math.max(...occurrences.map((occurrence) => occurrence.attempt)); + const latest = occurrences.find((occurrence) => occurrence.attempt === latestAttempt); + if (!latest || latest.status !== "completed" || latest.conclusion !== "success") { + throw new Error( + `latest ${requiredName} job did not complete successfully in attempt ${latestAttempt}; ${run.url}`, + ); + } + } +} + +export function validateBoundRun(payload: unknown, expected: PublicationRun): void { + const actual = validateRun(payload, 0, expected.workflowId); + if ( + actual.id !== expected.id || + actual.attempt !== expected.attempt || + actual.headSha !== expected.headSha || + actual.status !== "completed" || + actual.conclusion !== "success" + ) { + throw new Error( + `selected base-image workflow changed while evidence was verified; ${expected.url}`, + ); + } +} + +export async function collectPaginated( + request: (path: string) => Promise, + basePath: string, + collectionKey: "workflow_runs" | "jobs", + maxPages = MAX_API_PAGES, +): Promise { + if (!Number.isSafeInteger(maxPages) || maxPages < 1) { + throw new Error("pagination page cap must be a positive integer"); + } + const label = collectionKey === "workflow_runs" ? "workflow run" : "publisher job"; + const values: unknown[] = []; + const ids = new Set(); + let totalCount: number | undefined; + const separator = basePath.includes("?") ? "&" : "?"; + + for (let page = 1; page <= maxPages; page += 1) { + const response = asRecord(await request(`${basePath}${separator}page=${page}`)); + const pageTotal = Number(response.total_count); + if (!Number.isSafeInteger(pageTotal) || pageTotal < 0) { + throw new Error(`${label} total_count is invalid`); + } + if (totalCount === undefined) totalCount = pageTotal; + if (pageTotal !== totalCount) { + throw new Error(`${label} total_count changed during pagination`); + } + const pageValues = response[collectionKey]; + if (!Array.isArray(pageValues) || pageValues.length > PAGE_SIZE) { + throw new Error(`${label} page ${page} must contain at most ${PAGE_SIZE} entries`); + } + const expectedLength = Math.min(PAGE_SIZE, totalCount - values.length); + if (expectedLength < 0 || pageValues.length !== expectedLength) { + throw new Error(`${label} pagination is incomplete`); + } + for (const [index, value] of pageValues.entries()) { + const id = positiveSafeInteger(asRecord(value).id, `${label} page ${page} entry ${index} id`); + if (ids.has(id)) throw new Error(`${label} pagination contains duplicate id ${id}`); + ids.add(id); + values.push(value); + } + if (values.length === totalCount) { + return { total_count: totalCount, [collectionKey]: values }; + } + } + + throw new Error(`${label} pagination exceeded the ${maxPages}-page safety cap`); +} + +function annotationValue(value: string): string { + return value.replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A"); +} + +function publicationEvidenceError(error: unknown, run: PublicationRun): Error { + const message = error instanceof Error ? error.message : "unknown publisher evidence error"; + const context: string[] = []; + if (!message.includes(run.headSha)) context.push(`expected publisher SHA ${run.headSha}`); + if (!message.includes(run.url)) context.push(run.url); + return new Error([message, ...context].join("; ")); +} + +export async function waitForBaseImagePublication( + options: PublicationWaitOptions, +): Promise { + const now = options.now ?? performance.now.bind(performance); + const sleep = + options.sleep ?? ((milliseconds) => new Promise((done) => setTimeout(done, milliseconds))); + const notice = + options.notice ?? ((message) => console.log(`::notice::${annotationValue(message)}`)); + if (!Number.isSafeInteger(options.waitMs) || options.waitMs < 0) { + throw new Error("waitMs must be a non-negative integer"); + } + if (!Number.isSafeInteger(options.pollMs) || options.pollMs < 1) { + throw new Error("pollMs must be a positive integer"); + } + + const deadline = now() + options.waitMs; + const workflowId = validateWorkflow( + await options.request(`/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}`), + ); + const runsPath = `/repos/${REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?branch=${MAIN_BRANCH}&event=push&per_page=100`; + while (true) { + const runs = await collectPaginated(options.request, runsPath, "workflow_runs"); + const selection = selectPublicationRun(runs, options.history, workflowId); + if (selection.state === "ready") { + const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/jobs?filter=all&per_page=100`; + if (now() > deadline) { + throw new Error( + `timed out validating base-image publication for ${selection.run.headSha}; ${selection.run.url}`, + ); + } + try { + const jobs = await collectPaginated(options.request, jobsPath, "jobs"); + validatePublisherJobs(jobs, selection.run); + validateBoundRun( + await options.request(`/repos/${REPOSITORY}/actions/runs/${selection.run.id}`), + selection.run, + ); + } catch (error) { + throw publicationEvidenceError(error, selection.run); + } + if (now() > deadline) { + throw new Error( + `timed out validating base-image publication for ${selection.run.headSha}; ${selection.run.url}`, + ); + } + return selection.run; + } + + if (now() >= deadline) { + const pending = selection.state === "pending" ? `; ${selection.run.url}` : ""; + throw new Error( + `timed out waiting for base-image publication covering ${options.history.relevantSha}${pending}`, + ); + } + notice( + selection.state === "pending" + ? `Base-image publication is ${selection.run.status} for ${selection.run.headSha}; ${selection.run.url}` + : `Waiting for a trusted base-image push run covering ${options.history.relevantSha}`, + ); + await sleep(Math.min(options.pollMs, Math.max(1, deadline - now()))); + } +} + +function retryDelay(response: Response, attempt: number, now: () => number): number { + const retryAfter = response.headers.get("retry-after"); + if (retryAfter && /^(0|[1-9][0-9]*)$/u.test(retryAfter)) { + return Math.min(Number(retryAfter) * 1000, MAX_RETRY_DELAY_MS); + } + if (retryAfter) { + const retryDate = Date.parse(retryAfter); + if (Number.isFinite(retryDate)) { + return Math.min(Math.max(0, retryDate - now()), MAX_RETRY_DELAY_MS); + } + } + const reset = response.headers.get("x-ratelimit-reset"); + if (reset && /^(0|[1-9][0-9]*)$/u.test(reset)) { + return Math.min(Math.max(0, Number(reset) * 1000 - now()), MAX_RETRY_DELAY_MS); + } + return Math.min(attempt * 1000, MAX_RETRY_DELAY_MS); +} + +export async function githubRequest( + path: string, + token: string, + options: GithubRequestOptions = {}, +): Promise { + if (!path.startsWith(`/repos/${REPOSITORY}/`) || path.includes("\r") || path.includes("\n")) { + throw new Error("GitHub API path must stay within the canonical NemoClaw repository"); + } + const fetchImpl = options.fetchImpl ?? fetch; + const sleep = + options.sleep ?? ((milliseconds) => new Promise((done) => setTimeout(done, milliseconds))); + const now = options.now ?? Date.now; + const attempts = options.attempts ?? REQUEST_ATTEMPTS; + const timeoutMs = options.timeoutMs ?? REQUEST_TIMEOUT_MS; + if (!Number.isSafeInteger(attempts) || attempts < 1 || attempts > REQUEST_ATTEMPTS) { + throw new Error(`request attempts must be between 1 and ${REQUEST_ATTEMPTS}`); + } + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > REQUEST_TIMEOUT_MS) { + throw new Error(`request timeout must be between 1 and ${REQUEST_TIMEOUT_MS} milliseconds`); + } + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + let response: Response; + try { + response = await fetchImpl(`${API_ROOT}${path}`, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "User-Agent": "NemoClaw-base-image-publication-gate", + "X-GitHub-Api-Version": "2022-11-28", + }, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch { + if (attempt === attempts) { + throw new Error(`GitHub API request failed after ${attempts} attempts`); + } + await sleep(Math.min(attempt * 1000, MAX_RETRY_DELAY_MS)); + continue; + } + + if (!response.ok) { + const rateLimited = + response.status === 429 || + (response.status === 403 && response.headers.get("x-ratelimit-remaining") === "0"); + const transient = response.status === 408 || response.status >= 500 || rateLimited; + if (!transient || attempt === attempts) { + throw new Error(`GitHub API request failed with HTTP ${response.status}`); + } + await sleep(retryDelay(response, attempt, now)); + continue; + } + try { + return await response.json(); + } catch { + throw new Error("GitHub API response was not valid JSON"); + } + } + + throw new Error("GitHub API request failed unexpectedly"); +} + +function parseDurationArgument(argv: string[], name: string, defaultSeconds: number): number { + const index = argv.indexOf(name); + if (index < 0) return defaultSeconds; + if (index !== argv.lastIndexOf(name) || index + 1 >= argv.length) { + throw new Error(`${name} must be provided exactly once with a value`); + } + const raw = argv[index + 1]; + if (!/^(0|[1-9][0-9]*)$/u.test(raw)) throw new Error(`${name} must be whole seconds`); + const seconds = Number(raw); + if (!Number.isSafeInteger(seconds)) throw new Error(`${name} is too large`); + return seconds; +} + +export async function main(argv = process.argv.slice(2), env = process.env): Promise { + const known = new Set(["--wait-seconds", "--poll-seconds"]); + for (let index = 0; index < argv.length; index += 2) { + if (!known.has(argv[index]) || index + 1 >= argv.length) { + throw new Error(`unsupported argument ${argv[index] ?? "missing"}`); + } + } + const waitSeconds = parseDurationArgument(argv, "--wait-seconds", 3000); + const pollSeconds = parseDurationArgument(argv, "--poll-seconds", 15); + if (waitSeconds > 3000) throw new Error("--wait-seconds must not exceed 3000"); + if (pollSeconds < 1 || pollSeconds > 60) { + throw new Error("--poll-seconds must be between 1 and 60"); + } + + const token = env.GITHUB_TOKEN ?? ""; + const expectedSha = env.EXPECTED_SHA ?? ""; + const workspace = env.GITHUB_WORKSPACE ?? process.cwd(); + if (token.length === 0 || token.includes("\r") || token.includes("\n")) { + throw new Error("GITHUB_TOKEN must be a non-empty single-line value"); + } + sha(expectedSha, "EXPECTED_SHA"); + if (env.GITHUB_REPOSITORY !== REPOSITORY) { + throw new Error(`GITHUB_REPOSITORY must be ${REPOSITORY}`); + } + if (env.GITHUB_REF !== "refs/heads/main") { + throw new Error("GITHUB_REF must be refs/heads/main"); + } + if (env.GITHUB_EVENT_NAME !== "schedule" && env.GITHUB_EVENT_NAME !== "workflow_dispatch") { + throw new Error("GITHUB_EVENT_NAME must be schedule or workflow_dispatch"); + } + if (env.GITHUB_SHA !== expectedSha) { + throw new Error("EXPECTED_SHA must match GITHUB_SHA"); + } + + const workflowSource = readFileSync(resolve(workspace, WORKFLOW_PATH), "utf8"); + const paths = parseBaseImagePushPaths(workflowSource); + const history = resolveFirstParentHistory(expectedSha, paths); + const run = await waitForBaseImagePublication({ + history, + request: (path) => githubRequest(path, token), + waitMs: waitSeconds * 1000, + pollMs: pollSeconds * 1000, + }); + console.log( + `::notice title=Base-image publication verified::${annotationValue( + `All required publishers succeeded for ${run.headSha}; ${run.url}`, + )}`, + ); +} + +if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : "unknown base-image publication error"; + console.error(`::error title=Base-image publication gate failed::${annotationValue(message)}`); + process.exitCode = 1; + }); +} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 0fbec309f9a..994df0e58d7 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; import { RISK_RULES } from "../advisors/risk-plan.mts"; @@ -18,6 +19,24 @@ const GITHUB_SCRIPT_NODE24_ACTION = const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run --test-path"; const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; +const PUBLICATION_REQUIRED_CONDITION = "${{ steps.publication_mode.outputs.required == '1' }}"; +const PUBLICATION_CLASSIFIER_SCRIPT = + [ + "set -euo pipefail", + 'case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in', + " NVIDIA/NemoClaw:refs/heads/main:schedule:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)", + " required=1", + " ;;", + " NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:controller)", + " required=0", + " ;;", + " *)", + ' echo "::error::base-image publication mode is not trusted" >&2', + " exit 1", + " ;;", + "esac", + 'printf \'required=%s\\n\' "${required}" >> "${GITHUB_OUTPUT}"', + ].join("\n") + "\n"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; const ISSUE_MUTATION_BEYOND_COMMENT = /github\.rest\.issues\.(?:addAssignees|addLabels|create|deleteComment|lock|removeAssignees|removeLabel|setLabels|unlock|update|updateComment)\s*\(/u; @@ -44,7 +63,9 @@ type WorkflowJob = { if?: string; needs?: unknown; permissions?: WorkflowPermissions; + "runs-on"?: unknown; steps?: WorkflowStep[]; + "timeout-minutes"?: unknown; }; export type OperationsWorkflow = { @@ -215,11 +236,17 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): jobName === "report-to-pr" && step.name === "Check out the trusted E2E reporting helper" && step.with?.ref === "${{ github.workflow_sha }}"; + const trustedPublicationCheckout = + jobName === "base-image-publication" && + step.name === "Check out trusted E2E workflow" && + step.if === PUBLICATION_REQUIRED_CONDITION && + step.with?.ref === "${{ github.sha }}"; if ( step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && !trustedHermesFixtureCheckout && - !trustedReportHelperCheckout + !trustedReportHelperCheckout && + !trustedPublicationCheckout ) { errors.push(`${jobName} checkout must use the selected PR commit`); } @@ -227,6 +254,70 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): } } +export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): string[] { + const errors: string[] = []; + const job = workflow.jobs["base-image-publication"] ?? {}; + const expectedJob = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 55, + permissions: { + actions: "read", + contents: "read", + }, + steps: [ + { + id: "publication_mode", + name: "Classify base-image publication requirement", + env: { + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + EVENT_NAME: "${{ github.event_name }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + }, + shell: "bash", + run: PUBLICATION_CLASSIFIER_SCRIPT, + }, + { + name: "Check out trusted E2E workflow", + if: PUBLICATION_REQUIRED_CONDITION, + uses: "actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10", + with: { + ref: "${{ github.sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }, + }, + { + name: "Set up Node for publication verification", + if: PUBLICATION_REQUIRED_CONDITION, + uses: "actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e", + with: { + "node-version": 22, + }, + }, + { + name: "Verify applicable base-image publication", + if: PUBLICATION_REQUIRED_CONDITION, + env: { + EXPECTED_SHA: "${{ github.sha }}", + GITHUB_TOKEN: "${{ github.token }}", + }, + run: "node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30", + }, + ], + }; + + if (!isDeepStrictEqual(job, expectedJob)) { + errors.push( + "base-image-publication job must preserve its exact trusted-mode classifier, minimal permissions, pinned checkout, and verifier boundary", + ); + } + if (!needs(workflow.jobs["generate-matrix"] ?? {}).includes("base-image-publication")) { + errors.push("generate-matrix must wait for base-image-publication"); + } + return errors; +} + function validatePrGateEvidenceProducers(errors: string[], workflow: OperationsWorkflow): void { const requiredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); for (const jobId of requiredJobs) { @@ -593,6 +684,7 @@ export function validateE2eOperationsWorkflow( advisorPath = DEFAULT_ADVISOR_PATH, ): string[] { const errors: string[] = []; + errors.push(...validateBaseImagePublicationGate(workflow)); validatePrGateDispatch(errors, workflow); validatePrGateEvidenceProducers(errors, workflow); validateAggregation(errors, workflow); diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 20310564520..4ed8726c548 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -3195,7 +3195,9 @@ function validateApprovalWorkflowRun( const expectedUrl = `https://github.com/${options.repository}/actions/runs/${options.runId}`; const valid = value.id === options.runId && - value.name === WORKFLOW_NAME && + // The Actions REST API exposes the evaluated `run-name` as `name`, not the + // workflow's top-level name. Bind authority to the immutable workflow path + // and trusted workflow SHA below instead of mutable display text. value.event === "workflow_run" && value.path === PR_GATE_WORKFLOW_PATH && value.head_branch === "main" &&