diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 4db501235f7..23e841a42e2 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -451,7 +451,7 @@ describe("base-image publication evidence", () => { ); expect(selection).toMatchObject({ - state: "ready", + state: "selected", run: { headSha: EXPECTED_SHA }, }); }); @@ -475,7 +475,7 @@ describe("base-image publication evidence", () => { ); expect(selection).toMatchObject({ - state: "ready", + state: "selected", run: { id: 11, headSha: DESCENDANT_SHA }, }); }); @@ -495,7 +495,7 @@ describe("base-image publication evidence", () => { WORKFLOW_ID, ); - expect(selection).toMatchObject({ state: "ready", run: { id: RUN_ID } }); + expect(selection).toMatchObject({ state: "selected", run: { id: RUN_ID } }); }); it("rejects pre-rename workflow metadata inside the eligible history (#7372)", () => { @@ -508,7 +508,7 @@ describe("base-image publication evidence", () => { ).toThrow(/name must be Images \/ Base Images/u); }); - it("waits for missing and in-progress publication evidence (#7372)", () => { + it("selects an in-progress trusted publication run (#9549)", () => { expect(selectPublicationRun(runsPayload([]), history(), WORKFLOW_ID)).toEqual({ state: "missing", }); @@ -518,15 +518,15 @@ describe("base-image publication evidence", () => { history(), WORKFLOW_ID, ), - ).toMatchObject({ state: "pending", run: { status: "in_progress" } }); + ).toMatchObject({ state: "selected", run: { status: "in_progress" } }); }); it.each(["failure", "cancelled"] as const)( - "fails closed when publication concludes %s (#7372)", + "selects a terminal %s run for publisher validation (#9549)", (conclusion) => { - expect(() => + expect( selectPublicationRun(runsPayload([workflowRun({ conclusion })]), history(), WORKFLOW_ID), - ).toThrow(`base-image workflow for ${RELEVANT_SHA} concluded ${conclusion}; ${RUN_URL}`); + ).toMatchObject({ state: "selected", run: { conclusion } }); }, ); @@ -560,29 +560,67 @@ describe("base-image publication evidence", () => { ).toThrow(/incomplete/u); }); - it("requires every publisher latest attempt to complete successfully (#7372)", () => { + it("requires every publisher job to belong to the selected attempt (#9549)", () => { const run = selectedRun({ attempt: 2 }); - const jobs = [ - ...successfulJobs(), - publisherJob("Build and push Hermes base image", { - id: 4, - run_attempt: 1, - conclusion: "failure", - }), - publisherJob("Build and push Hermes base image", { - id: 5, - run_attempt: 2, - }), - ].filter((job, index) => index !== 1); + const jobs = successfulJobs({ runAttempt: 2 }); - expect(() => validatePublisherJobs({ total_count: jobs.length, jobs }, run)).not.toThrow(); + expect(validatePublisherJobs({ total_count: jobs.length, jobs }, run)).toBe("ready"); + expect(() => + validatePublisherJobs( + { + total_count: jobs.length, + jobs: jobs.map((job, index) => (index === 0 ? { ...job, run_attempt: 1 } : job)), + }, + run, + ), + ).toThrow(/provenance does not match/u); }); - it("reconfirms the selected successful run after reading job history (#7372)", () => { + it("classifies an incomplete required publisher as pending only while the selected run is in progress (#9549)", () => { + const jobs = successfulJobs().map((job) => + job.name === "Build and push Hermes base image" + ? { ...job, status: "in_progress", conclusion: null } + : job, + ); + + expect( + validatePublisherJobs( + { total_count: jobs.length, jobs }, + selectedRun({ status: "in_progress", conclusion: null }), + ), + ).toBe("pending"); + expect(() => validatePublisherJobs({ total_count: jobs.length, jobs }, selectedRun())).toThrow( + /not complete in terminal attempt 1/u, + ); + }); + + it.each(["failure", "cancelled", "skipped"])( + "rejects a required publisher that concludes %s before the workflow completes (#9549)", + (conclusion) => { + const jobs = successfulJobs().map((job) => + job.name === "Build and push Hermes base image" ? { ...job, conclusion } : job, + ); + + expect(() => + validatePublisherJobs( + { total_count: jobs.length, jobs }, + selectedRun({ status: "in_progress", conclusion: null }), + ), + ).toThrow(/did not complete successfully in attempt 1/u); + }, + ); + + it("reconfirms the selected run identity after reading job history (#9549)", () => { expect(() => validateBoundRun(workflowRun(), selectedRun())).not.toThrow(); expect(() => - validateBoundRun(workflowRun({ status: "in_progress", conclusion: null }), selectedRun()), - ).toThrow(/changed while evidence was verified/u); + validateBoundRun( + workflowRun({ conclusion: "cancelled" }), + selectedRun({ status: "in_progress", conclusion: null }), + ), + ).not.toThrow(); + expect(() => validateBoundRun(workflowRun({ run_attempt: 2 }), selectedRun())).toThrow( + /changed while evidence was verified/u, + ); }); it("exports the selected immutable publication identity for downstream qualification (#9049)", () => { @@ -609,7 +647,7 @@ describe("base-image publication evidence", () => { /duplicated in attempt/u, ], [ - "failed latest attempt", + "failed selected attempt", successfulJobs().map((job) => job.name === "Build and push Hermes base image" ? { ...job, conclusion: "failure" } : job, ), @@ -626,11 +664,12 @@ describe("base-image publication evidence", () => { ); }); - it("polls from missing through completion and verifies jobs (#7372)", async () => { + it("polls from missing through publisher completion and verifies jobs (#9549)", async () => { const responses = [ workflowMetadata(), runsPayload([]), runsPayload([workflowRun({ status: "queued", conclusion: null })]), + { total_count: 0, jobs: [] }, runsPayload([workflowRun()]), { total_count: 3, jobs: successfulJobs() }, workflowRun(), @@ -659,13 +698,75 @@ describe("base-image publication evidence", () => { "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml", "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", + `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/1/jobs?per_page=100&page=1`, "/repos/NVIDIA/NemoClaw/actions/workflows/base-image.yaml/runs?branch=main&event=push&per_page=100&page=1", - `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/jobs?filter=all&per_page=100&page=1`, + `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}/attempts/1/jobs?per_page=100&page=1`, `/repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}`, ]); expect(notices).toHaveLength(2); }); + it("accepts required publishers while managed-image jobs remain in progress (#9549)", async () => { + const inProgressRun = workflowRun({ status: "in_progress", conclusion: null }); + const jobs = [ + ...successfulJobs(), + publisherJob("Build and validate OpenClaw managed image (amd64)", { + id: 4, + status: "in_progress", + conclusion: null, + }), + ]; + const responses = [ + workflowMetadata(), + runsPayload([inProgressRun]), + { total_count: jobs.length, jobs }, + inProgressRun, + ]; + let sleeps = 0; + + await expect( + waitForBaseImagePublication({ + history: history(), + request: async () => responses.shift(), + waitMs: 100, + pollMs: 10, + sleep: async () => { + sleeps += 1; + }, + }), + ).resolves.toMatchObject({ id: RUN_ID, status: "in_progress" }); + expect(sleeps).toBe(0); + }); + + it.each(["failure", "cancelled"] as const)( + "accepts required publishers after unrelated downstream work concludes %s (#9549)", + async (conclusion) => { + const terminalRun = workflowRun({ conclusion }); + const jobs = [ + ...successfulJobs(), + publisherJob("Build and validate OpenClaw managed image (amd64)", { + id: 4, + conclusion, + }), + ]; + const responses = [ + workflowMetadata(), + runsPayload([terminalRun]), + { total_count: jobs.length, jobs }, + terminalRun, + ]; + + await expect( + waitForBaseImagePublication({ + history: history(), + request: async () => responses.shift(), + waitMs: 100, + pollMs: 10, + }), + ).resolves.toMatchObject({ id: RUN_ID, conclusion }); + }, + ); + it("reports the selected publisher SHA and run URL for invalid job evidence (#7372)", async () => { const jobs = successfulJobs().map((job, index) => index === 0 ? { ...job, run_id: RUN_ID + 1 } : job, diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index 83fdd1bf3ca..65de753e55f 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -93,8 +93,7 @@ export interface PublicationRun { export type PublicationSelection = | { state: "missing" } - | { state: "pending"; run: PublicationRun } - | { state: "ready"; run: PublicationRun }; + | { state: "selected"; run: PublicationRun }; export interface PublicationWaitOptions { history: FirstParentHistory; @@ -435,16 +434,10 @@ export function selectPublicationRun( ); } const run = nearest[0].run; - if (run.status !== "completed") return { state: "pending", run }; - if (run.conclusion !== "success") { - throw new Error( - `base-image workflow for ${run.headSha} concluded ${run.conclusion}; ${run.url}`, - ); - } - return { state: "ready", run }; + return { state: "selected", run }; } -export function validatePublisherJobs(payload: unknown, run: PublicationRun): void { +export function validatePublisherJobs(payload: unknown, run: PublicationRun): "pending" | "ready" { const response = asRecord(payload); const totalCount = Number(response.total_count); if (!Number.isSafeInteger(totalCount) || totalCount < 0) { @@ -454,52 +447,68 @@ export function validatePublisherJobs(payload: unknown, run: PublicationRun): vo throw new Error("publisher job listing is incomplete"); } - const jobsByName = new Map< - string, - Array<{ attempt: number; status: string; conclusion: string }> - >(); + const jobsByName = new Map(); for (const [index, value] of response.jobs.entries()) { const job = asRecord(value); positiveSafeInteger(job.id, `publisher job ${index} id`); const attempt = positiveSafeInteger(job.run_attempt, `publisher job ${index} attempt`); - if (job.run_id !== run.id || attempt > run.attempt || job.head_sha !== run.headSha) { + if (job.run_id !== run.id || attempt !== run.attempt || job.head_sha !== run.headSha) { throw new Error(`publisher job ${index} provenance does not match the selected run`); } if (typeof job.name !== "string" || job.name.length === 0) { throw new Error(`publisher job ${index} name is invalid`); } - if ( - job.status !== "completed" || - typeof job.conclusion !== "string" || - !COMPLETED_CONCLUSIONS.has(job.conclusion) - ) { - throw new Error(`publisher job ${job.name} completion evidence is invalid; ${run.url}`); + const requiredName = REQUIRED_PUBLISHER_JOBS.find((name) => name === job.name); + if (!requiredName) continue; + if (typeof job.status !== "string") { + throw new Error(`publisher job ${requiredName} status is invalid; ${run.url}`); } - const occurrences = jobsByName.get(job.name) ?? []; - if (occurrences.some((occurrence) => occurrence.attempt === attempt)) { - throw new Error(`publisher job ${job.name} is duplicated in attempt ${attempt}; ${run.url}`); + const status = job.status; + let conclusion: string | null = null; + if (status === "completed") { + if (typeof job.conclusion !== "string" || !COMPLETED_CONCLUSIONS.has(job.conclusion)) { + throw new Error(`publisher job ${requiredName} conclusion is invalid; ${run.url}`); + } + conclusion = job.conclusion; + } else if (!PENDING_RUN_STATUSES.has(status) || job.conclusion !== null) { + throw new Error(`publisher job ${requiredName} pending state is invalid; ${run.url}`); } - occurrences.push({ - attempt, - status: job.status, - conclusion: job.conclusion, - }); - jobsByName.set(job.name, occurrences); + if (jobsByName.has(requiredName)) { + throw new Error( + `publisher job ${requiredName} is duplicated in attempt ${run.attempt}; ${run.url}`, + ); + } + jobsByName.set(requiredName, { status, conclusion }); } + let pending = false; for (const requiredName of REQUIRED_PUBLISHER_JOBS) { - const occurrences = jobsByName.get(requiredName) ?? []; - if (occurrences.length === 0) { - throw new Error(`missing required ${requiredName} job; ${run.url}`); + const current = jobsByName.get(requiredName); + if (!current) { + if (run.status === "completed") { + throw new Error( + `missing required ${requiredName} job in attempt ${run.attempt}; ${run.url}`, + ); + } + pending = true; + continue; } - const latestAttempt = Math.max(...occurrences.map((occurrence) => occurrence.attempt)); - const latest = occurrences.find((occurrence) => occurrence.attempt === latestAttempt); - if (!latest || latest.status !== "completed" || latest.conclusion !== "success") { + if (current.status !== "completed") { + if (run.status === "completed") { + throw new Error( + `${requiredName} job is not complete in terminal attempt ${run.attempt}; ${run.url}`, + ); + } + pending = true; + continue; + } + if (current.conclusion !== "success") { throw new Error( - `latest ${requiredName} job did not complete successfully in attempt ${latestAttempt}; ${run.url}`, + `${requiredName} job did not complete successfully in attempt ${run.attempt}; ${run.url}`, ); } } + return pending ? "pending" : "ready"; } export function validateBoundRun(payload: unknown, expected: PublicationRun): void { @@ -507,9 +516,7 @@ export function validateBoundRun(payload: unknown, expected: PublicationRun): vo if ( actual.id !== expected.id || actual.attempt !== expected.attempt || - actual.headSha !== expected.headSha || - actual.status !== "completed" || - actual.conclusion !== "success" + actual.headSha !== expected.headSha ) { throw new Error( `selected base-image workflow changed while evidence was verified; ${expected.url}`, @@ -619,40 +626,45 @@ export async function waitForBaseImagePublication( while (true) { const runs = await collectPaginated(options.request, runsPath, "workflow_runs"); const selection = selectPublicationRun(runs, options.history, workflowId); - if (selection.state === "ready") { - const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/jobs?filter=all&per_page=100`; + if (selection.state === "selected") { + const jobsPath = `/repos/${REPOSITORY}/actions/runs/${selection.run.id}/attempts/${selection.run.attempt}/jobs?per_page=100`; if (now() > deadline) { throw new Error( `timed out validating base-image publication for ${selection.run.headSha}; ${selection.run.url}`, ); } + let publisherState: "pending" | "ready"; try { const jobs = await collectPaginated(options.request, jobsPath, "jobs"); - validatePublisherJobs(jobs, selection.run); - validateBoundRun( - await options.request(`/repos/${REPOSITORY}/actions/runs/${selection.run.id}`), - selection.run, - ); + publisherState = validatePublisherJobs(jobs, selection.run); + if (publisherState === "ready") { + validateBoundRun( + await options.request(`/repos/${REPOSITORY}/actions/runs/${selection.run.id}`), + selection.run, + ); + } } catch (error) { throw publicationEvidenceError(error, selection.run); } - if (now() > deadline) { - throw new Error( - `timed out validating base-image publication for ${selection.run.headSha}; ${selection.run.url}`, - ); + if (publisherState === "ready") { + if (now() > deadline) { + throw new Error( + `timed out validating base-image publication for ${selection.run.headSha}; ${selection.run.url}`, + ); + } + return selection.run; } - return selection.run; } if (now() >= deadline) { - const pending = selection.state === "pending" ? `; ${selection.run.url}` : ""; + const pending = selection.state === "selected" ? `; ${selection.run.url}` : ""; throw new Error( `timed out waiting for base-image publication covering ${options.history.relevantSha}${pending}`, ); } notice( - selection.state === "pending" - ? `Base-image publication is ${selection.run.status} for ${selection.run.headSha}; ${selection.run.url}` + selection.state === "selected" + ? `Required base image publishers are not complete for ${selection.run.headSha}; selected workflow run status ${selection.run.status}; ${selection.run.url}` : `Waiting for a trusted base-image push run covering ${options.history.relevantSha}`, ); await sleep(Math.min(options.pollMs, Math.max(1, deadline - now())));