Skip to content

fix(maintainer): accept retryable E2E history #7584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jyaunches merged 3 commits into from
Jul 26, 2026
Merged

fix(maintainer): accept retryable E2E history #7584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da5a9f1
fix(maintainer): accept retryable E2E history
cv Jul 26, 2026
18719e9
merge: refresh check gate retry repair on main
cv Jul 26, 2026
7cd0ff7
merge: refresh #7584 onto settled main
cv Jul 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 79 additions & 3 deletions .agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,19 @@ interface E2eCoordinationEvidence {
trustedLegacyCheckId?: number;
}

const E2E_RETRYABLE_FAILURE_MARKER_PREFIX = "<!-- nemoclaw-pr-e2e-retry:v1:";
const E2E_RETRYABLE_FAILURE_MARKER_SUFFIX = " -->";
const E2E_RETRYABLE_FAILURE_REASONS = new Set([
"prerequisite-ci",
"child-cancelled",
"evidence-download",
]);
const E2E_NEVER_RETRY_FAILURE_TITLES = new Set([
"Authorized E2E run requires reconciliation",
"PR base changed",
"Controller stopped early",
"Run could not start",
]);
function parseGitHubTimestamp(value: string | undefined): number {
const match = value?.match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,3})?Z$/u);
if (!match) return Number.NaN;
Expand All @@ -335,6 +348,48 @@ function parseGitHubTimestamp(value: string | undefined): number {
: Number.NaN;
}

function hasRetryableE2eFailureMarker(check: Record<string, unknown>): boolean {
if (check.status !== "completed" || check.conclusion !== "failure") return false;
const output = check.output;
if (typeof output !== "object" || output === null || Array.isArray(output)) return false;
const { summary, title } = output as Record<string, unknown>;
if (
(title !== undefined && title !== null && typeof title !== "string") ||
E2E_NEVER_RETRY_FAILURE_TITLES.has(typeof title === "string" ? title : "")
) {
return false;
}
if (typeof summary !== "string") return false;
const markerBoundary = `\n\n${E2E_RETRYABLE_FAILURE_MARKER_PREFIX}`;
const markerStart = summary.lastIndexOf(markerBoundary);
if (markerStart < 0) return false;
const marker = summary.slice(markerStart + 2);
if (!marker.endsWith(E2E_RETRYABLE_FAILURE_MARKER_SUFFIX)) return false;
const reason = marker.slice(
E2E_RETRYABLE_FAILURE_MARKER_PREFIX.length,
-E2E_RETRYABLE_FAILURE_MARKER_SUFFIX.length,
);
return (
E2E_RETRYABLE_FAILURE_REASONS.has(reason) &&
marker ===
`${E2E_RETRYABLE_FAILURE_MARKER_PREFIX}${reason}${E2E_RETRYABLE_FAILURE_MARKER_SUFFIX}`
);
}

function currentE2eCoordinationCheck(
checks: Array<Record<string, unknown>>,
): Record<string, unknown> | undefined {
if (checks.length === 0) return undefined;
const ordered = [...checks].sort((left, right) => (left.id as number) - (right.id as number));
const active = ordered.filter((check) => check.status !== "completed");
if (active.length > 1) return undefined;
if (ordered.slice(0, -1).some((check) => !hasRetryableE2eFailureMarker(check))) {
return undefined;
}
const current = ordered.at(-1)!;
if (active[0] && active[0].id !== current.id) return undefined;
return current;
}
function fetchE2eCoordinationEvidence(
repo: string,
exactDiff: ExactDiffIdentity,
Expand Down Expand Up @@ -391,9 +446,30 @@ function fetchE2eCoordinationEvidence(
}

const externalId = `nemoclaw-pr-e2e:v2:${exactDiff.number}:${exactDiff.headSha}:${exactDiff.baseSha}`;
const exactChecks = checkRuns.filter((check) => check.external_id === externalId);
if (exactChecks.length !== 1) return { valid: false };
const exact = exactChecks[0];
const claimedChecks = checkRuns.filter((check) => check.external_id === externalId);
if (
claimedChecks.some(
(check) =>
check.head_sha !== exactDiff.headSha ||
typeof check.name !== "string" ||
!checkNames.includes(check.name) ||
typeof check.app !== "object" ||
check.app === null ||
Array.isArray(check.app) ||
(check.app as Record<string, unknown>).id !== 15368,
)
) {
return { valid: false };
}
const currentNameChecks = claimedChecks.filter(
(check) => check.name === "E2E / PR Gate Coordination",
);
const exactChecks =
currentNameChecks.length > 0
? currentNameChecks
: claimedChecks.filter((check) => check.name === "E2E / PR Gate");
const exact = currentE2eCoordinationCheck(exactChecks);
if (!exact) return { valid: false };
const app = exact.app;
const startedAt =
typeof exact.started_at === "string" ? parseGitHubTimestamp(exact.started_at) : Number.NaN;
Expand Down
129 changes: 129 additions & 0 deletions test/skills/check-gates-retry-history.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";
import { coordinationCheck, runGate } from "./check-gates-test-fixtures.ts";

const SIGNED_BODY = "Signed-off-by: Example User <user@example.com>";

function retryableFailure(id: number, reason: string, title = "Retryable E2E failure") {
return coordinationCheck({
id,
conclusion: "failure",
output: {
title,
summary: `Retryable failure.\n\n<!-- nemoclaw-pr-e2e-retry:v1:${reason} -->`,
},
});
}

function gateOutput(checkRuns: unknown[]) {
return JSON.parse(
runGate({
body: SIGNED_BODY,
verified: true,
coordinationCheckPages: [{ total_count: checkRuns.length, check_runs: checkRuns }],
}).stdout,
);
}

function expectIncompleteEvidence(checkRuns: unknown[]) {
expect(gateOutput(checkRuns).gates.ci).toMatchObject({
pass: false,
failingChecks: ["E2E / PR Gate: latest attempt evidence incomplete"],
});
}

describe("maintainer merge-gate E2E retry history", () => {
it.each([
"prerequisite-ci",
"child-cancelled",
"evidence-download",
])("accepts a later successful coordination check after a %s retry failure", (reason) => {
const output = gateOutput([coordinationCheck({ id: 8002 }), retryableFailure(8001, reason)]);

expect(output).toMatchObject({ allPass: true, gates: { ci: { pass: true } } });
});

it.each([
["an older success", [coordinationCheck({ id: 8002 }), coordinationCheck({ id: 8001 })]],
[
"an older unmarked failure",
[
coordinationCheck({ id: 8002 }),
coordinationCheck({
id: 8001,
conclusion: "failure",
output: { title: "Unknown failure", summary: "No retry marker." },
}),
],
],
[
"an unsupported retry reason",
[coordinationCheck({ id: 8002 }), retryableFailure(8001, "product-failure")],
],
[
"trailing content after the retry marker",
[
coordinationCheck({ id: 8002 }),
coordinationCheck({
id: 8001,
conclusion: "failure",
output: {
title: "Prerequisite CI failed",
summary: "Failure.\n\n<!-- nemoclaw-pr-e2e-retry:v1:prerequisite-ci --> trailing",
},
}),
],
],
[
"a never-retry title carrying a supported marker",
[
coordinationCheck({ id: 8002 }),
retryableFailure(8001, "child-cancelled", "Authorized E2E run requires reconciliation"),
],
],
[
"an older active check",
[
coordinationCheck({ id: 8002 }),
coordinationCheck({ id: 8001, status: "in_progress", conclusion: null }),
],
],
[
"multiple active checks",
[
coordinationCheck({ id: 8002, status: "in_progress", conclusion: null }),
coordinationCheck({ id: 8001, status: "in_progress", conclusion: null }),
],
],
[
"an older check from another GitHub App",
[
coordinationCheck({ id: 8002 }),
{ ...retryableFailure(8001, "prerequisite-ci"), app: { id: 1234 } },
],
],
[
"an older check reported on another head",
[
coordinationCheck({ id: 8002 }),
{ ...retryableFailure(8001, "prerequisite-ci"), head_sha: "c".repeat(40) },
],
],
[
"any non-retryable check in older history",
[
coordinationCheck({ id: 8003 }),
coordinationCheck({
id: 8002,
conclusion: "failure",
output: { title: "Unknown failure", summary: "No retry marker." },
}),
retryableFailure(8001, "prerequisite-ci"),
],
],
])("fails closed with %s in the exact coordination history", (_name, checks) => {
expectIncompleteEvidence(checks);
});
});
Loading