Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b6892e6
perf(ci): shorten main and E2E feedback loops
prekshivyas Jul 20, 2026
908a2bd
chore(ci): sync speedup branch with main
prekshivyas Jul 20, 2026
3da8b5b
fix(ci): address speedup review edge cases
prekshivyas Jul 20, 2026
7766a8e
test(ci): cover scheduled Brev inspection
prekshivyas Jul 20, 2026
f6569e4
chore(ci): sync speedup branch with current main
prekshivyas Jul 20, 2026
cc900d6
docs(ci): align direct E2E gate guidance
prekshivyas Jul 21, 2026
ab3c76d
test(ci): enforce platform gate contracts
prekshivyas Jul 21, 2026
492c3bf
fix(e2e): reject ambiguous Brev inventory
prekshivyas Jul 21, 2026
7112cde
Merge branch 'main' into codex/ci-e2e-speedup
prekshivyas Jul 21, 2026
53a05e8
Merge branch 'main' into codex/ci-e2e-speedup
prekshivyas Jul 21, 2026
02f25d5
merge(ci): sync E2E speedup with main
prekshivyas Jul 22, 2026
b013ee5
docs(ci): align merged E2E budgets and safeguards
prekshivyas Jul 22, 2026
314ef46
test(ci): address speedup review follow-ups
cv Jul 22, 2026
2348422
merge(ci): reconcile delegated E2E approvals
cv Jul 22, 2026
916b824
docs(ci): document protected E2E authorization
cv Jul 22, 2026
a625749
Merge remote-tracking branch 'origin/main' into codex/review-pr7262
cv Jul 22, 2026
6d56aeb
Merge remote-tracking branch 'origin/main' into codex/review-pr7262
cv Jul 22, 2026
6b8280f
merge(ci): sync E2E speedup with current main
prekshivyas Jul 23, 2026
0091991
docs(e2e): align protected-reviewer guidance
prekshivyas Jul 23, 2026
c7e8182
merge: refresh PR #7262 onto main
cjagwani Jul 23, 2026
9585e67
merge(ci): sync staging Brev E2E
prekshivyas Jul 23, 2026
423ad38
merge(ci): reconcile concurrent PR refresh
prekshivyas Jul 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 68 additions & 19 deletions .agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ interface E2eCoordinationEvidence {
valid: boolean | null;
startedAt?: number;
completedAt?: number;
trustedLegacyCheckId?: number;
trustedCustomCheckId?: number;
}

function parseGitHubTimestamp(value: string | undefined): number {
Expand All @@ -339,7 +339,7 @@ function fetchE2eCoordinationEvidence(
repo: string,
exactDiff: ExactDiffIdentity,
): E2eCoordinationEvidence {
const checkNames = ["E2E / PR Gate Coordination", "E2E / PR Gate"];
const checkNames = ["E2E / PR Gate", "E2E / PR Gate Coordination"];
Comment thread
coderabbitai[bot] marked this conversation as resolved.
const checkRuns: Array<Record<string, unknown>> = [];
const ids = new Set<number>();
for (const checkName of checkNames) {
Expand Down Expand Up @@ -391,7 +391,13 @@ function fetchE2eCoordinationEvidence(
}

const externalId = `nemoclaw-pr-e2e:v2:${exactDiff.number}:${exactDiff.headSha}:${exactDiff.baseSha}`;
const exactChecks = checkRuns.filter((check) => check.external_id === externalId);
const currentExactChecks = checkRuns.filter(
(check) => check.name === "E2E / PR Gate" && check.external_id === externalId,
);
const formerExactChecks = checkRuns.filter(
(check) => check.name === "E2E / PR Gate Coordination" && check.external_id === externalId,
);
const exactChecks = currentExactChecks.length > 0 ? currentExactChecks : formerExactChecks;
if (exactChecks.length !== 1) return { valid: false };
const exact = exactChecks[0];
const app = exact.app;
Expand All @@ -416,7 +422,7 @@ function fetchE2eCoordinationEvidence(
valid,
...(valid ? { startedAt, completedAt } : {}),
...(valid && exact.name === "E2E / PR Gate"
? { trustedLegacyCheckId: exact.id as number }
? { trustedCustomCheckId: exact.id as number }
: {}),
};
}
Expand Down Expand Up @@ -939,11 +945,11 @@ function currentCheckRollup(
const actionRunId = (check: StatusCheck): string | undefined =>
check.detailsUrl?.match(/\/actions\/runs\/(\d+)(?:\/|$)/)?.[1];

const isTrustedLegacyE2eCheck = (check: StatusCheck): boolean =>
e2eCoordinationEvidence.trustedLegacyCheckId !== undefined &&
const isTrustedCustomE2eCheck = (check: StatusCheck): boolean =>
e2eCoordinationEvidence.trustedCustomCheckId !== undefined &&
check.name === "E2E / PR Gate" &&
check.detailsUrl?.match(/\/runs\/(\d+)(?:[/?#]|$)/u)?.[1] ===
String(e2eCoordinationEvidence.trustedLegacyCheckId);
String(e2eCoordinationEvidence.trustedCustomCheckId);

function runIdentityEvidence(
runId: string,
Expand Down Expand Up @@ -1038,7 +1044,7 @@ function currentCheckRollup(
);
if (
(requiredCheck || expectsActionEvidence) &&
group.some((check) => !actionRunId(check) && !isTrustedLegacyE2eCheck(check))
group.some((check) => !actionRunId(check) && !isTrustedCustomE2eCheck(check))
) {
incompleteAttemptEvidence.add(groupName);
}
Expand Down Expand Up @@ -1194,14 +1200,21 @@ function currentCheckRollup(
incompleteAttemptEvidence.add("checks");
incompleteAttemptEvidence.add("changes");
}
return { checks: current, incompleteAttemptEvidence: [...incompleteAttemptEvidence].sort() };
return {
checks: current,
incompleteAttemptEvidence: [...incompleteAttemptEvidence].sort(),
};
}

function checkCi(
statusCheckRollup: StatusCheck[] | null,
repo: string,
exactDiff: ExactDiffIdentity,
): GateResult & { failingChecks?: string[]; pendingChecks?: string[]; missingChecks?: string[] } {
): GateResult & {
failingChecks?: string[];
pendingChecks?: string[];
missingChecks?: string[];
} {
if (!statusCheckRollup || statusCheckRollup.length === 0) {
return { pass: false, details: "No status checks found" };
}
Expand Down Expand Up @@ -1266,7 +1279,11 @@ function checkCi(
};
}
if (pending.length > 0) {
return { pass: false, details: `${pending.length} pending check(s)`, pendingChecks: pending };
return {
pass: false,
details: `${pending.length} pending check(s)`,
pendingChecks: pending,
};
}
if (incompleteAttemptEvidence.size > 0) {
const incompleteNames = [...incompleteAttemptEvidence].sort();
Expand All @@ -1276,7 +1293,10 @@ function checkCi(
failingChecks: incompleteNames.map((name) => `${name}: latest attempt evidence incomplete`),
};
}
return { pass: true, details: `All ${currentChecks.length} current checks green` };
return {
pass: true,
details: `All ${currentChecks.length} current checks green`,
};
}

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1360,7 +1380,11 @@ function fetchCurrentBaseSha(repo: string, number: number): string | null {
}
}`,
]) as {
data?: { repository?: { pullRequest?: { baseRef?: { target?: { oid?: unknown } } } } };
data?: {
repository?: {
pullRequest?: { baseRef?: { target?: { oid?: unknown } } };
};
};
} | null;
const oid = response?.data?.repository?.pullRequest?.baseRef?.target?.oid;
return typeof oid === "string" && /^[0-9a-f]{40}$/i.test(oid) ? oid : null;
Expand Down Expand Up @@ -1430,7 +1454,10 @@ function checkCodeRabbit(

// Fail-closed: if we cannot reach the API, do not assume clean
if (!out) {
return { pass: false, details: "Could not fetch review threads (API error — fail-closed)" };
return {
pass: false,
details: "Could not fetch review threads (API error — fail-closed)",
};
}

let data: {
Expand All @@ -1440,7 +1467,13 @@ function checkCodeRabbit(
reviewThreads?: {
nodes?: Array<{
isResolved: boolean;
comments: { nodes: Array<{ author: { login: string }; body: string; path: string }> };
comments: {
nodes: Array<{
author: { login: string };
body: string;
path: string;
}>;
};
}>;
};
};
Expand All @@ -1450,7 +1483,10 @@ function checkCodeRabbit(
try {
data = JSON.parse(out);
} catch {
return { pass: false, details: "Could not parse review threads (invalid JSON — fail-closed)" };
return {
pass: false,
details: "Could not parse review threads (invalid JSON — fail-closed)",
};
}

const threads = data.data?.repository?.pullRequest?.reviewThreads?.nodes ?? [];
Expand Down Expand Up @@ -1479,7 +1515,10 @@ function checkCodeRabbit(
}

if (unresolved.length === 0) {
return { pass: true, details: "No unresolved major/critical CodeRabbit findings" };
return {
pass: true,
details: "No unresolved major/critical CodeRabbit findings",
};
}
return {
pass: false,
Expand Down Expand Up @@ -1533,7 +1572,11 @@ interface CommitVerificationRecord {

function normalizeCommitVerification(value: unknown): CommitVerificationRecord {
if (typeof value !== "object" || value === null || Array.isArray(value)) {
return { sha: "(unknown)", verified: false, reason: "malformed_commit_verification_data" };
return {
sha: "(unknown)",
verified: false,
reason: "malformed_commit_verification_data",
};
}

const record = value as Record<string, unknown>;
Expand Down Expand Up @@ -1848,7 +1891,13 @@ function main(): void {
coderabbit.pass &&
riskyCodeTested.pass &&
contributorCompliance.pass,
gates: { ci, conflicts, coderabbit, riskyCodeTested, contributorCompliance },
gates: {
ci,
conflicts,
coderabbit,
riskyCodeTested,
contributorCompliance,
},
advisories: { contributorApprovalOverlap },
};

Expand Down
29 changes: 16 additions & 13 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ env:
jobs:
generate-matrix:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
matrix: ${{ steps.controller_matrix.outputs.matrix || steps.matrix.outputs.matrix }}
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
Expand Down Expand Up @@ -780,7 +781,7 @@ jobs:
e2e-artifacts/live/skill-agent/*/shell/*.stdout.txt
e2e-artifacts/live/skill-agent/*/shell/*.stderr.txt

# Direct OpenClaw skills CLI contract.
# Direct OpenClaw skills CLI contract.

- name: Clean up Docker auth
if: always()
Expand Down Expand Up @@ -3626,7 +3627,7 @@ jobs:
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

# Provider-routed Model Router inference contract.
# Provider-routed Model Router inference contract.

- name: Clean up Docker auth
if: always()
Expand Down Expand Up @@ -4002,15 +4003,15 @@ jobs:
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

# ── Free-standing recovery targets (#2701) ─────────────────────────
# Recovery / disruption targets don't fit the steady-state expected-state
# registry that drives `live` above. They run as free-standing
# Vitest test files using the same `e2e-live` project, framework
# fixtures, and live-project gate — just outside the matrix.
#
# First failing-test-first guard for #2701 (gateway recovery does not
# restore the /tmp guard chain after pod recreate). Will fail on `main`
# until the #2701 fix lands; flips green afterwards.
# ── Free-standing recovery targets (#2701) ─────────────────────────
# Recovery / disruption targets don't fit the steady-state expected-state
# registry that drives `live` above. They run as free-standing
# Vitest test files using the same `e2e-live` project, framework
# fixtures, and live-project gate — just outside the matrix.
#
# First failing-test-first guard for #2701 (gateway recovery does not
# restore the /tmp guard chain after pod recreate). Will fail on `main`
# until the #2701 fix lands; flips green afterwards.

- name: Clean up Docker auth
if: always()
Expand Down Expand Up @@ -4851,11 +4852,12 @@ jobs:
# comment.
report-to-pr:
runs-on: ubuntu-latest
timeout-minutes: 15
# This entire workflow is dispatch-only. Keeping selective jobs in `needs`
# makes the report wait for and record any requested job without adding
# skipped checks to the normal pull_request workflow.
needs: &e2e-result-jobs
[
needs:
&e2e-result-jobs [
generate-matrix,
live,
shared-e2e,
Expand Down Expand Up @@ -4987,6 +4989,7 @@ jobs:
# ── Scheduled/manual scorecard ────────────────────────────────────────────
scorecard:
runs-on: ubuntu-latest
timeout-minutes: 15
needs: *e2e-result-jobs
if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }}
permissions:
Expand Down
10 changes: 8 additions & 2 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,7 @@ jobs:
- real-openclaw-dist-harness
- cli-tests
- plugin-tests
- sandbox-images-and-e2e
if: always()
permissions:
actions: read
Expand All @@ -267,6 +268,7 @@ jobs:
CLI_TESTS_RESULT: ${{ needs['cli-tests'].result }}
GH_TOKEN: ${{ github.token }}
PLUGIN_TESTS_RESULT: ${{ needs['plugin-tests'].result }}
SANDBOX_IMAGES_E2E_RESULT: ${{ needs['sandbox-images-and-e2e'].result }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
Expand Down Expand Up @@ -304,7 +306,7 @@ jobs:
local job_id=""
dependency_url="$RUN_URL"
case "$name" in
static-checks|build-typecheck|installer-integration|wechat-runtime-audit|reviewed-npm-audit|real-openclaw-dist-harness|cli-tests|plugin-tests) ;;
static-checks|build-typecheck|installer-integration|wechat-runtime-audit|reviewed-npm-audit|real-openclaw-dist-harness|cli-tests|plugin-tests|sandbox-images-and-e2e) ;;
*) return ;;
esac
load_job_listing
Expand Down Expand Up @@ -347,11 +349,15 @@ jobs:
require_success "real-openclaw-dist-harness" "$REAL_OPENCLAW_DIST_HARNESS_RESULT"
require_success "cli-tests" "$CLI_TESTS_RESULT"
require_success "plugin-tests" "$PLUGIN_TESTS_RESULT"
require_success "sandbox-images-and-e2e" "$SANDBOX_IMAGES_E2E_RESULT"

[ "$failed" -eq 0 ]

sandbox-images-and-e2e:
needs: checks
# The image workflow consumes the source revision directly. Start it after
# the cheap compile/static preflight instead of serializing it behind the
# complete coverage and integration suite.
needs: [static-checks, build-typecheck]
uses: ./.github/workflows/sandbox-images-and-e2e.yaml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/platform-vitest-main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,11 @@ jobs:

macos-vitest:
runs-on: macos-26
timeout-minutes: 60
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4]
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
Expand Down Expand Up @@ -143,11 +147,15 @@ jobs:
npm run build

- name: Run full Vitest suite on macOS
run: npx vitest run --testTimeout 60000
run: npx vitest run --testTimeout 60000 --shard="${{ matrix.shard }}/4"

wsl-vitest:
runs-on: windows-latest
timeout-minutes: 60
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4]
env:
WSL_DISTRO: Ubuntu
WSL_TEST_USER: nemoclaw-ci
Expand Down Expand Up @@ -305,14 +313,15 @@ jobs:
cd '$env:WSL_WORKDIR'
export NEMOCLAW_EXEC_TIMEOUT=60000
export NEMOCLAW_TEST_TIMEOUT=60000
npx vitest run --testTimeout 60000
npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4'
"@
$tmp = "$env:RUNNER_TEMP\wsl-step.sh"
[IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false))
$wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/')
wsl -d $env:WSL_DISTRO --user $env:WSL_TEST_USER -- bash -l $wslTmp

- name: Run root-required Vitest contracts in WSL
if: ${{ matrix.shard == 1 }}
shell: powershell
run: |
$script = @"
Expand Down
Loading
Loading