diff --git a/test/e2e/support/same-commit-reliability.test.ts b/test/e2e/support/same-commit-reliability.test.ts index 2f5dabfcf41..3c31bc424e8 100644 --- a/test/e2e/support/same-commit-reliability.test.ts +++ b/test/e2e/support/same-commit-reliability.test.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import path from "node:path"; + import { describe, expect, it, vi } from "vitest"; import { @@ -24,6 +27,7 @@ const SHA_A = "a".repeat(40); const SHA_B = "b".repeat(40); const REPOSITORY = "NVIDIA/NemoClaw"; const RETRY_WORKFLOW_PATH = ".github/workflows/e2e-main-retry.yaml"; +const REPORTER_PATH = path.resolve("tools/e2e/same-commit-reliability.mts"); function sample( runId: number, @@ -78,6 +82,25 @@ function controllerRun(id: number, overrides: Record = {}) { }; } +describe("same-commit reliability reporter entrypoint", () => { + it("loads with the raw Node strip-types runtime used by CI", () => { + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", REPORTER_PATH], + { + encoding: "utf8", + env: { ...process.env, GITHUB_TOKEN: "", SOURCE_RUN_ID: "" }, + killSignal: "SIGKILL", + timeout: 20_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("GITHUB_TOKEN is required"); + expect(result.stderr).not.toContain("ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX"); + }); +}); + function retryEvidence(runId: number, sourceSha: string): Buffer { return artifactZip([ { diff --git a/tools/e2e/retry-evidence.mts b/tools/e2e/retry-evidence.mts index c228370bb6c..beb6571d7b5 100644 --- a/tools/e2e/retry-evidence.mts +++ b/tools/e2e/retry-evidence.mts @@ -159,11 +159,11 @@ export function validateRetryEvidence(value: unknown): RetryEvidence | null { } export class RetryPolicyError extends Error { - constructor( - message: string, - readonly evidence: RetryEvidence, - ) { + readonly evidence: RetryEvidence; + + constructor(message: string, evidence: RetryEvidence) { super(message); + this.evidence = evidence; } } diff --git a/tools/e2e/same-commit-reliability.mts b/tools/e2e/same-commit-reliability.mts index b624b24f00d..dcbd0653ee8 100755 --- a/tools/e2e/same-commit-reliability.mts +++ b/tools/e2e/same-commit-reliability.mts @@ -768,7 +768,10 @@ export function formatReliabilityReport(groups: readonly ReliabilityGroup[]): st `| ${group.source} | ${group.candidateSha ? `\`${group.candidateSha.slice(0, 12)}\`` : "unclassified"} | ${group.runs} | ${group.passedFirstAttempt} | ${group.passedAfterRetry} | ${group.exhausted} | ${group.failedFirstAttempt} | ${group.superseded} | ${group.unclassified} | ${group.passFailFlips} | ${(group.firstPassRate * 100).toFixed(1)}% | ${group.recoveryRate === null ? "n/a" : `${(group.recoveryRate * 100).toFixed(1)}%`} | ${classes || "none"} | ${formatEvidenceCounts(group.evidence)} | ${formatEvidenceCounts(group.failureClassEvidence)} |`, ); } - lines.push("", `### Non-passing run links (maximum ${MAX_RUN_REFERENCES_PER_OUTCOME} per outcome)`); + lines.push( + "", + `### Non-passing run links (maximum ${MAX_RUN_REFERENCES_PER_OUTCOME} per outcome)`, + ); for (const group of groups) { for (const outcome of ["failed-first-attempt", "exhausted", "unclassified"] as const) { const referenceGroup = group.runReferences[outcome]; @@ -826,14 +829,20 @@ type GithubAttemptFailure = { type GithubFailureCategory = "invalid-json" | "too-large" | "transport"; class GithubHttpStatusError extends Error { - constructor(readonly status: number) { + readonly status: number; + + constructor(status: number) { super("GitHub HTTP status failure"); + this.status = status; } } class GithubCategorizedError extends Error { - constructor(readonly category: GithubFailureCategory) { + readonly category: GithubFailureCategory; + + constructor(category: GithubFailureCategory) { super(`GitHub HTTP ${category} failure`); + this.category = category; } }