Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions test/e2e/support/same-commit-reliability.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import path from "node:path";

import { describe, expect, it, vi } from "vitest";

import {
Expand All @@ -24,6 +27,7 @@ const SHA_A = "a".repeat(40);
const SHA_B = "b".repeat(40);
const REPOSITORY = "NVIDIA/NemoClaw";
const RETRY_WORKFLOW_PATH = ".github/workflows/e2e-main-retry.yaml";
const REPORTER_PATH = path.resolve("tools/e2e/same-commit-reliability.mts");

function sample(
runId: number,
Expand Down Expand Up @@ -78,6 +82,25 @@ function controllerRun(id: number, overrides: Record<string, unknown> = {}) {
};
}

describe("same-commit reliability reporter entrypoint", () => {
it("loads with the raw Node strip-types runtime used by CI", () => {
const result = spawnSync(
process.execPath,
["--experimental-strip-types", "--no-warnings", REPORTER_PATH],
{
encoding: "utf8",
env: { ...process.env, GITHUB_TOKEN: "", SOURCE_RUN_ID: "" },
killSignal: "SIGKILL",
timeout: 20_000,
},
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

expect(result.status).toBe(1);
expect(result.stderr).toContain("GITHUB_TOKEN is required");
expect(result.stderr).not.toContain("ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX");
});
});

function retryEvidence(runId: number, sourceSha: string): Buffer {
return artifactZip([
{
Expand Down
8 changes: 4 additions & 4 deletions tools/e2e/retry-evidence.mts
Original file line number Diff line number Diff line change
Expand Up @@ -159,11 +159,11 @@ export function validateRetryEvidence(value: unknown): RetryEvidence | null {
}

export class RetryPolicyError extends Error {
constructor(
message: string,
readonly evidence: RetryEvidence,
) {
readonly evidence: RetryEvidence;

constructor(message: string, evidence: RetryEvidence) {
super(message);
this.evidence = evidence;
}
}

Expand Down
15 changes: 12 additions & 3 deletions tools/e2e/same-commit-reliability.mts
Original file line number Diff line number Diff line change
Expand Up @@ -768,7 +768,10 @@ export function formatReliabilityReport(groups: readonly ReliabilityGroup[]): st
`| ${group.source} | ${group.candidateSha ? `\`${group.candidateSha.slice(0, 12)}\`` : "unclassified"} | ${group.runs} | ${group.passedFirstAttempt} | ${group.passedAfterRetry} | ${group.exhausted} | ${group.failedFirstAttempt} | ${group.superseded} | ${group.unclassified} | ${group.passFailFlips} | ${(group.firstPassRate * 100).toFixed(1)}% | ${group.recoveryRate === null ? "n/a" : `${(group.recoveryRate * 100).toFixed(1)}%`} | ${classes || "none"} | ${formatEvidenceCounts(group.evidence)} | ${formatEvidenceCounts(group.failureClassEvidence)} |`,
);
}
lines.push("", `### Non-passing run links (maximum ${MAX_RUN_REFERENCES_PER_OUTCOME} per outcome)`);
lines.push(
"",
`### Non-passing run links (maximum ${MAX_RUN_REFERENCES_PER_OUTCOME} per outcome)`,
);
for (const group of groups) {
for (const outcome of ["failed-first-attempt", "exhausted", "unclassified"] as const) {
const referenceGroup = group.runReferences[outcome];
Expand Down Expand Up @@ -826,14 +829,20 @@ type GithubAttemptFailure = {
type GithubFailureCategory = "invalid-json" | "too-large" | "transport";

class GithubHttpStatusError extends Error {
constructor(readonly status: number) {
readonly status: number;

constructor(status: number) {
super("GitHub HTTP status failure");
this.status = status;
}
}

class GithubCategorizedError extends Error {
constructor(readonly category: GithubFailureCategory) {
readonly category: GithubFailureCategory;

constructor(category: GithubFailureCategory) {
super(`GitHub HTTP ${category} failure`);
this.category = category;
}
}

Expand Down
Loading