feat(e2e): report same-commit reliability

.github/workflows/e2e-main-retry.yaml

@@ -66,3 +66,57 @@ jobs:
           path: ${{ runner.temp }}/e2e-main-retry-evidence.json
           if-no-files-found: warn
           retention-days: 14
+
+  report-same-commit-reliability:
+    needs: evaluate
+    if: >-
+      ${{
+      always() &&
+      github.run_attempt == 1 &&
+      github.repository == 'NVIDIA/NemoClaw' &&
+      github.event.workflow_run.status == 'completed' &&
+      (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') &&
+      github.event.workflow_run.path == '.github/workflows/e2e.yaml' &&
+      startsWith(github.event.workflow_run.display_title, 'E2E ') &&
+      github.event.workflow_run.head_branch == 'main' &&
+      github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw'
+      }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - name: Checkout trusted reliability reporter
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
+      - name: Build advisory same-commit reliability report
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          RELIABILITY_REPORT_PATH: ${{ runner.temp }}/same-commit-reliability.json
+          SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          node --experimental-strip-types --no-warnings \
+            tools/e2e/same-commit-reliability.mts \
+            | tee "${RUNNER_TEMP}/same-commit-reliability.md"
+          cat "${RUNNER_TEMP}/same-commit-reliability.md" >>"${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload advisory reliability evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: same-commit-reliability-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+          path: |
+            ${{ runner.temp }}/same-commit-reliability.json
+            ${{ runner.temp }}/same-commit-reliability.md
+          if-no-files-found: error
+          retention-days: 14

ci/source-shape-test-budget.json

@@ -121,6 +121,11 @@
       "test": "binds the host-dependency action and helper to their immutable reviewed revision (#6961)",
       "category": "security"
     },
+    {
+      "file": "test/e2e/support/same-commit-reliability.test.ts",
+      "test": "locks the reporter to canonical source and trusted controller identities (#9168)",
+      "category": "security"
+    },
     {
       "file": "test/e2e/support/e2e-expected-state.test.ts",
       "test": "compiles absence probes for every preflight failure contract",

scripts/scorecard/read-artifact-zip.mts

@@ -51,14 +51,117 @@ function isSafeExpectedFile(expectedFile: string): boolean {
     !expectedFile.endsWith("/") &&
     !expectedFile.includes("\\") &&
     !expectedFile.includes("\0") &&
-    segments.every((segment) => segment !== "" && segment !== "." && segment !== "..")
+    segments.every(
+      (segment) => segment !== "" && segment !== "." && segment !== "..",
+    )
   );
 }
 
 /**
- * Reads the exact bytes of one safe relative file from a GitHub artifact ZIP
- * without extracting paths to disk. Duplicate targets, links, encryption,
- * split archives, ZIP64, excess entries, and oversized payloads are rejected.
+ * Lists structurally validated regular-file entries from a GitHub artifact
+ * ZIP without extracting or inflating their contents. Relative names must not
+ * contain empty, dot, parent, backslash, NUL, absolute, or trailing-slash
+ * segments. Links, encryption, split archives, ZIP64, duplicate names,
+ * unsupported compression, inconsistent headers, and excess entries are
+ * rejected before callers select an allowlisted evidence file.
+ */
+export function listValidatedArtifactZipEntries(
+  archive: Buffer,
+  options: { maxEntries?: number },
+): string[] | null {
+  const maxEntries = options.maxEntries ?? 1000;
+  if (maxEntries < 1) return null;
+  const endOffset = findZipEndOfCentralDirectory(archive);
+  if (endOffset < 0) return null;
+  const entriesOnDisk = archive.readUInt16LE(endOffset + 8);
+  const totalEntries = archive.readUInt16LE(endOffset + 10);
+  const centralDirectorySize = archive.readUInt32LE(endOffset + 12);
+  const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16);
+  if (
+    archive.readUInt16LE(endOffset + 4) !== 0 ||
+    archive.readUInt16LE(endOffset + 6) !== 0 ||
+    entriesOnDisk !== totalEntries ||
+    totalEntries < 1 ||
+    totalEntries > maxEntries ||
+    centralDirectoryOffset + centralDirectorySize !== endOffset
+  ) {
+    return null;
+  }
+
+  const names: string[] = [];
+  const seen = new Set<string>();
+  let offset = centralDirectoryOffset;
+  for (let index = 0; index < totalEntries; index += 1) {
+    if (
+      offset + 46 > endOffset ||
+      archive.readUInt32LE(offset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE
+    ) {
+      return null;
+    }
+    const creatorSystem = archive.readUInt8(offset + 5);
+    const flags = archive.readUInt16LE(offset + 8);
+    const compressionMethod = archive.readUInt16LE(offset + 10);
+    const compressedSize = archive.readUInt32LE(offset + 20);
+    const fileNameLength = archive.readUInt16LE(offset + 28);
+    const extraLength = archive.readUInt16LE(offset + 30);
+    const commentLength = archive.readUInt16LE(offset + 32);
+    const diskStart = archive.readUInt16LE(offset + 34);
+    const externalAttributes = archive.readUInt32LE(offset + 38);
+    const localHeaderOffset = archive.readUInt32LE(offset + 42);
+    const entryEnd = offset + 46 + fileNameLength + extraLength + commentLength;
+    if (entryEnd > endOffset) return null;
+    const nameBytes = archive.subarray(
+      offset + 46,
+      offset + 46 + fileNameLength,
+    );
+    let name: string;
+    try {
+      name = new TextDecoder("utf-8", { fatal: true }).decode(nameBytes);
+    } catch {
+      return null;
+    }
+    const unixFileType = (externalAttributes >>> 16) & 0xf000;
+    if (
+      !isSafeExpectedFile(name) ||
+      seen.has(name) ||
+      diskStart !== 0 ||
+      (flags & 0x1) !== 0 ||
+      (compressionMethod !== 0 && compressionMethod !== 8) ||
+      (creatorSystem !== 0 && creatorSystem !== 3) ||
+      (creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) ||
+      localHeaderOffset + 30 > centralDirectoryOffset ||
+      archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE
+    ) {
+      return null;
+    }
+    const localFlags = archive.readUInt16LE(localHeaderOffset + 6);
+    const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8);
+    const localNameLength = archive.readUInt16LE(localHeaderOffset + 26);
+    const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28);
+    const localNameEnd = localHeaderOffset + 30 + localNameLength;
+    const dataEnd = localNameEnd + localExtraLength + compressedSize;
+    if (
+      localNameEnd > centralDirectoryOffset ||
+      dataEnd > centralDirectoryOffset ||
+      localFlags !== flags ||
+      localCompressionMethod !== compressionMethod ||
+      !archive.subarray(localHeaderOffset + 30, localNameEnd).equals(nameBytes)
+    ) {
+      return null;
+    }
+    seen.add(name);
+    names.push(name);
+    offset = entryEnd;
+  }
+  return offset === endOffset ? names.sort() : null;
+}
+
+/**
+ * Reads the exact bytes of one structurally validated relative regular-file
+ * entry from a GitHub artifact ZIP without extracting paths to disk. The same
+ * name, archive-layout, file-type, encryption, split/ZIP64, compression,
+ * header-consistency, entry-count, compressed-size, inflated-size, and CRC
+ * constraints used by the entry validator are enforced before bytes return.
  */
 export function readValidatedArtifactZipEntryBytes(
   archive: Buffer,
@@ -67,7 +170,11 @@ export function readValidatedArtifactZipEntryBytes(
 ): Buffer | null {
   const maxEntries = options.maxEntries ?? 1000;
   const expectedFileName = Buffer.from(expectedFile, "utf8");
-  if (!isSafeExpectedFile(expectedFile) || options.maxBytes < 1 || maxEntries < 1) {
+  if (
+    !isSafeExpectedFile(expectedFile) ||
+    options.maxBytes < 1 ||
+    maxEntries < 1
+  ) {
     return null;
   }
 
@@ -96,14 +203,16 @@ export function readValidatedArtifactZipEntryBytes(
   for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) {
     if (
       centralEntryOffset + 46 > endOffset ||
-      archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE
+      archive.readUInt32LE(centralEntryOffset) !==
+        ZIP_CENTRAL_DIRECTORY_SIGNATURE
     ) {
       return null;
     }
     const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28);
     const extraLength = archive.readUInt16LE(centralEntryOffset + 30);
     const commentLength = archive.readUInt16LE(centralEntryOffset + 32);
-    const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength;
+    const centralEntryEnd =
+      centralEntryOffset + 46 + fileNameLength + extraLength + commentLength;
     if (centralEntryEnd > endOffset) return null;
     const fileName = archive.subarray(
       centralEntryOffset + 46,
@@ -164,23 +273,31 @@ export function readValidatedArtifactZipEntryBytes(
     localFileNameEnd > centralDirectoryOffset ||
     localFlags !== flags ||
     localCompressionMethod !== compressionMethod ||
-    !archive.subarray(localHeaderOffset + 30, localFileNameEnd).equals(expectedFileName) ||
+    !archive
+      .subarray(localHeaderOffset + 30, localFileNameEnd)
+      .equals(expectedFileName) ||
     compressedDataEnd > centralDirectoryOffset
   ) {
     return null;
   }
 
-  const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd);
+  const compressedData = archive.subarray(
+    compressedDataOffset,
+    compressedDataEnd,
+  );
   let contents: Buffer;
   try {
     contents =
       compressionMethod === 0
         ? Buffer.from(compressedData)
-        : zlib.inflateRawSync(compressedData, { maxOutputLength: options.maxBytes });
+        : zlib.inflateRawSync(compressedData, {
+            maxOutputLength: options.maxBytes,
+          });
   } catch {
     return null;
   }
-  if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc) return null;
+  if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc)
+    return null;
   return contents;
 }
 
@@ -190,5 +307,11 @@ export function readValidatedArtifactZipEntry(
   expectedFile: string,
   options: { maxBytes: number; maxEntries?: number },
 ): string | null {
-  return readValidatedArtifactZipEntryBytes(archive, expectedFile, options)?.toString("utf8") ?? null;
+  return (
+    readValidatedArtifactZipEntryBytes(
+      archive,
+      expectedFile,
+      options,
+    )?.toString("utf8") ?? null
+  );
 }

test/e2e/README.md

@@ -581,6 +581,29 @@ graph as the live targets:
 
 - GitHub Actions run history is the authoritative record for push and
   manual E2E results.
+- `E2E / Main Retry` publishes an advisory same-commit reliability table for
+  trusted-main and explicit manual qualification runs. It keeps first-pass
+  success, pass-after-retry, exhausted retries, and pass/fail flips distinct,
+  and never treats retry records or runner-pressure classifications as proof
+  that a manual run reached a terminal result. Manual identity comes from the
+  run-bound dispatch receipt; its terminal result additionally requires at
+  least one canonical `nemoclaw-e2e-evidence-v1` manifest bound to the same run,
+  attempt, candidate SHA, trusted workflow repository, workflow SHA, and job
+  status. Trusted-main outcomes instead require the canonical retry-controller
+  artifact. Missing or malformed identity/outcome evidence leaves a run
+  unclassified. Retry and runner-pressure files contribute only fixed failure
+  classes: their complete, missing, or malformed state is reported separately,
+  so malformed classification data cannot erase an otherwise authenticated
+  outcome. Both evidence-state counts appear in the grouped JSON and Markdown
+  table. The table never changes a required check, release conclusion, or rerun
+  decision.
+- The report job appends the Markdown table to the GitHub Actions job summary
+  and uploads the bounded, allowlisted JSON and Markdown files as
+  `same-commit-reliability-<source-run-id>-<attempt>` with 14-day retention.
+  Artifact ZIP entries are structurally validated before an allowlisted file is
+  read: ambiguous relative paths, links, encryption, split/ZIP64 archives,
+  duplicate names, unsupported compression, inconsistent headers, excess
+  entries, oversized contents, and CRC mismatches are rejected.
 - Automated issue routing and the workflow's `issues: write` capability are
   retired. Any future issue escalation should use a separately reviewed
   exceptional threshold, such as the same lane failing twice consecutively or