ci: fix daily E2E accuracy pipeline

.github/workflows/e2e-accuracy-branch.yml

@@ -115,8 +115,11 @@ jobs:
     runs-on: prod-aisimulate-default-amd64-v1
     timeout-minutes: 360
     container:
-      # PostgreSQL 18 client reads the upstream custom dump without a server.
-      image: postgres:18-bookworm@sha256:1c59e2c3c818eaa0f0628f695b36e7c9e362d6b219b36a54a32df645cbd7e1af
+      # The runner executes as non-root; use its sudo-enabled CI image.
+      image: ${{ vars.CI_JOB_CONTAINER_IMAGE }}
+      env:
+        HOME: /__w/aisimulate/aisimulate
+        USER: runner
     env:
       OPENBLAS_NUM_THREADS: '1'
       OMP_NUM_THREADS: '1'
@@ -127,11 +130,21 @@ jobs:
         with:
           ref: ${{ github.sha }}
           persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.12"
       - name: Install the CPU campaign environment
         run: |
-          apt-get update
-          apt-get install -y --no-install-recommends python3 python3-venv zstd libgomp1
-          python3 -m venv "${RUNNER_TEMP}/accuracy-venv"
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends postgresql-common ca-certificates
+          # Enable the official PGDG repository for the required dump reader.
+          sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends postgresql-client-18 zstd libgomp1
+          echo "/usr/lib/postgresql/18/bin" >> "${GITHUB_PATH}"
+          /usr/lib/postgresql/18/bin/pg_restore --version
+          zstd --version
+          python -m venv "${RUNNER_TEMP}/accuracy-venv"
           echo "${RUNNER_TEMP}/accuracy-venv/bin" >> "${GITHUB_PATH}"
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
@@ -144,18 +157,22 @@ jobs:
           echo "${WHEEL_SHA256}  $(echo "${RUNNER_TEMP}"/accuracy-wheel/*.whl)" | sha256sum --check --strict
           python -m pip install "${RUNNER_TEMP}"/accuracy-wheel/*.whl
           python -m pip check
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: e2e-accuracy-dataset
+          path: ${{ runner.temp }}/accuracy-dataset
       - name: Fetch checksum-pinned public measurements
         run: |
           python scripts/fetch_accuracy_measurements.py \
-            --manifest .github/e2e-accuracy-dataset.json --output "${RUNNER_TEMP}/accuracy-measurements"
+            --manifest "${RUNNER_TEMP}/accuracy-dataset/e2e-accuracy-dataset.json" --output "${RUNNER_TEMP}/accuracy-measurements"
       - name: Run both predictors with bounded CPU workers and qualify the complete cohort
         env:
           EXPECTED_SHA: ${{ inputs.sha }}
           EVALUATED_BRANCH: ${{ inputs.branch }}
         run: |
           python scripts/run_e2e_accuracy.py \
             --tables "${RUNNER_TEMP}/accuracy-measurements/tables.json" \
-            --manifest .github/e2e-accuracy-dataset.json \
+            --manifest "${RUNNER_TEMP}/accuracy-dataset/e2e-accuracy-dataset.json" \
             --wheel "${RUNNER_TEMP}"/accuracy-wheel/*.whl \
             --commit "${EXPECTED_SHA}" --branch "${EVALUATED_BRANCH}" \
             --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \
@@ -177,8 +194,7 @@ jobs:
         with:
           name: e2e-accuracy-web-${{ inputs.artifact_key }}
           overwrite: true
-          path: |
-            ${{ runner.temp }}/accuracy-public/summary.json
-            ${{ runner.temp }}/accuracy-public/qualification.json
+          # A single directory lets the container hook remap the entire path.
+          path: ${{ runner.temp }}/accuracy-public/
           if-no-files-found: error
           retention-days: 90

.github/workflows/e2e-accuracy.yml

@@ -104,6 +104,24 @@ jobs:
             }
             core.setOutput('matrix', JSON.stringify({include: entries}));
 
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Resolve latest database dump once for all branches
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          python3 scripts/fetch_accuracy_measurements.py \
+            --manifest .github/e2e-accuracy-dataset.json --resolve-latest \
+            --output "${RUNNER_TEMP}/e2e-accuracy-dataset.json"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: e2e-accuracy-dataset
+          path: ${{ runner.temp }}/e2e-accuracy-dataset.json
+          overwrite: true
+          if-no-files-found: error
+          retention-days: 90
+
   branches:
     name: E2E accuracy (${{ matrix.branch }})
     needs: resolve

docs/ci.md

@@ -510,13 +510,22 @@ completed; inspect its wheel and crate publish jobs.
 
 The independent [E2E Accuracy Matrix](../.github/workflows/e2e-accuracy.yml)
 evaluates the scheduled main SHA and every discovered `release/*` head against
-checksum-pinned public measurements. A branch matrix runs at most two campaigns
+the latest published InferenceX database dump. The resolver freezes one manifest
+with verified part checksums for all branches and retains it as a 90-day artifact.
+A branch matrix runs at most two campaigns
 at once, with separate wheels, artifacts, and provenance. Main reuses a qualified
 amd64 wheel for its exact SHA when available, even while nightly staging waits for
 approval; otherwise it builds one. Each release builds its own wheel. Both the
 AISimulate replay and bundled legacy AIC baseline use their branch's wheel. Manual
 runs evaluate one explicit `main` or `release/*` SHA from the trusted main workflow.
 
+Campaign jobs use the configured `CI_JOB_CONTAINER_IMAGE`, Python 3.12, and
+`sudo` to install the PostgreSQL 18 client from the official PGDG repository,
+plus `zstd` and `libgomp1`. The runner executes container steps as a non-root
+user, so a bare PostgreSQL image cannot install these dependencies with `apt-get`.
+The client reads the dump without starting a database server. Artifact upload
+uses the output directory so the runner container hook remaps the full path.
+
 Complete campaigns upload sanitized `e2e-accuracy-web-<branch-key>` artifacts.
 Pages validates the branch's successful qualification job in the artifact's exact
 run attempt, producer, revision, coverage, and checksums before combining it with

pages/e2e-accuracy/README.md

@@ -127,11 +127,20 @@ fails without replacing its published evidence.
 
 ### Measurement and prediction policy
 
-- `.github/e2e-accuracy-dataset.json` pins the InferenceX release, every compressed
-  dump part's size and SHA-256, and selection policy. Refresh it in a reviewed PR
-  when adopting new measurements. A nightly reruns predictions against this fixed
-  silicon dataset; it does not collect new GPU measurements.
-- The downloader verifies every part, decompresses the public PostgreSQL archive,
+- Each pipeline resolves the newest published `db-dump/YYYY-MM-DD` release once,
+  excluding drafts, prereleases, and unrelated releases. It verifies `SHA256SUMS`
+  against GitHub's asset digest and requires every dump part to match the checksum
+  list. Missing or inconsistent assets fail the run instead of using older data.
+- All branches download the same resolved `e2e-accuracy-dataset` manifest artifact,
+  retained for 90 days. The campaign records its release tag and manifest hash.
+  Rerunning only failed jobs reuses that manifest; rerunning the whole workflow
+  resolves latest again. No new GPU measurements are collected.
+- `.github/e2e-accuracy-dataset.json` remains a pinned local reproduction fixture
+  and supplies the selection policy, maximum measurement age, and minimum disk
+  space. CI replaces its release and parts with the resolved snapshot and requires
+  at least the compressed dump size plus 10 GB of free disk space.
+- The downloader verifies every part and retries a failed part up to three total
+  attempts without redownloading verified parts. It decompresses the public PostgreSQL archive,
   and reads only `configs`, `benchmark_results`, and `workflow_runs` via COPY text.
   It never executes SQL from the dump. The September 14 release downloads about
   25 GB and requires at least 35 GB of free temporary disk. Decompression streams

scripts/fetch_accuracy_measurements.py

@@ -12,10 +12,13 @@
 
 import argparse
 import hashlib
+import http.client
 import json
+import os
 import re
 import shutil
 import subprocess
+import time
 import urllib.request
 from pathlib import Path
 
@@ -54,6 +57,76 @@ def validate_manifest(manifest: dict) -> None:
             raise ValueError("invalid pinned dump part")
 
 
+def release_items(path: str) -> list[dict]:
+    """Read every page of release metadata from the fixed upstream repository."""
+    items = []
+    for page in range(1, 101):
+        headers = {"Accept": "application/vnd.github+json", "User-Agent": "aisimulate-accuracy"}
+        if token := os.environ.get("GITHUB_TOKEN"):
+            headers["Authorization"] = f"Bearer {token}"
+        request = urllib.request.Request(
+            f"https://api.github.com/repos/SemiAnalysisAI/InferenceX-app/{path}?per_page=100&page={page}",
+            headers=headers,
+        )
+        with urllib.request.urlopen(request, timeout=120) as response:
+            batch = json.load(response)
+        if not isinstance(batch, list):
+            raise ValueError("invalid upstream release metadata")
+        items.extend(batch)
+        if len(batch) < 100:
+            return items
+    raise ValueError("upstream release pagination limit exceeded")
+
+
+def resolve_latest_manifest(template: dict) -> dict:
+    """Freeze the newest dated dump and its published checksums for one run."""
+    validate_manifest(template)
+    releases = [
+        release
+        for release in release_items("releases")
+        if not release.get("draft")
+        and not release.get("prerelease")
+        and re.fullmatch(r"db-dump/\d{4}-\d{2}-\d{2}", release.get("tag_name", ""))
+    ]
+    if not releases:
+        raise ValueError("no published database dump release")
+    release = max(releases, key=lambda item: item["tag_name"])
+    tag = release["tag_name"]
+    assets = release_items(f"releases/{int(release['id'])}/assets")
+    by_name = {asset["name"]: asset for asset in assets}
+    if len(by_name) != len(assets):
+        raise ValueError("duplicate release assets")
+    checksum_asset = by_name.get("SHA256SUMS", {})
+    if checksum_asset.get("state") != "uploaded":
+        raise ValueError("latest dump has no uploaded SHA256SUMS")
+    with urllib.request.urlopen(RELEASE_ROOT + tag + "/SHA256SUMS", timeout=120) as response:
+        checksums = response.read(1024 * 1024 + 1)
+    if (
+        len(checksums) > 1024 * 1024
+        or len(checksums) != checksum_asset.get("size")
+        or "sha256:" + hashlib.sha256(checksums).hexdigest() != checksum_asset.get("digest")
+    ):
+        raise ValueError("release checksum file digest or size mismatch")
+    parts = []
+    for line in checksums.decode("utf-8").splitlines():
+        match = re.fullmatch(r"([0-9a-f]{64}) [ *](\S+)", line)
+        if not match:
+            raise ValueError("invalid release checksum entry")
+        checksum, name = match.groups()
+        asset = by_name.get(name, {})
+        if asset.get("state") != "uploaded" or asset.get("digest") != "sha256:" + checksum:
+            raise ValueError("dump asset missing or checksum mismatch")
+        parts.append({"name": name, "sha256": checksum, "size": asset["size"]})
+    parts.sort(key=lambda part: part["name"])
+    if {part["name"] for part in parts} != {name for name in by_name if ".dump.zst.part" in name}:
+        raise ValueError("dump assets and checksum entries differ")
+    manifest = {**template, "release_tag": tag, "parts": parts}
+    validate_manifest(manifest)
+    # Leave room for extracted COPY data and tables.json in addition to the dump.
+    manifest["minimum_free_bytes"] = max(template["minimum_free_bytes"], sum(p["size"] for p in parts) + 10_000_000_000)
+    return manifest
+
+
 def digest(path: Path) -> str:
     with path.open("rb") as stream:
         return hashlib.file_digest(stream, "sha256").hexdigest()
@@ -144,6 +217,38 @@ def read_copy(stream):
     return tables
 
 
+def download_part(url: str, part: dict, target) -> None:
+    """Retry a failed part without retaining corrupt bytes or redownloading prior parts."""
+    offset = target.tell()
+    for attempt in range(1, 4):
+        target.seek(offset)
+        target.truncate()
+        sha = hashlib.sha256()
+        size = 0
+        try:
+            with urllib.request.urlopen(url, timeout=120) as response:
+                while block := response.read(1024 * 1024):
+                    sha.update(block)
+                    size += len(block)
+                    if size > part["size"]:
+                        raise ValueError("dump part exceeds pinned size")
+                    target.write(block)
+            if size != part["size"] or sha.hexdigest() != part["sha256"]:
+                raise ValueError(
+                    f"measurement part checksum or size mismatch: {part['name']}; "
+                    f"received {size}/{part['size']} bytes, "
+                    f"SHA256 {sha.hexdigest()}, expected {part['sha256']}"
+                )
+            return
+        except (OSError, http.client.HTTPException, ValueError) as error:
+            target.seek(offset)
+            target.truncate()
+            if attempt == 3:
+                raise
+            print(f"Retrying {part['name']} after attempt {attempt}/3: {error}", flush=True)
+            time.sleep(5 * attempt)
+
+
 def fetch(manifest: dict, output: Path) -> Path:
     validate_manifest(manifest)
     tag = manifest["release_tag"]
@@ -154,17 +259,7 @@ def fetch(manifest: dict, output: Path) -> Path:
         raise ValueError("insufficient disk for the pinned measurement dump")
     with compressed.open("wb") as target:
         for index, part in enumerate(manifest["parts"]):
-            sha = hashlib.sha256()
-            size = 0
-            with urllib.request.urlopen(RELEASE_ROOT + tag + "/" + part["name"], timeout=120) as response:
-                while block := response.read(1024 * 1024):
-                    sha.update(block)
-                    size += len(block)
-                    if size > part["size"]:
-                        raise ValueError("dump part exceeds pinned size")
-                    target.write(block)
-            if size != part["size"] or sha.hexdigest() != part["sha256"]:
-                raise ValueError("measurement part checksum or size mismatch")
+            download_part(RELEASE_ROOT + tag + "/" + part["name"], part, target)
             print(f"Verified measurement part {index + 1}/{len(manifest['parts'])}", flush=True)
     sql = output / "measurements.copy"
     # Serial pg_restore reads custom-format archives from stdin. Stream past
@@ -205,5 +300,14 @@ def fetch(manifest: dict, output: Path) -> Path:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--manifest", type=Path, required=True)
     parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument(
+        "--resolve-latest", action="store_true", help="Write a resolved manifest without downloading the dump"
+    )
     args = parser.parse_args()
-    fetch(json.loads(args.manifest.read_text()), args.output)
+    manifest = json.loads(args.manifest.read_text())
+    if args.resolve_latest:
+        resolved = resolve_latest_manifest(manifest)
+        args.output.write_text(json.dumps(resolved, sort_keys=True) + "\n")
+        print(f"Resolved {resolved['release_tag']} with {len(resolved['parts'])} verified part checksums")
+    else:
+        fetch(manifest, args.output)