feat(accuracy): run nightly main and release campaigns

.github/codeowners/areas.yaml

@@ -82,6 +82,8 @@ areas:
       - tests/test_ci_qualification.py
       - tests/test_pages_site.py
       - tests/test_e2e_accuracy_overview.py
+      - tests/test_e2e_accuracy_ui.mjs
+      - tests/test_e2e_accuracy_nightly.py
       - tests/test_source_compliance.py
 
 # More-specific shared entries are emitted after base area rules, preserving
@@ -113,6 +115,10 @@ shared:
     owners: [infra, maintainers]
   - glob: tests/test_e2e_accuracy_overview.py
     owners: [forward-pass-engine, infra, maintainers]
+  - glob: tests/test_e2e_accuracy_ui.mjs
+    owners: [forward-pass-engine, infra, maintainers]
+  - glob: tests/test_e2e_accuracy_nightly.py
+    owners: [forward-pass-engine, replay, infra, maintainers]
   - glob: docs/core-api.md
     owners: [forward-pass-engine, maintainers]
   - glob: python/aisimulate/src/aisimulate/__init__.py

.github/e2e-accuracy-dataset.json

@@ -0,0 +1,74 @@
+{
+  "max_age_days": 30,
+  "minimum_free_bytes": 35000000000,
+  "parts": [
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part00",
+      "sha256": "6cd2b2e7e8cd1386d59717a45f8a30af2ff6c21fd975185b5d713cb073720c9a",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part01",
+      "sha256": "778015c8e71b3d657fee394375b7041b4f4d20e1248d4b6a5a16e5d9176d3019",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part02",
+      "sha256": "589526775999cdabe61115688c6d5851ca5a20b5d6429c86b36678c772498a69",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part03",
+      "sha256": "ed627c9536807f64a0b43a1882cbd24bae4812bd1396f84145a9b1adce2817eb",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part04",
+      "sha256": "8a3310aa046aeeb6c4effed2e09c82a639edc0b1ae394d73aed7716e741d8718",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part05",
+      "sha256": "2e5865a091d0429c3644f8910d0d449f84d2951c8f9b9775a134b09f9b0cd258",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part06",
+      "sha256": "07fca5f5709010b08b6a0eb403e13d6531bff147dbaeb3072c37a7eaf6c04fc6",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part07",
+      "sha256": "63003df46a8c675ca403ca90017b3606da990afb327c5fbf0219e76a548592d4",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part08",
+      "sha256": "e1f4feefa3f06f5569509443b593075f6baadeed2d0e5834a4d5bca2c50aae8d",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part09",
+      "sha256": "4917b24bdcb7420322097f2b4f60ce84af91cbfdd8db3bc7f06504c4b1f30976",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part10",
+      "sha256": "a444cff40f4585912cbecd3a9be6e1bd61043836d305390479a8b92eb44cdc73",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part11",
+      "sha256": "90a444a161f699ae9e94abfebd2f54c54a4fe92667d62331b78fb8020059c873",
+      "size": 1992294400
+    },
+    {
+      "name": "inferencex-2026-09-14.dump.zst.part12",
+      "sha256": "5df713decfa3a393063575884ab2948eaaacdc744153e850f752f9631cc894c4",
+      "size": 746769630
+    }
+  ],
+  "release_tag": "db-dump/2026-09-14",
+  "schema_version": 1,
+  "selection_policy": "latest-complete-config-run-v1"
+}

.github/workflows/e2e-accuracy.yml

@@ -0,0 +1,241 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+name: E2E Accuracy Matrix
+
+on:
+  workflow_run:
+    workflows: [Nightly CI]
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      expected_sha:
+        description: Full source commit to evaluate (must belong to the selected branch)
+        required: true
+        type: string
+      branch:
+        description: Evaluated main or release/* branch
+        required: true
+        default: main
+        type: string
+
+permissions:
+  contents: read
+
+# Every completed campaign keeps its own provenance; never cancel a release
+# evaluation merely because a main nightly completes.
+concurrency:
+  group: e2e-accuracy-${{ github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  resolve:
+    name: Resolve exact accuracy source
+    if: github.repository == 'ai-dynamo/aisimulate' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      run: ${{ steps.source.outputs.run }}
+      sha: ${{ steps.source.outputs.sha }}
+      branch: ${{ steps.source.outputs.branch }}
+      nightly-run: ${{ steps.source.outputs.nightly-run }}
+    steps:
+      - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        id: source
+        env:
+          EXPECTED_SHA: ${{ inputs.expected_sha }}
+          EVALUATED_BRANCH: ${{ inputs.branch }}
+        with:
+          script: |
+            let sha = process.env.EXPECTED_SHA;
+            let branch = process.env.EVALUATED_BRANCH;
+            let nightlyRun = '';
+            if (context.eventName === 'workflow_run') {
+              const run = context.payload.workflow_run;
+              if (run.repository.full_name !== context.repo.owner + '/' + context.repo.repo ||
+                  run.head_repository.full_name !== run.repository.full_name ||
+                  run.event !== 'schedule' || run.head_branch !== 'main' ||
+                  run.path !== '.github/workflows/nightly-ci.yml') {
+                throw new Error('Untrusted nightly producer');
+              }
+              const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
+                ...context.repo, run_id: run.id, filter: 'latest', per_page: 100,
+              });
+              if (!jobs.some(job => job.name === 'Build artifacts (amd64)' && job.conclusion === 'success')) {
+                core.notice('Nightly has no completed amd64 wheel; retain the prior accuracy snapshot.');
+                core.setOutput('run', 'false');
+                return;
+              }
+              sha = run.head_sha;
+              branch = 'main';
+              nightlyRun = String(run.id);
+            }
+            if (!/^[0-9a-f]{40}$/.test(sha || '') ||
+                !(branch === 'main' || /^release\/[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(branch || ''))) {
+              throw new Error('Specify a full source SHA and main or release/* branch');
+            }
+            core.setOutput('run', 'true');
+            core.setOutput('sha', sha);
+            core.setOutput('branch', branch);
+            core.setOutput('nightly-run', nightlyRun);
+
+  wheel:
+    name: Prepare exact accuracy wheel
+    needs: resolve
+    if: needs.resolve.outputs.run == 'true'
+    runs-on: prod-aisimulate-default-amd64-v1
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      actions: read
+    container:
+      # Same manylinux builder as the FPE matrix.
+      image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb
+    outputs:
+      sha256: ${{ steps.verify.outputs.sha256 }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ needs.resolve.outputs.sha }}
+          path: source
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Verify source belongs to the evaluated branch
+        env:
+          EXPECTED_SHA: ${{ needs.resolve.outputs.sha }}
+          EVALUATED_BRANCH: ${{ needs.resolve.outputs.branch }}
+        run: |
+          test "$(git -C source rev-parse HEAD)" = "${EXPECTED_SHA}"
+          git -C source merge-base --is-ancestor "${EXPECTED_SHA}" "refs/remotes/origin/${EVALUATED_BRANCH}"
+          test -z "$(git -C source status --porcelain)"
+      - uses: ./.github/actions/setup-manylinux-python-rust
+        if: needs.resolve.outputs.nightly-run == ''
+      - name: Build a manual revision's wheel
+        if: needs.resolve.outputs.nightly-run == ''
+        run: |
+          python -m pip install 'maturin>=1.12,<2'
+          python source/scripts/build_manylinux_wheel.py --output-dir "${RUNNER_TEMP}/accuracy-wheel"
+          test -z "$(git -C source status --porcelain)"
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        if: needs.resolve.outputs.nightly-run != ''
+        with:
+          name: nightly-dist-amd64
+          path: ${{ runner.temp }}/accuracy-wheel
+          run-id: ${{ needs.resolve.outputs.nightly-run }}
+          github-token: ${{ github.token }}
+      - name: Verify wheel checksum and nightly producer revision
+        id: verify
+        env:
+          EXPECTED_SHA: ${{ needs.resolve.outputs.sha }}
+          NIGHTLY_RUN: ${{ needs.resolve.outputs.nightly-run }}
+        run: |
+          /opt/python/cp312-cp312/bin/python - <<'PY'
+          import hashlib, json, os
+          from pathlib import Path
+          directory = Path(os.environ['RUNNER_TEMP']) / 'accuracy-wheel'
+          wheels = list(directory.glob('aisimulate-*.whl'))
+          if len(wheels) != 1:
+              raise SystemExit('expected one unified wheel')
+          wheel = wheels[0]
+          digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
+          if os.environ['NIGHTLY_RUN']:
+              provenance = json.loads((directory / 'provenance.json').read_text())
+              if (provenance['commit'] != os.environ['EXPECTED_SHA'] or
+                  provenance['repository'] != 'ai-dynamo/aisimulate' or
+                  provenance['event'] != 'schedule' or provenance['ref'] != 'refs/heads/main' or
+                  provenance['arch'] != 'amd64' or provenance['artifacts'][wheel.name] != digest):
+                  raise SystemExit('nightly wheel provenance mismatch')
+              checksums = dict(line.split(maxsplit=1)[::-1] for line in (directory / 'SHA256SUMS.txt').read_text().splitlines())
+              for name in (wheel.name, 'provenance.json'):
+                  if hashlib.sha256((directory / name).read_bytes()).hexdigest() != checksums[name]:
+                      raise SystemExit('nightly artifact checksum mismatch')
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
+              stream.write(f'sha256={digest}\n')
+          PY
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: e2e-accuracy-wheel
+          path: ${{ runner.temp }}/accuracy-wheel/aisimulate-*.whl
+          if-no-files-found: error
+          compression-level: 0
+          retention-days: 7
+
+  campaign:
+    name: Run and qualify E2E accuracy
+    needs: [resolve, wheel]
+    runs-on: prod-aisimulate-default-amd64-v1
+    timeout-minutes: 360
+    container:
+      # PostgreSQL 18 client reads the upstream custom dump without a server.
+      image: postgres:18-bookworm@sha256:1c59e2c3c818eaa0f0628f695b36e7c9e362d6b219b36a54a32df645cbd7e1af
+    env:
+      OPENBLAS_NUM_THREADS: '1'
+      OMP_NUM_THREADS: '1'
+      RAYON_NUM_THREADS: '1'
+      TOKENIZERS_PARALLELISM: 'false'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - name: Install the CPU campaign environment
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends python3 python3-venv zstd libgomp1
+          python3 -m venv "${RUNNER_TEMP}/accuracy-venv"
+          echo "${RUNNER_TEMP}/accuracy-venv/bin" >> "${GITHUB_PATH}"
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: e2e-accuracy-wheel
+          path: ${{ runner.temp }}/accuracy-wheel
+      - name: Install the exact wheel shared by both predictors
+        env:
+          WHEEL_SHA256: ${{ needs.wheel.outputs.sha256 }}
+        run: |
+          echo "${WHEEL_SHA256}  $(echo "${RUNNER_TEMP}"/accuracy-wheel/*.whl)" | sha256sum --check --strict
+          python -m pip install "${RUNNER_TEMP}"/accuracy-wheel/*.whl
+          python -m pip check
+      - name: Fetch checksum-pinned public measurements
+        run: |
+          python scripts/fetch_accuracy_measurements.py \
+            --manifest .github/e2e-accuracy-dataset.json --output "${RUNNER_TEMP}/accuracy-measurements"
+      - name: Run both predictors with bounded CPU workers and qualify the complete cohort
+        env:
+          EXPECTED_SHA: ${{ needs.resolve.outputs.sha }}
+          EVALUATED_BRANCH: ${{ needs.resolve.outputs.branch }}
+        run: |
+          python scripts/run_e2e_accuracy.py \
+            --tables "${RUNNER_TEMP}/accuracy-measurements/tables.json" \
+            --manifest .github/e2e-accuracy-dataset.json \
+            --wheel "${RUNNER_TEMP}"/accuracy-wheel/*.whl \
+            --commit "${EXPECTED_SHA}" --branch "${EVALUATED_BRANCH}" \
+            --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \
+            --workers 6 --point-timeout 180 --output "${RUNNER_TEMP}/accuracy-public"
+      - name: Report advisory accuracy and coverage
+        run: |
+          python - <<'PY' >> "${GITHUB_STEP_SUMMARY}"
+          import json, os
+          from pathlib import Path
+          summary = json.loads((Path(os.environ['RUNNER_TEMP']) / 'accuracy-public/summary.json').read_text())
+          print('## E2E accuracy (advisory)\n')
+          print('Evaluated revision:', summary['snapshot']['evaluated_revision'])
+          print('Measurements:', summary['snapshot']['release_tag'])
+          print('\n```json\n' + json.dumps(summary['totals'], indent=2) + '\n```')
+          print('\nCampaign outcomes:', summary['snapshot']['campaign']['outcomes'])
+          print('\nAccuracy values do not gate release staging. Incomplete campaigns cannot publish.')
+          PY
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: e2e-accuracy-web
+          path: |
+            ${{ runner.temp }}/accuracy-public/summary.json
+            ${{ runner.temp }}/accuracy-public/qualification.json
+          if-no-files-found: error
+          retention-days: 90