feat: add fork choice compliance test for pre-gloas forks

.github/workflows/comptests.yml

@@ -0,0 +1,40 @@
+name: Fork-choice compliance tests
+
+# Don't cancel an in-flight run when the next cron fires
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+on:
+  schedule:
+    - cron: "0 3 * * *" # Runs at 3am UTC everyday
+  workflow_dispatch:
+
+jobs:
+  comptests:
+    name: Fork-choice compliance tests
+    # Don't run scheduled copies of this workflow on forks.
+    if: ${{ github.event_name != 'schedule' || github.repository == 'ChainSafe/lodestar' }}
+    runs-on: warp-ubuntu-2204-x64-4x
+    # Single vitest worker (one test file): ~10 min per fork locally, 6 pre-gloas forks.
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: "./.github/actions/setup-and-build"
+        with:
+          node: 24
+
+      - name: Restore compliance vectors cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: packages/beacon-node/spec-tests-comptests
+          key: comptest-data-${{ hashFiles('spec-tests-version.json') }}
+      - name: Download compliance vectors
+        run: pnpm download-comptests
+
+      - name: Fork-choice compliance tests
+        run: pnpm test:comptest
+        working-directory: packages/beacon-node

AGENTS.md

@@ -106,6 +106,12 @@ pnpm download-spec-tests 2026-04-14                            # latest successf
 pnpm download-spec-tests latest <owner>/consensus-specs        # fork
 pnpm download-spec-tests latest <owner>/consensus-specs <ref>  # fork + branch
 
+# Fork-choice compliance tests (model-generated vectors, standalone flow;
+# runs nightly in CI, not per-PR). Same version pin as the spec tests.
+pnpm download-comptests
+pnpm test:comptest
+SPEC_FILTER_FORK=deneb pnpm test:comptest   # single fork (from packages/beacon-node)
+
 # Run e2e tests (requires docker environment)
 ./scripts/run_e2e_env.sh start
 pnpm test:e2e

CLAUDE.md

@@ -25,6 +25,10 @@ pnpm vitest run --project unit test/unit/path/to/test.test.ts
 pnpm download-spec-tests
 pnpm test:spec
 
+# Fork-choice compliance tests (standalone, nightly CI; download first)
+pnpm download-comptests
+pnpm test:comptest
+
 # Docs lint (markdown)
 pnpm docs:lint
 pnpm docs:lint:fix

configs/vitest.config.spec.ts

@@ -5,6 +5,7 @@ export const specProjectMinimal = defineProject({
   test: {
     name: "spec-minimal",
     include: ["**/test/spec/**/*.test.ts"],
+    exclude: ["**/test/spec/comptest/**"],
     setupFiles: [
       path.join(__dirname, "../scripts/vitest/setupFiles/customMatchers.ts"),
       path.join(__dirname, "../scripts/vitest/setupFiles/dotenv.ts"),
@@ -26,6 +27,7 @@ export const specProjectMainnet = defineProject({
   test: {
     name: "spec-mainnet",
     include: ["**/test/spec/**/*.test.ts"],
+    exclude: ["**/test/spec/comptest/**"],
     setupFiles: [
       path.join(__dirname, "../scripts/vitest/setupFiles/customMatchers.ts"),
       path.join(__dirname, "../scripts/vitest/setupFiles/dotenv.ts"),
@@ -42,3 +44,24 @@ export const specProjectMainnet = defineProject({
     },
   },
 });
+
+// Fork-choice compliance suite (`pnpm test:comptest`) — deliberately its own project so the
+// regular spec projects never pick it up.
+// Fixtures via `pnpm download-comptests`.
+export const specProjectComptest = defineProject({
+  test: {
+    name: "comptest",
+    include: ["**/test/spec/comptest/**/*.test.ts"],
+    setupFiles: [
+      path.join(__dirname, "../scripts/vitest/setupFiles/customMatchers.ts"),
+      path.join(__dirname, "../scripts/vitest/setupFiles/dotenv.ts"),
+      path.join(__dirname, "../scripts/vitest/setupFiles/lodestarPreset.ts"),
+    ],
+    testTimeout: 1000 * 60 * 15,
+    hookTimeout: 1000 * 60 * 15,
+    pool: "forks",
+    env: {
+      LODESTAR_PRESET: "minimal",
+    },
+  },
+});

package.json

@@ -30,6 +30,8 @@
     "test:browsers": "vitest run --project browser",
     "test:e2e": "vitest run --project e2e --project e2e-mainnet",
     "download-spec-tests": "pnpm -r download-spec-tests",
+    "download-comptests": "pnpm -r download-comptests",
+    "test:comptest": "vitest run --project comptest",
     "test:spec": "vitest run --project spec-minimal --project spec-mainnet",
     "benchmark": "pnpm benchmark:files 'packages/*/test/perf/**/*.test.ts'",
     "benchmark:files": "NODE_OPTIONS='--max-old-space-size=8192 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",

packages/beacon-node/package.json

@@ -100,6 +100,8 @@
     "test:sim": "vitest run test/sim/**/*.test.ts",
     "test:sim:blobs": "vitest run test/sim/4844-interop.test.ts",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
+    "download-comptests": "node --loader=ts-node/esm test/spec/downloadComptests.ts",
+    "test:comptest": "vitest run --project comptest",
     "test:spec:bls": "vitest run --project spec-minimal test/spec/bls/",
     "test:spec:general": "vitest run --project spec-minimal test/spec/general/",
     "test:spec:minimal": "vitest run --project spec-minimal test/spec/presets/",

packages/beacon-node/test/spec/comptest/fork_choice_compliance.test.ts

@@ -0,0 +1,23 @@
+import fs from "node:fs";
+import path from "node:path";
+import {ACTIVE_PRESET} from "@lodestar/params";
+import {comptestsSpecTests} from "../specTestVersioning.js";
+import {forkChoiceTestRunner} from "../utils/forkChoiceTestRunner.js";
+import {specTestIterator} from "../utils/specTestIterator.js";
+import {RunnerType} from "../utils/types.js";
+
+// Fork-choice compliance suite (`pnpm test:comptest`) — a standalone flow parallel to the
+// standard spec tests: its own vitest project, its own fixture directory
+// (`pnpm download-comptests`), sharing only the fork-choice test runner.
+
+const presetDir = path.join(comptestsSpecTests.outputDir, "tests", ACTIVE_PRESET);
+const hasComplianceFixtures =
+  fs.existsSync(presetDir) &&
+  fs.readdirSync(presetDir).some((fork) => fs.existsSync(path.join(presetDir, fork, "fork_choice_compliance")));
+if (!hasComplianceFixtures) {
+  throw Error("No fork_choice_compliance fixtures found — run `pnpm download-comptests` first");
+}
+
+specTestIterator(presetDir, {
+  fork_choice_compliance: {type: RunnerType.default, fn: forkChoiceTestRunner({onlyPredefinedResponses: false})},
+});

packages/beacon-node/test/spec/downloadComptests.ts

@@ -0,0 +1,7 @@
+import {downloadTests} from "@lodestar/spec-test-util/downloadTests";
+import {comptestsSpecTests} from "./specTestVersioning.js";
+
+await downloadTests(comptestsSpecTests, console.log).catch((e: Error) => {
+  console.error(e);
+  process.exit(1);
+});