Add Fork Choice compliance test generator

Makefile

@@ -37,6 +37,7 @@ NORM = $(shell tput sgr0)
 # Print target descriptions.
 help:
 	@echo "make $(BOLD)clean$(NORM)      -- delete all untracked files"
+	@echo "make $(BOLD)comptests$(NORM)  -- generate compliance tests"
 	@echo "make $(BOLD)coverage$(NORM)   -- run pyspec tests with coverage"
 	@echo "make $(BOLD)kzg_setups$(NORM) -- generate trusted setups"
 	@echo "make $(BOLD)lint$(NORM)       -- run the linters"
@@ -242,6 +243,27 @@ kzg_setups: pyspec
 			--output-dir $(CURDIR)/presets/$$preset/trusted_setups; \
 	done
 
+COMP_TEST_VECTOR_DIR = $(CURDIR)/../compliance-spec-tests/tests
+
+# Generate compliance tests (fork choice).
+# This will forcibly rebuild pyspec just in case.
+# To generate compliance tests with a particular configuration, append fc_gen_config=<config>,
+# where <config> can be either tiny, small or standard, eg:
+#   make comptests fc_gen_config=standard
+# One can specify threads=N, fork=<fork> or preset=<preset> as with reftests, eg:
+#   make comptests fc_gen_config=standard fork=deneb preset=mainnet threads=8
+comptests: FC_GEN_CONFIG := $(if $(fc_gen_config),$(fc_gen_config),tiny)
+comptests: MAYBE_THREADS := $(if $(threads),--threads=$(threads),--fc-gen-multi-processing)
+comptests: MAYBE_FORKS := $(if $(fork),--forks $(subst ${COMMA}, ,$(fork)))
+comptests: MAYBE_PRESETS := $(if $(preset),--presets $(subst ${COMMA}, ,$(preset)))
+comptests: pyspec
+	@$(PYTHON_VENV) -m tests.generators.compliance_runners.fork_choice.test_gen \
+		--output $(COMP_TEST_VECTOR_DIR) \
+		--fc-gen-config $(FC_GEN_CONFIG) \
+		$(MAYBE_THREADS) \
+		$(MAYBE_FORKS) \
+		$(MAYBE_PRESETS)
+
 ###############################################################################
 # Cleaning
 ###############################################################################

pyproject.toml

@@ -48,10 +48,12 @@ lint = [
 ]
 generator = [
   "filelock==3.18.0",
+  "minizinc==0.10.0",
   "pathos==0.3.4",
   "pytest==8.4.0",
   "python-snappy==0.7.3",
   "rich==14.0.0",
+  "tqdm==4.67.1",
 ]
 docs = [
   "mdx-truly-sane-lists==1.3",

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -114,10 +114,13 @@ def run_func():
     yield from with_blob_data(spec, blob_data, run_func)
 
 
-def add_attestation(spec, store, attestation, test_steps, is_from_block=False):
-    spec.on_attestation(store, attestation, is_from_block=is_from_block)
+def add_attestation(spec, store, attestation, test_steps, is_from_block=False, valid=True):
+    run_on_attestation(spec, store, attestation, is_from_block=is_from_block, valid=valid)
     yield get_attestation_file_name(attestation), attestation
-    test_steps.append({"attestation": get_attestation_file_name(attestation)})
+    if valid:
+        test_steps.append({"attestation": get_attestation_file_name(attestation)})
+    else:
+        test_steps.append({"attestation": get_attestation_file_name(attestation), "valid": False})
 
 
 def add_attestations(spec, store, attestations, test_steps, is_from_block=False):
@@ -348,24 +351,39 @@ def output_head_check(spec, store, test_steps):
     )
 
 
-def output_store_checks(spec, store, test_steps):
-    test_steps.append(
-        {
-            "checks": {
-                "time": int(store.time),
-                "head": get_formatted_head_output(spec, store),
-                "justified_checkpoint": {
-                    "epoch": int(store.justified_checkpoint.epoch),
-                    "root": encode_hex(store.justified_checkpoint.root),
-                },
-                "finalized_checkpoint": {
-                    "epoch": int(store.finalized_checkpoint.epoch),
-                    "root": encode_hex(store.finalized_checkpoint.root),
-                },
-                "proposer_boost_root": encode_hex(store.proposer_boost_root),
+def output_store_checks(spec, store, test_steps, with_viable_for_head_weights=False):
+    checks = {
+        "time": int(store.time),
+        "head": get_formatted_head_output(spec, store),
+        "justified_checkpoint": {
+            "epoch": int(store.justified_checkpoint.epoch),
+            "root": encode_hex(store.justified_checkpoint.root),
+        },
+        "finalized_checkpoint": {
+            "epoch": int(store.finalized_checkpoint.epoch),
+            "root": encode_hex(store.finalized_checkpoint.root),
+        },
+        "proposer_boost_root": encode_hex(store.proposer_boost_root),
+    }
+
+    if with_viable_for_head_weights:
+        filtered_block_roots = spec.get_filtered_block_tree(store).keys()
+        leaves_viable_for_head = [
+            root
+            for root in filtered_block_roots
+            if not any(c for c in filtered_block_roots if store.blocks[c].parent_root == root)
+        ]
+
+        viable_for_head_roots_and_weights = [
+            {
+                "root": encode_hex(viable_for_head_root),
+                "weight": int(spec.get_weight(store, viable_for_head_root)),
             }
-        }
-    )
+            for viable_for_head_root in leaves_viable_for_head
+        ]
+        checks["viable_for_head_roots_and_weights"] = viable_for_head_roots_and_weights
+
+    test_steps.append({"checks": checks})
 
 
 def apply_next_epoch_with_attestations(

tests/formats/fork_choice/README.md

@@ -173,6 +173,10 @@ finalized_checkpoint: {
     root: string,             -- Encoded 32-byte value from store.finalized_checkpoint.root
 }
 proposer_boost_root: string   -- Encoded 32-byte value from store.proposer_boost_root
+viable_for_head_roots_and_weights: [{
+    root: string,             -- Encoded 32-byte value of filtered_block_tree leaf blocks
+    weight: int               -- Integer value from get_weight(store, viable_block_root)
+}]
 ```
 
 Additionally, these fields if `get_proposer_head` and `should_override_forkchoice_update` features are implemented:
@@ -196,6 +200,10 @@ For example:
     proposer_boost_root: '0xdaa1d49d57594ced0c35688a6da133abb086d191a2ebdfd736fad95299325aeb'
     get_proposer_head: '0xdaa1d49d57594ced0c35688a6da133abb086d191a2ebdfd736fad95299325aeb'
     should_override_forkchoice_update: {validator_is_connected: false, result: false}
+    viable_for_head_roots_and_weights: [
+      {root: '0x533290b6f44d31c925acd08dfc8448624979d48c40b877d4e6714648866c9ddb', weight: 192000000000},
+      {root: '0x5cfb9d9099cdf1d8ab68ce96cdae9f0fa6eef16914a01070580dfdc1d2d59ec3', weight: 544000000000}
+    ]
 ```
 
 *Note*: Each `checks` step may include one or multiple items. Each item has to be checked against the current store.

tests/generators/compliance_runners/fork_choice/README.md

@@ -0,0 +1,53 @@
+# Fork choice compliance test generator
+
+Fork Choice test generator intended to produce tests to validate conformance to the specs of various Fork Choice implementations.
+
+Implementation of the approach described in the [Fork Choice compliance testing framework](https://hackmd.io/@ericsson49/fork-choice-implementation-vs-spec-testing).
+
+Preliminary research has been also performed in this [repo](https://github.com/txrx-research/fork_choice_test_generation/tree/main).
+
+To simplify adoption of the tests, we follow the test format described in the [fork choice test formats documentation](../../../formats/fork_choice/README.md), with a minor exception (new check added).
+
+This work was supported by a grant from the Ethereum Foundation.
+
+# Pre-requisites
+
+Install pyspec using the top-level Makefile, this will install necessary pre-requiesites.
+
+```
+> make pyspec
+```
+
+# Generating tests
+
+From the root directory:
+
+```
+> python -m tests.generators.compliance_runners.fork_choice.test_gen -o ${test_dir} --fc-gen-config ${config}
+```
+where `config` can be either: `tiny`, `small` or `standard.
+
+Or specify path to the configuration file directly:
+
+```
+> python -m tests.generators.compliance_runners.fork_choice.test_gen -o ${test_dir} --fc-gen-config-path ${config_path}
+```
+
+There are three configurations in the repo: [tiny](tiny/), [small](small/) and [standard](standard/).
+
+# Running tests
+
+From the root directory:
+
+```
+> python -m tests.generators.compliance_runners.fork_choice.test_run -i ${test_dir}
+```
+
+# Generating configurations
+
+Files in [tiny](tiny/), [small](small/) and [standard](standard/) are generated with [generate_test_instances.py](generate_test_instances.py), e.g.
+```
+> python -m tests.generators.compliance_runners.fork_choice.generate_test_instances
+```
+
+But one normally doesn't need to generate them.