From e8f6f3d082e94ad20ccf146bc468018aafcaebf2 Mon Sep 17 00:00:00 2001
From: Leo Lara <leo@leolara.me>
Date: Wed, 27 Aug 2025 05:44:12 +0000
Subject: [PATCH 01/11] Improvements to the tests

---
 Makefile                          |  19 +-
 tests/infra/test_template_test.py | 291 ++++++++++++++++--------------
 2 files changed, 171 insertions(+), 139 deletions(-)

diff --git a/Makefile b/Makefile
index b53e7e89b6..8d2cac44b5 100644
--- a/Makefile
+++ b/Makefile
@@ -114,9 +114,26 @@ test: pyspec
 		$(PRESET) \
 		$(BLS) \
 		--junitxml=$(TEST_REPORT_DIR)/test_results.xml \
-		$(CURDIR)/tests/infra \
 		$(PYSPEC_DIR)/eth2spec
 
+# Run test framework tests.
+#
+# To run a specific test, append k=<test>, eg:
+#   make test k=test_verify_kzg_proof
+# To run tests with a specific bls library, append bls=<bls>, eg:
+#   make test bls=arkworks
+test_infra: MAYBE_TEST := $(if $(k),-k=$(k))
+# Disable parallelism which running a specific test.
+# Parallelism makes debugging difficult (print doesn't work).
+test_infra: MAYBE_PARALLEL := $(if $(k),,-n auto)
+test_infra: pyspec
+	@mkdir -p $(TEST_REPORT_DIR)
+	@$(PYTHON_VENV) -m pytest \
+		$(MAYBE_PARALLEL) \
+		--capture=no \
+		$(MAYBE_TEST) \
+		$(CURDIR)/tests/infra
+
 ###############################################################################
 # Coverage
 ###############################################################################
diff --git a/tests/infra/test_template_test.py b/tests/infra/test_template_test.py
index c1a547c0a0..e04da37fe1 100644
--- a/tests/infra/test_template_test.py
+++ b/tests/infra/test_template_test.py
@@ -1,9 +1,9 @@
 import inspect
 import sys
 import types
-import unittest
 from collections.abc import Callable
-from unittest import TestCase
+
+import pytest
 
 from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB, PHASE0
 from eth2spec.test.helpers.typing import SpecForkName
@@ -185,39 +185,37 @@ def wrapper(*args, **kwargs):
 ### @template_test decorators family tests
 
 
-class TestTemplateTestDecorator(TestCase):
-    """Test suite for the template_test decorator."""
+@pytest.fixture
+def test_module():
+    """Create a mock module for testing."""
+    module = types.ModuleType("test_module")
+    sys.modules["test_module"] = module
+    yield module
+    # Cleanup
+    if "test_module" in sys.modules:
+        del sys.modules["test_module"]
 
-    def setUp(self):
-        """Set up test fixtures."""
-        # Create a mock module for testing
-        self.test_module = types.ModuleType("test_module")
-        sys.modules["test_module"] = self.test_module
 
-        # Store original functions for restoration
-        self.original_currentframe = inspect.currentframe
-        self.original_getmodule = inspect.getmodule
+@pytest.fixture
+def mock_utility(test_module):
+    """Create mock framework utility."""
+    utility = MockFrameworkUtility(test_module)
+    yield utility
+    # Cleanup
+    utility.restore()
 
-        # Create mock framework utility
-        self.mock_utility = MockFrameworkUtility(self.test_module)
 
-    def tearDown(self):
-        """Clean up after tests."""
-        # Remove test module from sys.modules
-        if "test_module" in sys.modules:
-            del sys.modules["test_module"]
-
-        # Restore original functions
-        self.mock_utility.restore()
+class TestTemplateTestDecorator:
+    """Test suite for the template_test decorator."""
 
-    def _mock_currentframe(self, module_name="test_module"):
+    def _mock_currentframe(self, mock_utility, module_name="test_module"):
         """Create a mock frame that simulates being called from a specific module."""
-        return self.mock_utility.create_mock_currentframe(module_name)
+        return mock_utility.create_mock_currentframe(module_name)
 
-    def test_basic_functionality(self):
+    def test_basic_functionality(self, test_module, mock_utility):
         """Test basic template_test decorator functionality."""
         # Mock the frame to simulate being called from test_module
-        inspect.currentframe = self._mock_currentframe("test_module")
+        inspect.currentframe = self._mock_currentframe(mock_utility, "test_module")
 
         @template_test
         def _test_template(param1, param2):
@@ -228,18 +226,18 @@ def test_func():
 
         # Call the template function - it returns None
         result = _test_template("foo", "bar")
-        self.assertIsNone(result)
+        assert result is None
 
         # Verify the test was registered in the module
         test_name = "test_generated_foo_bar"
-        self.assertTrue(hasattr(self.test_module, test_name))
-        registered_func = getattr(self.test_module, test_name)
-        self.assertTrue(callable(registered_func))
-        self.assertEqual(registered_func(), "test_foo_bar")
+        assert hasattr(test_module, test_name)
+        registered_func = getattr(test_module, test_name)
+        assert callable(registered_func)
+        assert registered_func() == "test_foo_bar"
 
-    def test_template_with_kwargs(self):
+    def test_template_with_kwargs(self, test_module, mock_utility):
         """Test template function with keyword arguments."""
-        inspect.currentframe = self._mock_currentframe("test_module")
+        inspect.currentframe = self._mock_currentframe(mock_utility, "test_module")
 
         @template_test
         def _test_template(param1, param2=None, param3="default"):
@@ -251,9 +249,9 @@ def test_func():
         _test_template("pos", param2="kw", param3="arg")
 
         test_name = "test_kwargs_pos_kw_arg"
-        self.assertTrue(hasattr(self.test_module, test_name))
-        test_func = getattr(self.test_module, test_name)
-        self.assertEqual(test_func(), "test_pos_kw_arg")
+        assert hasattr(test_module, test_name)
+        test_func = getattr(test_module, test_name)
+        assert test_func() == "test_pos_kw_arg"
 
     def test_frame_inspection_error_no_current_frame(self):
         """Test error handling when currentframe returns None."""
@@ -268,21 +266,19 @@ def test_func():
             return test_func, "test_name"
 
         # The error should happen when calling the decorated function
-        with self.assertRaises(RuntimeError) as context:
+        with pytest.raises(RuntimeError) as excinfo:
             _test_template()
 
-        self.assertEqual(
-            str(context.exception), "Could not determine target module for test registration"
-        )
+        assert str(excinfo.value) == "Could not determine target module for test registration"
 
-    def test_instantiate_module_parameter(self):
+    def test_instantiate_module_parameter(self, test_module, mock_utility):
         """Test that the _instantiate_module parameter allows registering tests in a specific module."""
         # Create a separate module for instantiation
         target_module = types.ModuleType("target_module")
         sys.modules["target_module"] = target_module
 
         try:
-            inspect.currentframe = self._mock_currentframe("test_module")
+            inspect.currentframe = self._mock_currentframe(mock_utility, "test_module")
 
             @template_test
             def _test_template(param):
@@ -294,47 +290,57 @@ def test_func():
             # Call without _instantiate_module - should register in caller's module
             _test_template("default")
             test_name1 = "test_instantiate_default"
-            self.assertTrue(hasattr(self.test_module, test_name1))
-            self.assertFalse(hasattr(target_module, test_name1))
+            assert hasattr(test_module, test_name1)
+            assert not hasattr(target_module, test_name1)
 
             # Call with _instantiate_module - should register in target module
             _test_template("target", _instantiate_module=target_module)
             test_name2 = "test_instantiate_target"
-            self.assertFalse(hasattr(self.test_module, test_name2))
-            self.assertTrue(hasattr(target_module, test_name2))
+            assert not hasattr(test_module, test_name2)
+            assert hasattr(target_module, test_name2)
 
             # Verify they are registered and callable from their modules
-            self.assertEqual(getattr(self.test_module, test_name1)(), "test_default")
-            self.assertEqual(getattr(target_module, test_name2)(), "test_target")
+            assert getattr(test_module, test_name1)() == "test_default"
+            assert getattr(target_module, test_name2)() == "test_target"
 
         finally:
             del sys.modules["target_module"]
 
 
-class TestTemplateTestIntegration(TestCase):
-    """Integration tests for template_test decorator with existing test infrastructure."""
+@pytest.fixture
+def integration_test_module():
+    """Create a mock module for integration testing."""
+    module = types.ModuleType("integration_test_module")
+    sys.modules["integration_test_module"] = module
+    yield module
+    # Cleanup
+    if "integration_test_module" in sys.modules:
+        del sys.modules["integration_test_module"]
 
-    def setUp(self):
-        """Set up test fixtures for integration tests."""
-        self.test_module = types.ModuleType("integration_test_module")
-        sys.modules["integration_test_module"] = self.test_module
-        self.original_currentframe = inspect.currentframe
-        self.original_getmodule = inspect.getmodule
-        self.mock_utility = MockFrameworkUtility(self.test_module)
 
-    def tearDown(self):
-        """Clean up after integration tests."""
-        if "integration_test_module" in sys.modules:
-            del sys.modules["integration_test_module"]
-        self.mock_utility.restore()
+@pytest.fixture
+def integration_mock_utility(integration_test_module):
+    """Create mock framework utility for integration tests."""
+    utility = MockFrameworkUtility(integration_test_module)
+    yield utility
+    # Cleanup
+    utility.restore()
+
+
+class TestTemplateTestIntegration:
+    """Integration tests for template_test decorator with existing test infrastructure."""
 
-    def _mock_currentframe(self, module_name="integration_test_module"):
+    def _mock_currentframe(self, integration_mock_utility, module_name="integration_test_module"):
         """Create a mock frame for integration tests."""
-        return self.mock_utility.create_mock_currentframe(module_name)
+        return integration_mock_utility.create_mock_currentframe(module_name)
 
-    def test_integration_with_mock_spec_decorators(self):
+    def test_integration_with_mock_spec_decorators(
+        self, integration_test_module, integration_mock_utility
+    ):
         """Test template_test integration with mock spec-style decorators."""
-        inspect.currentframe = self._mock_currentframe("integration_test_module")
+        inspect.currentframe = self._mock_currentframe(
+            integration_mock_utility, "integration_test_module"
+        )
 
         # Mock some spec-style decorators
         mock_spec_test = MockDecoratorUtility.create_mock_spec_test()
@@ -354,16 +360,20 @@ def test_fork_transition(spec, phases):
 
         # Verify the integration works
         test_name = "test_fork_transition_phase0_to_altair"
-        self.assertTrue(hasattr(self.test_module, test_name))
+        assert hasattr(integration_test_module, test_name)
 
         # Test the decorated function
-        test_func = getattr(self.test_module, test_name)
+        test_func = getattr(integration_test_module, test_name)
         result = test_func(spec="mock_spec", phases=["phase0", "altair"])
-        self.assertEqual(result, "transition_phase0_to_altair")
+        assert result == "transition_phase0_to_altair"
 
-    def test_template_with_generator_functions(self):
+    def test_template_with_generator_functions(
+        self, integration_test_module, integration_mock_utility
+    ):
         """Test template_test with generator functions (common in spec tests)."""
-        inspect.currentframe = self._mock_currentframe("integration_test_module")
+        inspect.currentframe = self._mock_currentframe(
+            integration_mock_utility, "integration_test_module"
+        )
 
         @template_test
         def _test_template(test_case):
@@ -380,29 +390,36 @@ def test_generator(spec, state):
 
         # Verify registration
         test_name = "test_generator_block_processing"
-        self.assertTrue(hasattr(self.test_module, test_name))
+        assert hasattr(integration_test_module, test_name)
 
         # Test the generator function
-        test_func = getattr(self.test_module, test_name)
+        test_func = getattr(integration_test_module, test_name)
         gen = test_func(spec="mock_spec", state="mock_state")
         pre_result = next(gen)
         post_result = next(gen)
 
-        self.assertEqual(pre_result[0], "pre")
-        self.assertEqual(pre_result[1]["case"], "block_processing")
-        self.assertEqual(post_result[0], "post")
-        self.assertEqual(post_result[1]["result"], "processed_block_processing")
+        assert pre_result[0] == "pre"
+        assert pre_result[1]["case"] == "block_processing"
+        assert post_result[0] == "post"
+        assert post_result[1]["result"] == "processed_block_processing"
 
     def _mock_currentframe_with_function_call(
-        self, module_name="integration_test_module", function_name="test_function"
+        self,
+        integration_mock_utility,
+        module_name="integration_test_module",
+        function_name="test_function",
     ):
         """Create a mock frame that simulates being called from within a function."""
-        return self.mock_utility.create_mock_currentframe_with_function(module_name, function_name)
+        return integration_mock_utility.create_mock_currentframe_with_function(
+            module_name, function_name
+        )
 
-    def test_template_called_from_within_function(self):
+    def test_template_called_from_within_function(
+        self, integration_test_module, integration_mock_utility
+    ):
         """Test template_test decorator when called from within a function (non-module-level)."""
         inspect.currentframe = self._mock_currentframe_with_function_call(
-            "integration_test_module", "function_that_calls_template"
+            integration_mock_utility, "integration_test_module", "function_that_calls_template"
         )
 
         @template_test
@@ -421,43 +438,43 @@ def function_that_calls_template():
 
         # The template should still register in the module, not the function's local scope
         test_name = "test_function_scoped_example"
-        self.assertTrue(hasattr(self.test_module, test_name))
+        assert hasattr(integration_test_module, test_name)
 
         # The registered function should work correctly
-        registered_func = getattr(self.test_module, test_name)
-        self.assertEqual(registered_func(), "function_scoped_test_example")
+        registered_func = getattr(integration_test_module, test_name)
+        assert registered_func() == "function_scoped_test_example"
 
 
-class TestTemplateUpgradeDecorators(TestCase):
-    """Test suite for template_test_upgrades_from and template_test_upgrades_from_to decorators."""
+@pytest.fixture
+def upgrade_test_module():
+    """Create a mock module for upgrade testing."""
+    module = types.ModuleType("upgrade_test_module")
+    sys.modules["upgrade_test_module"] = module
+    yield module
+    # Cleanup
+    if "upgrade_test_module" in sys.modules:
+        del sys.modules["upgrade_test_module"]
 
-    def setUp(self):
-        """Set up test fixtures."""
-        # Create a mock module for testing
-        self.test_module = types.ModuleType("upgrade_test_module")
-        sys.modules["upgrade_test_module"] = self.test_module
 
-        # Store original functions for restoration
-        self.original_currentframe = inspect.currentframe
-        self.original_getmodule = inspect.getmodule
-        self.mock_utility = MockFrameworkUtility(self.test_module)
+@pytest.fixture
+def upgrade_mock_utility(upgrade_test_module):
+    """Create mock framework utility for upgrade tests."""
+    utility = MockFrameworkUtility(upgrade_test_module)
+    yield utility
+    # Cleanup
+    utility.restore()
 
-    def tearDown(self):
-        """Clean up after tests."""
-        # Remove test module from sys.modules
-        if "upgrade_test_module" in sys.modules:
-            del sys.modules["upgrade_test_module"]
 
-        # Restore original functions
-        self.mock_utility.restore()
+class TestTemplateUpgradeDecorators:
+    """Test suite for template_test_upgrades_from and template_test_upgrades_from_to decorators."""
 
-    def _mock_currentframe(self, module_name="upgrade_test_module"):
+    def _mock_currentframe(self, upgrade_mock_utility, module_name="upgrade_test_module"):
         """Create a mock frame that simulates being called from a specific module."""
-        return self.mock_utility.create_mock_currentframe(module_name)
+        return upgrade_mock_utility.create_mock_currentframe(module_name)
 
-    def test_template_test_upgrades_from_basic(self):
+    def test_template_test_upgrades_from_basic(self, upgrade_test_module, upgrade_mock_utility):
         """Test basic functionality of template_test_upgrades_from decorator."""
-        inspect.currentframe = self._mock_currentframe("upgrade_test_module")
+        inspect.currentframe = self._mock_currentframe(upgrade_mock_utility, "upgrade_test_module")
 
         @template_test_upgrades_from(CAPELLA)
         def _template_upgrade(pre_spec: SpecForkName, post_spec: SpecForkName):
@@ -471,27 +488,27 @@ def test_func():
 
         # The decorator should have registered tests for CAPELLA->DENEB, DENEB->ELECTRA, ELECTRA->FULU
         # Check that tests were registered for the expected upgrades
-        self.assertTrue(hasattr(self.test_module, "test_upgrade_capella_to_deneb"))
-        self.assertTrue(hasattr(self.test_module, "test_upgrade_deneb_to_electra"))
-        self.assertTrue(hasattr(self.test_module, "test_upgrade_electra_to_fulu"))
+        assert hasattr(upgrade_test_module, "test_upgrade_capella_to_deneb")
+        assert hasattr(upgrade_test_module, "test_upgrade_deneb_to_electra")
+        assert hasattr(upgrade_test_module, "test_upgrade_electra_to_fulu")
 
         # Should NOT have registered for earlier forks
-        self.assertFalse(hasattr(self.test_module, "test_upgrade_phase0_to_altair"))
-        self.assertFalse(hasattr(self.test_module, "test_upgrade_altair_to_bellatrix"))
+        assert not hasattr(upgrade_test_module, "test_upgrade_phase0_to_altair")
+        assert not hasattr(upgrade_test_module, "test_upgrade_altair_to_bellatrix")
 
         # Execute the registered tests to verify they work
-        test_capella_deneb = getattr(self.test_module, "test_upgrade_capella_to_deneb")
-        self.assertEqual(test_capella_deneb(), "upgrade_capella_to_deneb")
+        test_capella_deneb = getattr(upgrade_test_module, "test_upgrade_capella_to_deneb")
+        assert test_capella_deneb() == "upgrade_capella_to_deneb"
 
-        test_deneb_electra = getattr(self.test_module, "test_upgrade_deneb_to_electra")
-        self.assertEqual(test_deneb_electra(), "upgrade_deneb_to_electra")
+        test_deneb_electra = getattr(upgrade_test_module, "test_upgrade_deneb_to_electra")
+        assert test_deneb_electra() == "upgrade_deneb_to_electra"
 
-        test_electra_fulu = getattr(self.test_module, "test_upgrade_electra_to_fulu")
-        self.assertEqual(test_electra_fulu(), "upgrade_electra_to_fulu")
+        test_electra_fulu = getattr(upgrade_test_module, "test_upgrade_electra_to_fulu")
+        assert test_electra_fulu() == "upgrade_electra_to_fulu"
 
-    def test_template_test_upgrades_from_to_basic(self):
+    def test_template_test_upgrades_from_to_basic(self, upgrade_test_module, upgrade_mock_utility):
         """Test basic functionality of template_test_upgrades_from_to decorator."""
-        inspect.currentframe = self._mock_currentframe("upgrade_test_module")
+        inspect.currentframe = self._mock_currentframe(upgrade_mock_utility, "upgrade_test_module")
 
         @template_test_upgrades_from_to(PHASE0, BELLATRIX)
         def _template_range_upgrade(pre_spec: SpecForkName, post_spec: SpecForkName):
@@ -504,26 +521,28 @@ def test_func():
         _template_range_upgrade()
 
         # Should have registered tests for PHASE0->ALTAIR, ALTAIR->BELLATRIX, and BELLATRIX->CAPELLA
-        self.assertTrue(hasattr(self.test_module, "test_range_phase0_to_altair"))
-        self.assertTrue(hasattr(self.test_module, "test_range_altair_to_bellatrix"))
-        self.assertTrue(hasattr(self.test_module, "test_range_bellatrix_to_capella"))
+        assert hasattr(upgrade_test_module, "test_range_phase0_to_altair")
+        assert hasattr(upgrade_test_module, "test_range_altair_to_bellatrix")
+        assert hasattr(upgrade_test_module, "test_range_bellatrix_to_capella")
 
         # Should NOT have registered beyond CAPELLA
-        self.assertFalse(hasattr(self.test_module, "test_range_capella_to_deneb"))
+        assert not hasattr(upgrade_test_module, "test_range_capella_to_deneb")
 
         # Execute the registered tests
-        test_phase0_altair = getattr(self.test_module, "test_range_phase0_to_altair")
-        self.assertEqual(test_phase0_altair(), "range_phase0_to_altair")
+        test_phase0_altair = getattr(upgrade_test_module, "test_range_phase0_to_altair")
+        assert test_phase0_altair() == "range_phase0_to_altair"
 
-        test_altair_bellatrix = getattr(self.test_module, "test_range_altair_to_bellatrix")
-        self.assertEqual(test_altair_bellatrix(), "range_altair_to_bellatrix")
+        test_altair_bellatrix = getattr(upgrade_test_module, "test_range_altair_to_bellatrix")
+        assert test_altair_bellatrix() == "range_altair_to_bellatrix"
 
-        test_bellatrix_capella = getattr(self.test_module, "test_range_bellatrix_to_capella")
-        self.assertEqual(test_bellatrix_capella(), "range_bellatrix_to_capella")
+        test_bellatrix_capella = getattr(upgrade_test_module, "test_range_bellatrix_to_capella")
+        assert test_bellatrix_capella() == "range_bellatrix_to_capella"
 
-    def test_template_test_upgrades_from_to_single_upgrade(self):
+    def test_template_test_upgrades_from_to_single_upgrade(
+        self, upgrade_test_module, upgrade_mock_utility
+    ):
         """Test template_test_upgrades_from_to with a single upgrade."""
-        inspect.currentframe = self._mock_currentframe("upgrade_test_module")
+        inspect.currentframe = self._mock_currentframe(upgrade_mock_utility, "upgrade_test_module")
 
         @template_test_upgrades_from_to(DENEB, DENEB)
         def _template_single(pre_spec: SpecForkName, post_spec: SpecForkName):
@@ -536,12 +555,8 @@ def test_func():
         _template_single()
 
         # Should only register the upgrade from DENEB to ELECTRA
-        self.assertTrue(hasattr(self.test_module, "test_single_deneb_to_electra"))
+        assert hasattr(upgrade_test_module, "test_single_deneb_to_electra")
 
         # Should NOT register others
-        self.assertFalse(hasattr(self.test_module, "test_single_capella_to_deneb"))
-        self.assertFalse(hasattr(self.test_module, "test_single_electra_to_fulu"))
-
-
-if __name__ == "__main__":
-    unittest.main()
+        assert not hasattr(upgrade_test_module, "test_single_capella_to_deneb")
+        assert not hasattr(upgrade_test_module, "test_single_electra_to_fulu")

From 3321ef26c87427666ce9b1f26746437addfe400c Mon Sep 17 00:00:00 2001
From: Leo Lara <leolara@users.noreply.github.com>
Date: Thu, 28 Aug 2025 23:48:47 +0700
Subject: [PATCH 02/11] Update Makefile

Co-authored-by: danceratopz <danceratopz@gmail.com>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 8d2cac44b5..fa3d648fb9 100644
--- a/Makefile
+++ b/Makefile
@@ -122,7 +122,7 @@ test: pyspec
 #   make test k=test_verify_kzg_proof
 # To run tests with a specific bls library, append bls=<bls>, eg:
 #   make test bls=arkworks
-test_infra: MAYBE_TEST := $(if $(k),-k=$(k))
+fw_test: MAYBE_TEST := $(if $(k),-k=$(k))
 # Disable parallelism which running a specific test.
 # Parallelism makes debugging difficult (print doesn't work).
 test_infra: MAYBE_PARALLEL := $(if $(k),,-n auto)

From 5dcd5a9b4bbef8dbf3bad2432edf2ab0a483a692 Mon Sep 17 00:00:00 2001
From: Leo Lara <leolara@users.noreply.github.com>
Date: Thu, 28 Aug 2025 23:50:00 +0700
Subject: [PATCH 03/11] Apply suggestion from @danceratopz

Co-authored-by: danceratopz <danceratopz@gmail.com>
---
 tests/infra/test_template_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/infra/test_template_test.py b/tests/infra/test_template_test.py
index e04da37fe1..beb1af5954 100644
--- a/tests/infra/test_template_test.py
+++ b/tests/infra/test_template_test.py
@@ -246,7 +246,7 @@ def test_func():
 
             return test_func, f"test_kwargs_{param1}_{param2}_{param3}"
 
-        _test_template("pos", param2="kw", param3="arg")
+        _test_template("pos1", param2="kwarg1", param3="kwarg2")
 
         test_name = "test_kwargs_pos_kw_arg"
         assert hasattr(test_module, test_name)

From ec3edc031e8a2273dd73dc2a21f580d12f9016cc Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Thu, 28 Aug 2025 14:47:22 -0500
Subject: [PATCH 04/11] Add fw=true option to make test

---
 Makefile | 28 +++++++---------------------
 1 file changed, 7 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index fa3d648fb9..a573948537 100644
--- a/Makefile
+++ b/Makefile
@@ -97,13 +97,16 @@ TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 #   make test preset=mainnet fork=deneb k=test_verify_kzg_proof
 # To run tests with a specific bls library, append bls=<bls>, eg:
 #   make test bls=arkworks
+# To only run the framework tests, append fw=true, eg:
+#   make test fw=true
 test: MAYBE_TEST := $(if $(k),-k=$(k))
 # Disable parallelism which running a specific test.
 # Parallelism makes debugging difficult (print doesn't work).
 test: MAYBE_PARALLEL := $(if $(k),,-n auto)
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
-test: PRESET := --preset=$(if $(preset),$(preset),minimal)
-test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
+test: PRESET := $(if $(filter true,$(fw)),,--preset=$(if $(preset),$(preset),minimal))
+test: BLS := $(if $(filter true,$(fw)),,--bls-type=$(if $(bls),$(bls),fastest))
+test: MAYBE_ETH2SPEC := $(if $(filter true,$(fw)),,$(PYSPEC_DIR)/eth2spec)
 test: pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
@@ -114,25 +117,8 @@ test: pyspec
 		$(PRESET) \
 		$(BLS) \
 		--junitxml=$(TEST_REPORT_DIR)/test_results.xml \
-		$(PYSPEC_DIR)/eth2spec
-
-# Run test framework tests.
-#
-# To run a specific test, append k=<test>, eg:
-#   make test k=test_verify_kzg_proof
-# To run tests with a specific bls library, append bls=<bls>, eg:
-#   make test bls=arkworks
-fw_test: MAYBE_TEST := $(if $(k),-k=$(k))
-# Disable parallelism which running a specific test.
-# Parallelism makes debugging difficult (print doesn't work).
-test_infra: MAYBE_PARALLEL := $(if $(k),,-n auto)
-test_infra: pyspec
-	@mkdir -p $(TEST_REPORT_DIR)
-	@$(PYTHON_VENV) -m pytest \
-		$(MAYBE_PARALLEL) \
-		--capture=no \
-		$(MAYBE_TEST) \
-		$(CURDIR)/tests/infra
+		$(CURDIR)/tests/infra \
+		$(MAYBE_ETH2SPEC)
 
 ###############################################################################
 # Coverage

From d05890ba83a6dc310c0be606ce1d0faeb11864fe Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Thu, 28 Aug 2025 15:03:32 -0500
Subject: [PATCH 05/11] Add framework tests step

---
 .github/workflows/run-tests.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 703ee0f2b0..f9c679ead1 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -53,8 +53,21 @@ jobs:
       - name: Check fork comments
         run: python3 scripts/check_fork_comments.py
 
+  framework:
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - name: Run framework tests
+        run: make test fw=true
+
   tests:
-    needs: [lint, whitespace, comments]
+    needs: [lint, whitespace, comments, framework]
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     strategy:
       matrix:

From ab76fc9896f06d1ce90c538ee3f2a9041715e890 Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Thu, 28 Aug 2025 20:59:30 -0500
Subject: [PATCH 06/11] Fix merge mistakes

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cd69420356..7579ea30a7 100644
--- a/Makefile
+++ b/Makefile
@@ -71,7 +71,7 @@ help-verbose:
 	@echo "    preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
 	@echo "    bls=<type>      BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
 	@echo "    fw=<bool>       Run only the framework tests if true"
-  @echo ""
+	@echo ""
 	@echo "  Examples:"
 	@echo "    make test"
 	@echo "    make test k=test_verify_kzg_proof"
@@ -79,7 +79,7 @@ help-verbose:
 	@echo "    make test preset=mainnet"
 	@echo "    make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
 	@echo "    make test bls=arkworks"
-  @echo "    make test fw=true"
+	@echo "    make test fw=true"
 	@echo ""
 	@echo "$(BOLD)make coverage$(NORM)"
 	@echo ""

From b599bd1349d907d349228c9986f8fd6eacbf5304 Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Thu, 28 Aug 2025 23:07:19 -0500
Subject: [PATCH 07/11] Rethink solution

---
 .github/workflows/run-tests.yml |  4 ++--
 Makefile                        | 21 +++++++++++----------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 38b6feb5c7..f5ed70aeed 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -64,7 +64,7 @@ jobs:
           python-version: '3.13'
           cache: 'pip'
       - name: Run framework tests
-        run: make test fw=true
+        run: make test component=fw
 
   tests:
     needs: [lint, whitespace, comments, framework]
@@ -91,4 +91,4 @@ jobs:
           python-version: '3.13'
           cache: 'pip'
       - name: Run pyspec tests for ${{ matrix.fork }}
-        run: make test preset=minimal fork=${{ matrix.fork }}
+        run: make test component=pyspec preset=minimal fork=${{ matrix.fork }}
diff --git a/Makefile b/Makefile
index 7579ea30a7..836ab3ad39 100644
--- a/Makefile
+++ b/Makefile
@@ -66,11 +66,11 @@ help-verbose:
 	@echo "  by default using pytest with the minimal preset and fastest BLS library."
 	@echo ""
 	@echo "  Parameters:"
-	@echo "    k=<test>        Run specific test by name"
-	@echo "    fork=<fork>     Test specific fork (phase0, altair, bellatrix, capella, etc.)"
-	@echo "    preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
-	@echo "    bls=<type>      BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
-	@echo "    fw=<bool>       Run only the framework tests if true"
+	@echo "    k=<test>          Run specific test by name"
+	@echo "    fork=<fork>       Test specific fork (phase0, altair, bellatrix, capella, etc.)"
+	@echo "    preset=<preset>   Use specific preset (mainnet or minimal; default: minimal)"
+	@echo "    bls=<type>        BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
+	@echo "    component=<value> Test component: (all, pyspec, fw; default: all)"
 	@echo ""
 	@echo "  Examples:"
 	@echo "    make test"
@@ -79,7 +79,7 @@ help-verbose:
 	@echo "    make test preset=mainnet"
 	@echo "    make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
 	@echo "    make test bls=arkworks"
-	@echo "    make test fw=true"
+	@echo "    make test component=fw"
 	@echo ""
 	@echo "$(BOLD)make coverage$(NORM)"
 	@echo ""
@@ -227,9 +227,10 @@ test: MAYBE_TEST := $(if $(k),-k=$(k))
 # Parallelism makes debugging difficult (print doesn't work).
 test: MAYBE_PARALLEL := $(if $(k),,-n auto)
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
-test: PRESET := $(if $(filter true,$(fw)),,--preset=$(if $(preset),$(preset),minimal))
-test: BLS := $(if $(filter true,$(fw)),,--bls-type=$(if $(bls),$(bls),fastest))
-test: MAYBE_ETH2SPEC := $(if $(filter true,$(fw)),,$(PYSPEC_DIR)/eth2spec)
+test: PRESET := $(if $(filter fw,$(component)),,--preset=$(if $(preset),$(preset),minimal))
+test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
+test: MAYBE_ETH2SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth2spec)
+test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra)
 test: _pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
@@ -240,7 +241,7 @@ test: _pyspec
 		$(PRESET) \
 		$(BLS) \
 		--junitxml=$(TEST_REPORT_DIR)/test_results.xml \
-		$(CURDIR)/tests/infra \
+		$(MAYBE_INFRA) \
 		$(MAYBE_ETH2SPEC)
 
 ###############################################################################

From 4daf57688cad831e03741d805024882033bb044a Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Thu, 28 Aug 2025 23:12:39 -0500
Subject: [PATCH 08/11] Update release & nightly workflows

---
 .github/workflows/nightly-tests.yml |  2 +-
 .github/workflows/release.yml       | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml
index 12f24f45e5..9666e8af5f 100644
--- a/.github/workflows/nightly-tests.yml
+++ b/.github/workflows/nightly-tests.yml
@@ -42,4 +42,4 @@ jobs:
         with:
           python-version: '3.13'
       - name: test-${{ matrix.fork }}
-        run: make test preset=mainnet fork=${{ matrix.fork }}
+        run: make test component=pyspec preset=mainnet fork=${{ matrix.fork }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ee3833552a..e1f0bbefcb 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -37,17 +37,23 @@ jobs:
             exit 1
           fi
 
+      # Ensure framework tests pass
+      - name: Run tests for minimal
+        run: |
+          cd consensus-specs
+          make test component=fw
+
       # Ensure minimal tests pass
       - name: Run tests for minimal
         run: |
           cd consensus-specs
-          make test preset=minimal
+          make test component=pyspec preset=minimal
 
       # Ensure mainnet tests pass
       - name: Run tests for mainnet
         run: |
           cd consensus-specs
-          make test preset=mainnet
+          make test component=pyspec preset=mainnet
 
       # Add support for large files
       - name: Install Git LFS

From 38bb4b639985382160cbdf3e33168768311a0709 Mon Sep 17 00:00:00 2001
From: Justin Traglia <95511699+jtraglia@users.noreply.github.com>
Date: Fri, 29 Aug 2025 06:49:39 -0500
Subject: [PATCH 09/11] Fix workflow step name

Co-authored-by: Leo Lara <leolara@users.noreply.github.com>
---
 .github/workflows/release.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e1f0bbefcb..bf9b787d8c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -38,7 +38,8 @@ jobs:
           fi
 
       # Ensure framework tests pass
-      - name: Run tests for minimal
+      - name: Run tests for test framework
+      - continue-on-error: true
         run: |
           cd consensus-specs
           make test component=fw

From 1874b0c470508100e59fa61569ec7260a85c3985 Mon Sep 17 00:00:00 2001
From: Justin Traglia <jtraglia@pm.me>
Date: Fri, 29 Aug 2025 06:56:40 -0500
Subject: [PATCH 10/11] Fix mistake & remove continue-on-error line

---
 .github/workflows/release.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bf9b787d8c..110b283ce4 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -39,7 +39,6 @@ jobs:
 
       # Ensure framework tests pass
       - name: Run tests for test framework
-      - continue-on-error: true
         run: |
           cd consensus-specs
           make test component=fw

From 5ba45c78662fef590b510f7386141d9b7c80605b Mon Sep 17 00:00:00 2001
From: Leo Lara <leo@leolara.me>
Date: Mon, 15 Sep 2025 07:51:29 +0000
Subject: [PATCH 11/11] Fix test

---
 tests/infra/test_template_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/infra/test_template_test.py b/tests/infra/test_template_test.py
index beb1af5954..179b8ba137 100644
--- a/tests/infra/test_template_test.py
+++ b/tests/infra/test_template_test.py
@@ -248,10 +248,10 @@ def test_func():
 
         _test_template("pos1", param2="kwarg1", param3="kwarg2")
 
-        test_name = "test_kwargs_pos_kw_arg"
+        test_name = "test_kwargs_pos1_kwarg1_kwarg2"
         assert hasattr(test_module, test_name)
         test_func = getattr(test_module, test_name)
-        assert test_func() == "test_pos_kw_arg"
+        assert test_func() == "test_pos1_kwarg1_kwarg2"
 
     def test_frame_inspection_error_no_current_frame(self):
         """Test error handling when currentframe returns None."""