From 131e706979a9f5c28609ec7a117ef74922cdbc3f Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Wed, 28 May 2025 23:52:36 +0000 Subject: [PATCH 01/16] feat(docs): Generate EIP Checklists --- docs/filling_tests/eip_checklist.md | 81 ++++++ docs/scripts/gen_test_case_reference.py | 4 + .../eip_testing_checklist_template.md | 6 + pyproject.toml | 1 + src/cli/pytest_commands/__init__.py | 2 + src/cli/pytest_commands/checklist.py | 86 ++++++ src/pytest_plugins/filler/eip_checklist.py | 266 ++++++++++++++++++ src/pytest_plugins/filler/filler.py | 12 +- .../filler/gen_test_doc/gen_test_doc.py | 8 +- .../filler/gen_test_doc/page_props.py | 26 +- src/pytest_plugins/shared/execute_fill.py | 11 + 11 files changed, 496 insertions(+), 7 deletions(-) create mode 100644 docs/filling_tests/eip_checklist.md create mode 100644 src/cli/pytest_commands/checklist.py create mode 100644 src/pytest_plugins/filler/eip_checklist.py diff --git a/docs/filling_tests/eip_checklist.md b/docs/filling_tests/eip_checklist.md new file mode 100644 index 00000000000..8d93f4b6fda --- /dev/null +++ b/docs/filling_tests/eip_checklist.md @@ -0,0 +1,81 @@ +# EIP Checklist Generation + +The EIP checklist feature helps track test coverage for EIP implementations by automatically generating filled checklists based on test markers. + +## Overview + +When implementing tests for an EIP, you can mark specific tests as covering checklist items from the [EIP testing checklist template](../writing_tests/checklist_templates/eip_testing_checklist_template.md). The framework will then generate a filled checklist showing which items have been implemented. + +## Using the `pytest.mark.eip_checklist` Marker + +To mark a test as implementing a specific checklist item: + +```python +import pytest +from ethereum_test_tools import StateTestFiller + +@pytest.mark.eip_checklist("new_transaction_type/test/intrinsic_validity/gas_limit/exact") +def test_exact_intrinsic_gas(state_test: StateTestFiller): + """Test transaction with exact intrinsic gas limit.""" + # Test implementation + pass +``` + +### Marker Parameters + +- **First positional parameter** (required): The checklist item ID from the template +- **`eip` keyword parameter** (optional): List of additional EIPs covered by the test + +Example with multiple EIPs: + +```python +@pytest.mark.eip_checklist("new_transaction_type/test/signature/invalid/v/0", eip=[7702, 2930]) +def test_invalid_signature(state_test: StateTestFiller): + """Test invalid signature that affects multiple EIPs.""" + pass +``` + +## Generating Checklists + +### During Normal Test Filling + +When running the `fill` command, checklists are automatically generated alongside fixtures: + +```bash +fill tests/prague/eip7702_set_code_tx +``` + +The checklist will be created in the output directory: `fixtures/eip7702/eip7702_checklist.md` + +### Automatic Generation in Documentation + +When building the documentation with `mkdocs`, checklists are automatically generated for all EIPs that have tests with checklist markers. The checklists appear in the test documentation alongside the test modules. + +### Using the Dedicated `checklist` Command + +To generate only checklists without filling fixtures: + +```bash +# Generate checklists for all EIPs +checklist + +# Generate checklist for specific EIP +checklist --eip 7702 + +# Generate checklists for specific test path +checklist tests/prague/eip7702* + +# Specify output directory +checklist --output ./my-checklists + +# Multiple EIPs +checklist --eip 7702 --eip 2930 +``` + +## Output Format + +The generated checklist will show: +- โœ“ for completed items +- Test names that implement each item (up to 3, with "..." if more) +- Empty cells for uncompleted items +- A percentage of covered checklist items diff --git a/docs/scripts/gen_test_case_reference.py b/docs/scripts/gen_test_case_reference.py index b3f43ae901d..fa0c056e603 100644 --- a/docs/scripts/gen_test_case_reference.py +++ b/docs/scripts/gen_test_case_reference.py @@ -43,9 +43,13 @@ "filterwarnings=ignore::pytest.PytestAssertRewriteWarning", # suppress warnings due to reload "-p", "pytest_plugins.filler.gen_test_doc.gen_test_doc", + "-p", + "pytest_plugins.filler.eip_checklist", + "--collect-only", "--gen-docs", f"--gen-docs-target-fork={TARGET_FORK}", f"--until={GENERATE_UNTIL_FORK}", + "--checklist-doc-gen", "--skip-index", "-m", "not blockchain_test_engine", diff --git a/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md b/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md index bd63ee657c2..04751ebe2c9 100644 --- a/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md +++ b/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md @@ -3,6 +3,12 @@ Depending on the changes introduced by an EIP, the following template is the minimum baseline to guarantee test coverage of the Execution Layer features. +## Checklist Progress Tracker + +| Total Checklist Items | Covered Checklist Items | Percentage | +| --------------------- | ----------------------- | ---------- | +| TOTAL_CHECKLIST_ITEMS | COVERED_CHECKLIST_ITEMS | PERCENTAGE | + ## General #### Code coverage diff --git a/pyproject.toml b/pyproject.toml index 39ec9da6042..d1d7dfa2e7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ checkfixtures = "cli.check_fixtures:check_fixtures" check_eip_versions = "cli.pytest_commands.check_eip_versions:check_eip_versions" consume = "cli.pytest_commands.consume:consume" protec = "cli.pytest_commands.consume:consume" +checklist = "cli.pytest_commands.checklist:checklist" genindex = "cli.gen_index:generate_fixtures_index_cli" gentest = "cli.gentest:generate" eofwrap = "cli.eofwrap:eof_wrap" diff --git a/src/cli/pytest_commands/__init__.py b/src/cli/pytest_commands/__init__.py index b1496dfb7e9..0c465f49c73 100644 --- a/src/cli/pytest_commands/__init__.py +++ b/src/cli/pytest_commands/__init__.py @@ -19,6 +19,8 @@ fill --help # for example, or consume engine +# or +checklist --help ``` They can also be executed (and debugged) directly in an interactive python diff --git a/src/cli/pytest_commands/checklist.py b/src/cli/pytest_commands/checklist.py new file mode 100644 index 00000000000..4d1de1c39e0 --- /dev/null +++ b/src/cli/pytest_commands/checklist.py @@ -0,0 +1,86 @@ +"""CLI entry point for the `checklist` pytest-based command.""" + +from typing import List + +import click + +from .base import PytestCommand, common_pytest_options +from .processors import HelpFlagsProcessor + + +class ChecklistCommand(PytestCommand): + """Pytest command for generating EIP checklists.""" + + def __init__(self): + """Initialize checklist command with processors.""" + super().__init__( + config_file="pytest.ini", + argument_processors=[ + HelpFlagsProcessor("checklist"), + ], + ) + + def process_arguments(self, pytest_args: List[str]) -> List[str]: + """Process arguments, ensuring checklist generation is enabled.""" + processed_args = super().process_arguments(pytest_args) + + # Add collect-only flag to avoid running tests + if "--collect-only" not in processed_args: + processed_args.append("--collect-only") + + processed_args.extend(["-p", "pytest_plugins.filler.eip_checklist"]) + + return processed_args + + +@click.command( + context_settings={ + "ignore_unknown_options": True, + } +) +@common_pytest_options +@click.option( + "--output", + "-o", + type=click.Path(file_okay=False, dir_okay=True, writable=True), + default="./checklists", + help="Directory to output the generated checklists (default: ./checklists)", +) +@click.option( + "--eip", + "-e", + type=int, + multiple=True, + help="Generate checklist only for specific EIP(s)", +) +def checklist(pytest_args: List[str], output: str, eip: tuple, **kwargs) -> None: + """ + Generate EIP test checklists based on pytest.mark.eip_checklist markers. + + This command scans test files for eip_checklist markers and generates + filled checklists showing which checklist items have been implemented. + + Examples: + # Generate checklists for all EIPs + eest checklist + + # Generate checklist for specific EIP + eest checklist --eip 7702 + + # Generate checklists for specific test path + eest checklist tests/prague/eip7702* + + # Specify output directory + eest checklist --output ./my-checklists + + """ + # Add output directory to pytest args + extended_args = list(pytest_args) + extended_args.extend(["--checklist-output", output]) + + # Add EIP filter if specified + for eip_num in eip: + extended_args.extend(["--checklist-eip", str(eip_num)]) + + command = ChecklistCommand() + command.execute(extended_args) diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py new file mode 100644 index 00000000000..4a81746e800 --- /dev/null +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -0,0 +1,266 @@ +""" +Pytest plugin for generating EIP test completion checklists. + +This plugin collects checklist markers from tests and generates a filled checklist +for each EIP based on the template at +docs/writing_tests/checklist_templates/eip_testing_checklist_template.md +""" + +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +import pytest + +from .gen_test_doc.page_props import EipChecklistPageProps + + +def pytest_addoption(parser: pytest.Parser): + """Add command-line options for checklist generation.""" + group = parser.getgroup("checklist", "EIP checklist generation options") + group.addoption( + "--checklist-output", + action="store", + dest="checklist_output", + type=Path, + default=Path("./checklists"), + help="Directory to output the generated checklists", + ) + group.addoption( + "--checklist-eip", + action="append", + dest="checklist_eips", + type=int, + default=[], + help="Generate checklist only for specific EIP(s)", + ) + group.addoption( + "--checklist-doc-gen", + action="store_true", + dest="checklist_doc_gen", + default=False, + help="Generate checklists for documentation (uses mkdocs_gen_files)", + ) + + +TITLE_LINE = "# EIP Execution Layer Testing Checklist Template" +PERCENTAGE_LINE = "| TOTAL_CHECKLIST_ITEMS | COVERED_CHECKLIST_ITEMS | PERCENTAGE |" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): # noqa: D103 + config.pluginmanager.register(EIPChecklistCollector(), "eip-checklist-collector") + + +@dataclass +class EIPItem: + """Represents an EIP checklist item.""" + + id: str + description: str + tests: List[str] + covered: bool = False + + @classmethod + def from_checklist_line(cls, line: str) -> "EIPItem | None": + """Create an EIP item from a checklist line.""" + match = re.match(r"\|\s*`([^`]+)`\s*\|\s*([^|]+)\s*\|", line) + if not match: + return None + return cls(id=match.group(1), description=match.group(2), tests=[], covered=False) + + def with_tests(self, tests: List[str]) -> "EIPItem": + """Return a new EIP item with the given tests.""" + new_tests = sorted(set(self.tests + tests)) + return EIPItem( + id=self.id, + description=self.description, + tests=new_tests, + covered=len(new_tests) > 0, + ) + + def __str__(self) -> str: + """Return a string representation of the EIP item.""" + return ( + f"| `{self.id}` " + f"| {self.description} " + f"| {'โœ…' if self.covered else ' '} " + f"| {', '.join(self.tests)} " + "|" + ) + + +class EIPChecklistCollector: + """Collects and manages EIP checklist items from test markers.""" + + def __init__(self): + """Initialize the EIP checklist collector.""" + self.eip_checklist_items: Dict[int, Dict[str, Set[Tuple[str, str]]]] = defaultdict( + lambda: defaultdict(set) + ) + self.template_path = ( + Path(__file__).parents[3] + / "docs" + / "writing_tests" + / "checklist_templates" + / "eip_testing_checklist_template.md" + ) + self.template_content: Optional[str] = None + self.template_items: Dict[str, Tuple[EIPItem, int]] = {} # ID -> (item, line_number) + self.eip_paths: Dict[int, Path] = {} + + def load_template(self) -> None: + """Load and parse the checklist template.""" + if not self.template_path.exists(): + pytest.fail(f"EIP checklist template not found at {self.template_path}") + + self.template_content = self.template_path.read_text() + + # Parse the template to extract checklist item IDs and descriptions + lines = self.template_content.splitlines() + for i, line in enumerate(lines): + # Match lines that contain checklist items with IDs in backticks + item = EIPItem.from_checklist_line(line) + if item: + self.template_items[item.id] = (item, i + 1) + + def extract_eip_from_path(self, test_path: Path) -> Optional[int]: + """Extract EIP number from test file path.""" + # Look for patterns like eip1234_ or eip1234/ in the path + for part_idx, part in enumerate(test_path.parts): + match = re.match(r"eip(\d+)", part) + if match: + eip = int(match.group(1)) + if eip not in self.eip_paths: + self.eip_paths[eip] = test_path.parents[len(test_path.parents) - part_idx - 2] + return eip + return None + + def collect_from_item(self, item: pytest.Item) -> None: + """Collect checklist markers from a test item.""" + for marker in item.iter_markers("eip_checklist"): + if not marker.args: + pytest.fail( + f"eip_checklist marker on {item.nodeid} must have at least one argument " + "(item_id)" + ) + additional_eips = marker.kwargs.get("eip", []) + if not isinstance(additional_eips, list): + additional_eips = [additional_eips] + + # Get the primary EIP from the test path + primary_eip = self.extract_eip_from_path(Path(item.location[0])) + if primary_eip is None: + if not additional_eips: + pytest.fail( + f"Could not extract EIP number from test path: {item.path}. " + "Marker 'eip_checklist' can only be used in tests that are located in a " + "directory named after the EIP number, or with the eip keyword argument." + ) + eips = additional_eips + else: + eips = [primary_eip] + additional_eips + + if any(not isinstance(eip, int) for eip in eips): + pytest.fail( + "EIP numbers must be integers. Found non-integer EIPs in " + f"{item.nodeid}: {eips}" + ) + + for item_id in marker.args: + for eip in eips: + self.eip_checklist_items[eip][item_id].add((item.nodeid, item.name)) + + def generate_filled_checklist_lines(self, eip: int) -> List[str]: + """Generate the filled checklist lines for a specific EIP.""" + if not self.template_content or not self.template_items: + self.load_template() + + # Get all checklist items for this EIP + eip_items = self.eip_checklist_items.get(eip, {}) + + # Create a copy of the template content + filled_content = self.template_content + lines = filled_content.splitlines() + + # Process each line in reverse order to avoid index shifting + total_items = len(self.template_items) + covered_items = 0 + for item_id, (checklist_item, line_num) in sorted( + self.template_items.items(), key=lambda x: x[1][1], reverse=True + ): + if item_id in eip_items: + # Find the line with this item ID + line_idx = line_num - 1 + checklist_item = checklist_item.with_tests( + [f"`{test_node_id}`" for test_node_id, _ in eip_items[item_id]] + ) + lines[line_idx] = str(checklist_item) + covered_items += 1 + + percentage = round(covered_items / total_items * 100) + completness_emoji = "๐ŸŸข" if percentage == 100 else "๐ŸŸก" if percentage > 50 else "๐Ÿ”ด" + lines[lines.index(PERCENTAGE_LINE)] = ( + f"| {total_items} | {covered_items} | {completness_emoji} {percentage:.2f}% |" + ) + + # Replace the title line with the EIP number + lines[lines.index(TITLE_LINE)] = f"# EIP-{eip} Test Checklist" + + return lines + + def generate_filled_checklist(self, eip: int, output_dir: Path) -> Path: + """Generate a filled checklist for a specific EIP.""" + lines = self.generate_filled_checklist_lines(eip) + + output_dir = output_dir / f"eip{eip}_checklist.md" + + # Write the filled checklist + output_dir.parent.mkdir(exist_ok=True, parents=True) + output_dir.write_text("\n".join(lines)) + + return output_dir + + def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): + """Collect checklist markers during test collection.""" + for item in items: + if item.get_closest_marker("derived_test"): + continue + self.collect_from_item(item) + + if not self.eip_checklist_items: + return + + # Check which mode we are in + checklist_doc_gen = config.getoption("checklist_doc_gen", False) + checklist_output = config.getoption("checklist_output", Path("checklists")) + checklist_eips = config.getoption("checklist_eips", []) + + if checklist_doc_gen: + config.checklist_props = {} + + # Generate a checklist for each EIP + for eip in sorted(self.eip_checklist_items.keys()): + # Skip if specific EIPs were requested and this isn't one of them + if checklist_eips and eip not in checklist_eips: + continue + + if checklist_doc_gen: + if eip not in self.eip_paths: + continue + eip_path = self.eip_paths[eip] + config.checklist_props[eip_path / "checklist.md"] = EipChecklistPageProps( + title=f"EIP-{eip} Test Checklist", + source_code_url="", + target_or_valid_fork="mainnet", + path=eip_path / "checklist.md", + pytest_node_id="", + package_name="eip_checklist", + eip=eip, + lines=self.generate_filled_checklist_lines(eip), + ) + else: + checklist_path = self.generate_filled_checklist(eip, checklist_output) + print(f"Generated EIP-{eip} checklist: {checklist_path}") diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index e1a4a1256f2..b0d63c421b7 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -39,6 +39,7 @@ labeled_format_parameter_set, ) from ..spec_version_checker.spec_version_checker import get_ref_spec_from_module +from . import eip_checklist # noqa: F401 from .fixture_output import FixtureOutput @@ -796,12 +797,15 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): """ for test_type in BaseTest.spec_types.values(): if test_type.pytest_parameter_name() in metafunc.fixturenames: + parameters = [] + for i, format_with_or_without_label in enumerate(test_type.supported_fixture_formats): + parameter = labeled_format_parameter_set(format_with_or_without_label) + if i > 0: + parameter.marks.append(pytest.mark.derived_test) + parameters.append(parameter) metafunc.parametrize( [test_type.pytest_parameter_name()], - [ - labeled_format_parameter_set(format_with_or_without_label) - for format_with_or_without_label in test_type.supported_fixture_formats - ], + parameters, scope="function", indirect=True, ) diff --git a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py index aa91aeba446..f272021fc0c 100644 --- a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py +++ b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py @@ -254,7 +254,7 @@ def __init__(self, config) -> None: self.page_props: PagePropsLookup = {} @pytest.hookimpl(hookwrapper=True, trylast=True) - def pytest_collection_modifyitems(self, session, config, items): + def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): """Generate html doc for each test item that pytest has collected.""" yield @@ -264,11 +264,15 @@ def pytest_collection_modifyitems(self, session, config, items): for item in items: # group test case by test function functions[get_test_function_id(item)].append(item) + if hasattr(config, "checklist_props"): + checklist_props = config.checklist_props + self.page_props = {**self.page_props, **checklist_props} + # the heavy work self.create_function_page_props(functions) self.create_module_page_props() # add the pages to the page_props dict - self.page_props = {**self.function_page_props, **self.module_page_props} + self.page_props = {**self.page_props, **self.function_page_props, **self.module_page_props} # this adds pages for the intermediate directory structure (tests, tests/berlin) self.add_directory_page_props() # add other interesting pages diff --git a/src/pytest_plugins/filler/gen_test_doc/page_props.py b/src/pytest_plugins/filler/gen_test_doc/page_props.py index 67a02467a9b..652d958c4df 100644 --- a/src/pytest_plugins/filler/gen_test_doc/page_props.py +++ b/src/pytest_plugins/filler/gen_test_doc/page_props.py @@ -123,6 +123,24 @@ def write_page(self, jinja2_env: Environment): destination.write(line) +@dataclass +class EipChecklistPageProps(PagePropsBase): + """Properties used to generate the EIP checklist page.""" + + eip: int + lines: List[str] + + @property + def target_output_file(self) -> Path: + """Get the target output file for this page.""" + return self.path + + def write_page(self, jinja2_env: Environment): + """Write the page to the target directory.""" + with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + destination.write("\n".join(self.lines)) + + @dataclass class TestCase: """Properties used to define a single test case in test function parameter tables.""" @@ -256,7 +274,13 @@ def write_page(self, jinja2_env: Environment): destination.write(line) -PageProps = DirectoryPageProps | ModulePageProps | FunctionPageProps | MarkdownPageProps +PageProps = ( + DirectoryPageProps + | ModulePageProps + | FunctionPageProps + | MarkdownPageProps + | EipChecklistPageProps +) PagePropsLookup = Dict[str, PageProps] ModulePagePropsLookup = Dict[str, ModulePageProps] FunctionPagePropsLookup = Dict[str, FunctionPageProps] diff --git a/src/pytest_plugins/shared/execute_fill.py b/src/pytest_plugins/shared/execute_fill.py index ae9706624c1..7eb41db754e 100644 --- a/src/pytest_plugins/shared/execute_fill.py +++ b/src/pytest_plugins/shared/execute_fill.py @@ -89,6 +89,17 @@ def pytest_configure(config: pytest.Config): "markers", "exception_test: Negative tests that include an invalid block or transaction.", ) + config.addinivalue_line( + "markers", + "eip_checklist(item_id, eip=None): Mark a test as implementing a specific checklist item. " + "The first positional parameter is the checklist item ID. " + "The optional 'eip' keyword parameter specifies additional EIPs covered by the test.", + ) + config.addinivalue_line( + "markers", + "derived_test: Mark a test as a derived test (E.g. a BlockchainTest that is derived " + "from a StateTest).", + ) @pytest.fixture(autouse=True) From 528d66ac8851807149fa84a68925982da1855a5e Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 14:39:39 +0000 Subject: [PATCH 02/16] fix(docs): Correctly include checklist gen in mkdocs flow --- docs/filling_tests/eip_checklist.md | 3 +- src/pytest_plugins/filler/eip_checklist.py | 51 +++++++++++-------- src/pytest_plugins/filler/filler.py | 2 +- .../filler/gen_test_doc/page_props.py | 5 ++ 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/docs/filling_tests/eip_checklist.md b/docs/filling_tests/eip_checklist.md index 8d93f4b6fda..50d9f137a6c 100644 --- a/docs/filling_tests/eip_checklist.md +++ b/docs/filling_tests/eip_checklist.md @@ -75,7 +75,8 @@ checklist --eip 7702 --eip 2930 ## Output Format The generated checklist will show: -- โœ“ for completed items + +- โœ… for completed items - Test names that implement each item (up to 3, with "..." if more) - Empty cells for uncompleted items - A percentage of covered checklist items diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py index 4a81746e800..51a9c6f3fc2 100644 --- a/src/pytest_plugins/filler/eip_checklist.py +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -6,6 +6,7 @@ docs/writing_tests/checklist_templates/eip_testing_checklist_template.md """ +import logging import re from collections import defaultdict from dataclasses import dataclass @@ -16,6 +17,8 @@ from .gen_test_doc.page_props import EipChecklistPageProps +logger = logging.getLogger("mkdocs") + def pytest_addoption(parser: pytest.Parser): """Add command-line options for checklist generation.""" @@ -95,7 +98,7 @@ def __str__(self) -> str: class EIPChecklistCollector: """Collects and manages EIP checklist items from test markers.""" - def __init__(self): + def __init__(self: "EIPChecklistCollector"): """Initialize the EIP checklist collector.""" self.eip_checklist_items: Dict[int, Dict[str, Set[Tuple[str, str]]]] = defaultdict( lambda: defaultdict(set) @@ -107,16 +110,14 @@ def __init__(self): / "checklist_templates" / "eip_testing_checklist_template.md" ) - self.template_content: Optional[str] = None - self.template_items: Dict[str, Tuple[EIPItem, int]] = {} # ID -> (item, line_number) self.eip_paths: Dict[int, Path] = {} - def load_template(self) -> None: - """Load and parse the checklist template.""" if not self.template_path.exists(): pytest.fail(f"EIP checklist template not found at {self.template_path}") self.template_content = self.template_path.read_text() + self.template_items: Dict[str, Tuple[EIPItem, int]] = {} # ID -> (item, line_number) + self.all_ids: Set[str] = set() # Parse the template to extract checklist item IDs and descriptions lines = self.template_content.splitlines() @@ -125,20 +126,20 @@ def load_template(self) -> None: item = EIPItem.from_checklist_line(line) if item: self.template_items[item.id] = (item, i + 1) + self.all_ids.add(item.id) - def extract_eip_from_path(self, test_path: Path) -> Optional[int]: + def extract_eip_from_path(self, test_path: Path) -> Tuple[Optional[int], Optional[Path]]: """Extract EIP number from test file path.""" # Look for patterns like eip1234_ or eip1234/ in the path for part_idx, part in enumerate(test_path.parts): match = re.match(r"eip(\d+)", part) if match: eip = int(match.group(1)) - if eip not in self.eip_paths: - self.eip_paths[eip] = test_path.parents[len(test_path.parents) - part_idx - 2] - return eip - return None + eip_path = test_path.parents[len(test_path.parents) - part_idx - 2] + return eip, eip_path + return None, None - def collect_from_item(self, item: pytest.Item) -> None: + def collect_from_item(self, item: pytest.Item, primary_eip: Optional[int]) -> None: """Collect checklist markers from a test item.""" for marker in item.iter_markers("eip_checklist"): if not marker.args: @@ -151,7 +152,6 @@ def collect_from_item(self, item: pytest.Item) -> None: additional_eips = [additional_eips] # Get the primary EIP from the test path - primary_eip = self.extract_eip_from_path(Path(item.location[0])) if primary_eip is None: if not additional_eips: pytest.fail( @@ -170,14 +170,18 @@ def collect_from_item(self, item: pytest.Item) -> None: ) for item_id in marker.args: + if item_id not in self.all_ids: + # TODO: If we decide to do the starts-with matching, we have to change this. + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for test {item.nodeid}" + ) + continue for eip in eips: self.eip_checklist_items[eip][item_id].add((item.nodeid, item.name)) def generate_filled_checklist_lines(self, eip: int) -> List[str]: """Generate the filled checklist lines for a specific EIP.""" - if not self.template_content or not self.template_items: - self.load_template() - # Get all checklist items for this EIP eip_items = self.eip_checklist_items.get(eip, {}) @@ -200,7 +204,7 @@ def generate_filled_checklist_lines(self, eip: int) -> List[str]: lines[line_idx] = str(checklist_item) covered_items += 1 - percentage = round(covered_items / total_items * 100) + percentage = covered_items / total_items * 100 completness_emoji = "๐ŸŸข" if percentage == 100 else "๐ŸŸก" if percentage > 50 else "๐Ÿ”ด" lines[lines.index(PERCENTAGE_LINE)] = ( f"| {total_items} | {covered_items} | {completness_emoji} {percentage:.2f}% |" @@ -226,9 +230,12 @@ def generate_filled_checklist(self, eip: int, output_dir: Path) -> Path: def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): """Collect checklist markers during test collection.""" for item in items: + eip, eip_path = self.extract_eip_from_path(Path(item.location[0])) + if eip_path is not None: + self.eip_paths[eip] = eip_path if item.get_closest_marker("derived_test"): continue - self.collect_from_item(item) + self.collect_from_item(item, eip) if not self.eip_checklist_items: return @@ -238,8 +245,7 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes checklist_output = config.getoption("checklist_output", Path("checklists")) checklist_eips = config.getoption("checklist_eips", []) - if checklist_doc_gen: - config.checklist_props = {} + checklist_props = {} # Generate a checklist for each EIP for eip in sorted(self.eip_checklist_items.keys()): @@ -248,10 +254,8 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes continue if checklist_doc_gen: - if eip not in self.eip_paths: - continue eip_path = self.eip_paths[eip] - config.checklist_props[eip_path / "checklist.md"] = EipChecklistPageProps( + checklist_props[eip_path / "checklist.md"] = EipChecklistPageProps( title=f"EIP-{eip} Test Checklist", source_code_url="", target_or_valid_fork="mainnet", @@ -264,3 +268,6 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes else: checklist_path = self.generate_filled_checklist(eip, checklist_output) print(f"Generated EIP-{eip} checklist: {checklist_path}") + + if checklist_doc_gen: + config.checklist_props = checklist_props # type: ignore diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index b0d63c421b7..add8a3de827 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -801,7 +801,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): for i, format_with_or_without_label in enumerate(test_type.supported_fixture_formats): parameter = labeled_format_parameter_set(format_with_or_without_label) if i > 0: - parameter.marks.append(pytest.mark.derived_test) + parameter.marks.append(pytest.mark.derived_test) # type: ignore parameters.append(parameter) metafunc.parametrize( [test_type.pytest_parameter_name()], diff --git a/src/pytest_plugins/filler/gen_test_doc/page_props.py b/src/pytest_plugins/filler/gen_test_doc/page_props.py index 652d958c4df..cba7ac0fe64 100644 --- a/src/pytest_plugins/filler/gen_test_doc/page_props.py +++ b/src/pytest_plugins/filler/gen_test_doc/page_props.py @@ -130,6 +130,11 @@ class EipChecklistPageProps(PagePropsBase): eip: int lines: List[str] + @property + def template(self) -> str: + """Get the jinja2 template used to render this page.""" + raise Exception("EipChecklistPageProps does not have a template") + @property def target_output_file(self) -> Path: """Get the target output file for this page.""" From 9b58fcb2baa59907244614159371f7ba70f5412b Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 15:23:37 +0000 Subject: [PATCH 03/16] fix(plugins/filler): remove unused import --- src/pytest_plugins/filler/filler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index add8a3de827..7a1728b4380 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -39,7 +39,6 @@ labeled_format_parameter_set, ) from ..spec_version_checker.spec_version_checker import get_ref_spec_from_module -from . import eip_checklist # noqa: F401 from .fixture_output import FixtureOutput From 1b3065eb68bf39325970acd8aa7518074093cc37 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 16:56:06 +0000 Subject: [PATCH 04/16] fix(plugins): Tox, unit tests --- src/pytest_plugins/filler/eip_checklist.py | 2 +- .../filler/gen_test_doc/gen_test_doc.py | 2 +- .../filler/gen_test_doc/page_props.py | 32 ++++--- .../filler/tests/test_eip_checklist.py | 92 +++++++++++++++++++ 4 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 src/pytest_plugins/filler/tests/test_eip_checklist.py diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py index 51a9c6f3fc2..df4fa2b37ca 100644 --- a/src/pytest_plugins/filler/eip_checklist.py +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -231,7 +231,7 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes """Collect checklist markers during test collection.""" for item in items: eip, eip_path = self.extract_eip_from_path(Path(item.location[0])) - if eip_path is not None: + if eip_path is not None and eip is not None: self.eip_paths[eip] = eip_path if item.get_closest_marker("derived_test"): continue diff --git a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py index f272021fc0c..52f63604ae3 100644 --- a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py +++ b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py @@ -596,4 +596,4 @@ def sort_by_fork_deployment_and_path(x: PageProps) -> Tuple[Any, ...]: def write_pages(self) -> None: """Write all pages to the target directory.""" for page in self.page_props.values(): - page.write_page(self.jinja2_env) + page.write_page(mkdocs_gen_files, self.jinja2_env) diff --git a/src/pytest_plugins/filler/gen_test_doc/page_props.py b/src/pytest_plugins/filler/gen_test_doc/page_props.py index cba7ac0fe64..ec5e075fb88 100644 --- a/src/pytest_plugins/filler/gen_test_doc/page_props.py +++ b/src/pytest_plugins/filler/gen_test_doc/page_props.py @@ -14,9 +14,8 @@ from abc import abstractmethod from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Dict, List +from typing import IO, Any, ContextManager, Dict, List, Protocol -import mkdocs_gen_files # type: ignore from jinja2 import Environment from ethereum_test_tools import Opcodes @@ -79,6 +78,17 @@ def nav_path_to_sanitized_str_tuple(nav_path: Path) -> tuple: return tuple(sanitize_string_title(part) for part in nav_path.parts) +class FileOpener(Protocol): + """ + Protocol to replace `mkdocs_gen_files` so it doesn't have to be imported/installed for + unit tests. + """ + + def open(self, path: Path, mode: str) -> ContextManager[IO[Any]]: + """Open a file for writing.""" + raise NotImplementedError + + @dataclass class PagePropsBase: """ @@ -114,11 +124,11 @@ def nav_entry(self, top_level_nav_entry: str) -> tuple: path = top_level_nav_entry / Path(*self.path.parts[1:]).with_suffix("") return nav_path_to_sanitized_str_tuple(path) - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """Write the page to the target directory.""" template = jinja2_env.get_template(self.template) rendered_content = template.render(**asdict(self)) - with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + with file_opener.open(self.target_output_file, "w") as destination: for line in rendered_content.splitlines(keepends=True): destination.write(line) @@ -140,9 +150,9 @@ def target_output_file(self) -> Path: """Get the target output file for this page.""" return self.path - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """Write the page to the target directory.""" - with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + with file_opener.open(self.target_output_file, "w") as destination: destination.write("\n".join(self.lines)) @@ -187,20 +197,20 @@ def nav_entry(self, top_level_nav_entry) -> tuple: nav_path_prefix = super().nav_entry(top_level_nav_entry) # already sanitized return (*nav_path_prefix, f"{self.title}") - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """ Test functions also get a static HTML page with parametrized test cases. This is intended for easier viewing (without mkdocs styling) of the data-table that documents the parametrized test cases. """ - super().write_page(jinja2_env) + super().write_page(file_opener, jinja2_env) if not self.cases: return html_template = jinja2_env.get_template("function.html.j2") rendered_html_content = html_template.render(**asdict(self)) html_output_file = self.target_output_file.with_suffix(".html") - with mkdocs_gen_files.open(html_output_file, "w") as destination: + with file_opener.open(html_output_file, "w") as destination: for line in rendered_html_content.splitlines(keepends=True): destination.write(line) @@ -263,7 +273,7 @@ def target_output_file(self) -> Path: """Get the target output file for this page.""" return self.path - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """ Write the page to the target directory. @@ -272,7 +282,7 @@ def write_page(self, jinja2_env: Environment): template = jinja2_env.get_template(self.template) rendered_content = template.render(**asdict(self)) with open(self.path, "r") as md_source: - with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + with file_opener.open(self.target_output_file, "w") as destination: for line in rendered_content.splitlines(keepends=True): destination.write(line) for line in md_source: diff --git a/src/pytest_plugins/filler/tests/test_eip_checklist.py b/src/pytest_plugins/filler/tests/test_eip_checklist.py new file mode 100644 index 00000000000..13bbfcb436a --- /dev/null +++ b/src/pytest_plugins/filler/tests/test_eip_checklist.py @@ -0,0 +1,92 @@ +"""Test the EIP checklist plugin functionality.""" + +import textwrap + + +def test_eip_checklist_collection(testdir): + """Test that checklist markers are collected correctly.""" + # Create the test in an EIP-specific directory + tests_dir = testdir.mkdir("tests") + + prague_tests_dir = tests_dir.mkdir("prague") + eip_7702_tests_dir = prague_tests_dir.mkdir("eip7702_set_code_tx") + test_7702_module = eip_7702_tests_dir.join("test_eip7702.py") + test_7702_module.write( + textwrap.dedent( + """ + import pytest + from ethereum_test_tools import StateTestFiller + + REFERENCE_SPEC_GIT_PATH = "N/A" + REFERENCE_SPEC_VERSION = "N/A" + + @pytest.mark.valid_at("Prague") + @pytest.mark.eip_checklist( + "new_transaction_type/test/intrinsic_validity/gas_limit/exact" + ) + def test_exact_gas(state_test: StateTestFiller): + pass + + @pytest.mark.valid_at("Prague") + @pytest.mark.eip_checklist( + "new_transaction_type/test/signature/invalid/v/2", + eip=[7702, 2930] + ) + def test_invalid_v(state_test: StateTestFiller): + pass + """ + ) + ) + + berlin_tests_dir = tests_dir.mkdir("berlin") + eip_2930_tests_dir = berlin_tests_dir.mkdir("eip2930_set_code_tx") + test_2930_module = eip_2930_tests_dir.join("test_eip2930.py") + test_2930_module.write( + textwrap.dedent( + """ + import pytest + from ethereum_test_tools import StateTestFiller + + REFERENCE_SPEC_GIT_PATH = "N/A" + REFERENCE_SPEC_VERSION = "N/A" + + @pytest.mark.valid_at("Berlin") + def test_berlin_one(state_test: StateTestFiller): + pass + """ + ) + ) + # Run pytest with checklist-only mode + testdir.copy_example(name="pytest.ini") + result = testdir.runpytest( + "-p", + "pytest_plugins.filler.eip_checklist", + "--collect-only", + "--checklist-output", + str(testdir.tmpdir / "checklists"), + str(tests_dir), + ) + result.assert_outcomes( + passed=0, + failed=0, + skipped=0, + errors=0, + ) + + # Check that checklists were generated + checklist_dir = testdir.tmpdir / "checklists" + assert checklist_dir.exists() + checklist_file = checklist_dir / "eip7702_checklist.md" + assert checklist_file.exists() + + # Verify the checklist contains the expected markers + content = checklist_file.read() + assert "โœ…" in content + assert "test_exact_gas" in content + assert "test_invalid_v" in content + + checklist_file = checklist_dir / "eip2930_checklist.md" + assert checklist_file.exists() + content = checklist_file.read() + assert "โœ…" in content + assert "test_invalid_v" in content From 5503491098d083341cbde223ad0c9e31cf06b023 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 17:12:37 +0000 Subject: [PATCH 05/16] fix(clis): Checklist command help --- src/cli/pytest_commands/checklist.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/cli/pytest_commands/checklist.py b/src/cli/pytest_commands/checklist.py index 4d1de1c39e0..57e874f0df2 100644 --- a/src/cli/pytest_commands/checklist.py +++ b/src/cli/pytest_commands/checklist.py @@ -4,8 +4,7 @@ import click -from .base import PytestCommand, common_pytest_options -from .processors import HelpFlagsProcessor +from .base import PytestCommand class ChecklistCommand(PytestCommand): @@ -15,9 +14,6 @@ def __init__(self): """Initialize checklist command with processors.""" super().__init__( config_file="pytest.ini", - argument_processors=[ - HelpFlagsProcessor("checklist"), - ], ) def process_arguments(self, pytest_args: List[str]) -> List[str]: @@ -33,12 +29,7 @@ def process_arguments(self, pytest_args: List[str]) -> List[str]: return processed_args -@click.command( - context_settings={ - "ignore_unknown_options": True, - } -) -@common_pytest_options +@click.command() @click.option( "--output", "-o", @@ -53,7 +44,7 @@ def process_arguments(self, pytest_args: List[str]) -> List[str]: multiple=True, help="Generate checklist only for specific EIP(s)", ) -def checklist(pytest_args: List[str], output: str, eip: tuple, **kwargs) -> None: +def checklist(output: str, eip: tuple, **kwargs) -> None: """ Generate EIP test checklists based on pytest.mark.eip_checklist markers. @@ -75,12 +66,16 @@ def checklist(pytest_args: List[str], output: str, eip: tuple, **kwargs) -> None """ # Add output directory to pytest args - extended_args = list(pytest_args) - extended_args.extend(["--checklist-output", output]) + args = ["-p", "pytest_plugins.filler.eip_checklist"] + args.extend(["--checklist-output", output]) # Add EIP filter if specified for eip_num in eip: - extended_args.extend(["--checklist-eip", str(eip_num)]) + args.extend(["--checklist-eip", str(eip_num)]) command = ChecklistCommand() - command.execute(extended_args) + command.execute(args) + + +if __name__ == "__main__": + checklist() From d01530ac7048a2f4e3bc107c5c44262331e877fd Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 17:17:26 +0000 Subject: [PATCH 06/16] docs: Changelog --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 371b260da10..5a54ce07cdd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -53,6 +53,7 @@ Users can select any of the artifacts depending on their testing needs for their - ๐Ÿž Fix bug in ported-from plugin and coverage script that made PRs fail with modified tests that contained no ported tests ([#1661](https://github.com/ethereum/execution-spec-tests/pull/1661)). - ๐Ÿ”€ Refactor the `click`-based CLI interface used for pytest-based commands (`fill`, `execute`, `consume`) to make them more extensible ([#1654](https://github.com/ethereum/execution-spec-tests/pull/1654)). - ๐Ÿ”€ Split `src/ethereum_test_types/types.py` into several files to improve code organization ([#1665](https://github.com/ethereum/execution-spec-tests/pull/1665)). +- โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679)). ### ๐Ÿงช Test Cases From d80a009507cb8ce8ea37c2d2e9724f33e1f27269 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 17:22:11 +0000 Subject: [PATCH 07/16] fix(docs): Outdated docs --- docs/filling_tests/eip_checklist.md | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/filling_tests/eip_checklist.md b/docs/filling_tests/eip_checklist.md index 50d9f137a6c..65438e695c2 100644 --- a/docs/filling_tests/eip_checklist.md +++ b/docs/filling_tests/eip_checklist.md @@ -26,7 +26,7 @@ def test_exact_intrinsic_gas(state_test: StateTestFiller): - **First positional parameter** (required): The checklist item ID from the template - **`eip` keyword parameter** (optional): List of additional EIPs covered by the test -Example with multiple EIPs: +Example with multiple EIPs covered by the same test: ```python @pytest.mark.eip_checklist("new_transaction_type/test/signature/invalid/v/0", eip=[7702, 2930]) @@ -37,20 +37,6 @@ def test_invalid_signature(state_test: StateTestFiller): ## Generating Checklists -### During Normal Test Filling - -When running the `fill` command, checklists are automatically generated alongside fixtures: - -```bash -fill tests/prague/eip7702_set_code_tx -``` - -The checklist will be created in the output directory: `fixtures/eip7702/eip7702_checklist.md` - -### Automatic Generation in Documentation - -When building the documentation with `mkdocs`, checklists are automatically generated for all EIPs that have tests with checklist markers. The checklists appear in the test documentation alongside the test modules. - ### Using the Dedicated `checklist` Command To generate only checklists without filling fixtures: @@ -62,9 +48,6 @@ checklist # Generate checklist for specific EIP checklist --eip 7702 -# Generate checklists for specific test path -checklist tests/prague/eip7702* - # Specify output directory checklist --output ./my-checklists @@ -72,6 +55,10 @@ checklist --output ./my-checklists checklist --eip 7702 --eip 2930 ``` +### Automatic Generation in Documentation + +When building the documentation with `mkdocs`, checklists are automatically generated for all EIPs that have tests with checklist markers. The checklists appear in the test documentation alongside the test modules. + ## Output Format The generated checklist will show: From 3c1400d3e62c330fea5e565ba4bbbd15f7805c92 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 29 May 2025 17:48:40 +0000 Subject: [PATCH 08/16] fix(plugins): Collect-only parameter --- docs/scripts/gen_test_case_reference.py | 1 - src/cli/pytest_commands/checklist.py | 6 +----- src/pytest_plugins/filler/eip_checklist.py | 13 +++++++++++-- src/pytest_plugins/shared/helpers.py | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/scripts/gen_test_case_reference.py b/docs/scripts/gen_test_case_reference.py index fa0c056e603..d5579a850d2 100644 --- a/docs/scripts/gen_test_case_reference.py +++ b/docs/scripts/gen_test_case_reference.py @@ -45,7 +45,6 @@ "pytest_plugins.filler.gen_test_doc.gen_test_doc", "-p", "pytest_plugins.filler.eip_checklist", - "--collect-only", "--gen-docs", f"--gen-docs-target-fork={TARGET_FORK}", f"--until={GENERATE_UNTIL_FORK}", diff --git a/src/cli/pytest_commands/checklist.py b/src/cli/pytest_commands/checklist.py index 57e874f0df2..47421730388 100644 --- a/src/cli/pytest_commands/checklist.py +++ b/src/cli/pytest_commands/checklist.py @@ -21,9 +21,6 @@ def process_arguments(self, pytest_args: List[str]) -> List[str]: processed_args = super().process_arguments(pytest_args) # Add collect-only flag to avoid running tests - if "--collect-only" not in processed_args: - processed_args.append("--collect-only") - processed_args.extend(["-p", "pytest_plugins.filler.eip_checklist"]) return processed_args @@ -66,8 +63,7 @@ def checklist(output: str, eip: tuple, **kwargs) -> None: """ # Add output directory to pytest args - args = ["-p", "pytest_plugins.filler.eip_checklist"] - args.extend(["--checklist-output", output]) + args = ["--checklist-output", output] # Add EIP filter if specified for eip_num in eip: diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py index df4fa2b37ca..9c4f6016ac6 100644 --- a/src/pytest_plugins/filler/eip_checklist.py +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -227,13 +227,19 @@ def generate_filled_checklist(self, eip: int, output_dir: Path) -> Path: return output_dir + @pytest.hookimpl(tryfirst=True) + def pytest_runtestloop(self, session): + """Skip test execution, only generate checklists.""" + session.testscollected = 0 + return True + def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): """Collect checklist markers during test collection.""" for item in items: eip, eip_path = self.extract_eip_from_path(Path(item.location[0])) if eip_path is not None and eip is not None: self.eip_paths[eip] = eip_path - if item.get_closest_marker("derived_test"): + if item.get_closest_marker("derived_test") or item.get_closest_marker("skip"): continue self.collect_from_item(item, eip) @@ -247,6 +253,9 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes checklist_props = {} + if not self.eip_checklist_items: + pytest.exit("\nNo EIPs found with checklist markers.", returncode=pytest.ExitCode.OK) + # Generate a checklist for each EIP for eip in sorted(self.eip_checklist_items.keys()): # Skip if specific EIPs were requested and this isn't one of them @@ -267,7 +276,7 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes ) else: checklist_path = self.generate_filled_checklist(eip, checklist_output) - print(f"Generated EIP-{eip} checklist: {checklist_path}") + print(f"\nGenerated EIP-{eip} checklist: {checklist_path}") if checklist_doc_gen: config.checklist_props = checklist_props # type: ignore diff --git a/src/pytest_plugins/shared/helpers.py b/src/pytest_plugins/shared/helpers.py index 52cfa6e811d..bbe720b652f 100644 --- a/src/pytest_plugins/shared/helpers.py +++ b/src/pytest_plugins/shared/helpers.py @@ -18,6 +18,8 @@ def is_help_or_collectonly_mode(config: pytest.Config) -> bool: or config.getoption("show_ported_from", default=False) or config.getoption("links_as_filled", default=False) or config.getoption("help", default=False) + or config.pluginmanager.has_plugin("pytest_plugins.filler.eip_checklist") + or config.pluginmanager.has_plugin("pytest_plugins.filler.gen_test_doc.gen_test_doc") ) From 4dee8ba067ca094dea1491ddbc3a7d03fd901876 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Fri, 30 May 2025 23:04:16 +0000 Subject: [PATCH 09/16] feat(plugins/eip_checklist): Allow starts-with covered items, n/a, and externally covered items --- src/pytest_plugins/filler/eip_checklist.py | 380 +++++++++++------- .../filler/tests/test_eip_checklist.py | 21 + 2 files changed, 263 insertions(+), 138 deletions(-) diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py index 9c4f6016ac6..7499cbc0935 100644 --- a/src/pytest_plugins/filler/eip_checklist.py +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -8,10 +8,9 @@ import logging import re -from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Set, Tuple import pytest @@ -50,6 +49,16 @@ def pytest_addoption(parser: pytest.Parser): TITLE_LINE = "# EIP Execution Layer Testing Checklist Template" PERCENTAGE_LINE = "| TOTAL_CHECKLIST_ITEMS | COVERED_CHECKLIST_ITEMS | PERCENTAGE |" +TEMPLATE_PATH = ( + Path(__file__).parents[3] + / "docs" + / "writing_tests" + / "checklist_templates" + / "eip_testing_checklist_template.md" +) +TEMPLATE_CONTENT = TEMPLATE_PATH.read_text() +EXTERNAL_COVERAGE_FILE_NAME = "eip_checklist_external_coverage.txt" +NOT_APPLICABLE_FILE_NAME = "eip_checklist_not_applicable.txt" @pytest.hookimpl(tryfirst=True) @@ -57,78 +66,209 @@ def pytest_configure(config): # noqa: D103 config.pluginmanager.register(EIPChecklistCollector(), "eip-checklist-collector") -@dataclass +@dataclass(kw_only=True) class EIPItem: """Represents an EIP checklist item.""" id: str + line_number: int description: str - tests: List[str] - covered: bool = False + tests: Set[str] + not_applicable_reason: str = "" + external_coverage_reason: str = "" @classmethod - def from_checklist_line(cls, line: str) -> "EIPItem | None": + def from_checklist_line(cls, *, line: str, line_number: int) -> "EIPItem | None": """Create an EIP item from a checklist line.""" match = re.match(r"\|\s*`([^`]+)`\s*\|\s*([^|]+)\s*\|", line) if not match: return None - return cls(id=match.group(1), description=match.group(2), tests=[], covered=False) - - def with_tests(self, tests: List[str]) -> "EIPItem": - """Return a new EIP item with the given tests.""" - new_tests = sorted(set(self.tests + tests)) - return EIPItem( - id=self.id, - description=self.description, - tests=new_tests, - covered=len(new_tests) > 0, + return cls( + id=match.group(1), + line_number=line_number, + description=match.group(2), + tests=set(), ) + @property + def covered(self) -> bool: + """Return True if the item is covered by at least one test.""" + return len(self.tests) > 0 or self.external_coverage + + @property + def external_coverage(self) -> bool: + """Return True if the item is covered by an external test/procedure.""" + return self.external_coverage_reason != "" + + @property + def not_applicable(self) -> bool: + """Return True if the item is not applicable.""" + return self.not_applicable_reason != "" + def __str__(self) -> str: """Return a string representation of the EIP item.""" - return ( - f"| `{self.id}` " - f"| {self.description} " - f"| {'โœ…' if self.covered else ' '} " - f"| {', '.join(self.tests)} " - "|" + status = " " + tests = "" + if self.external_coverage: + status = "โœ…" + tests = self.external_coverage_reason + elif self.covered: + status = "โœ…" + tests = ", ".join(sorted(self.tests)) + elif self.not_applicable: + status = "N/A" + tests = self.not_applicable_reason + + return f"| `{self.id}` | {self.description} | {status} | {tests} |" + + +TEMPLATE_ITEMS: Dict[str, EIPItem] = {} +# Parse the template to extract checklist item IDs and descriptions +for i, line in enumerate(TEMPLATE_CONTENT.splitlines()): + # Match lines that contain checklist items with IDs in backticks + if item := EIPItem.from_checklist_line(line=line, line_number=i + 1): + TEMPLATE_ITEMS[item.id] = item + +ALL_IDS = set(TEMPLATE_ITEMS.keys()) + + +def resolve_id(item_id: str) -> Set[str]: + """Resolve an item ID to a set of checklist IDs.""" + covered_ids = {checklist_id for checklist_id in ALL_IDS if checklist_id.startswith(item_id)} + return covered_ids + + +@dataclass(kw_only=True) +class EIP: + """Represents an EIP and its checklist.""" + + number: int + items: Dict[str, EIPItem] = field(default_factory=TEMPLATE_ITEMS.copy) + path: Path | None = None + + def add_covered_test(self, checklist_id: str, node_id: str) -> None: + """Add a covered test to the EIP.""" + self.items[checklist_id].tests.add(node_id) + + @property + def covered_items(self) -> int: + """Return the number of covered items.""" + return sum(1 for item in self.items.values() if item.covered) + + @property + def total_items(self) -> int: + """Return the number of total items.""" + return sum(1 for item in self.items.values() if not item.not_applicable) + + @property + def percentage(self) -> float: + """Return the percentage of covered items.""" + return self.covered_items / self.total_items * 100 if self.total_items else 0 + + @property + def completness_emoji(self) -> str: + """Return the completness emoji.""" + return "๐ŸŸข" if self.percentage == 100 else "๐ŸŸก" if self.percentage > 50 else "๐Ÿ”ด" + + def mark_not_applicable(self): + """Read the not-applicable items from the EIP.""" + if self.path is None: + return + not_applicable_path = self.path / NOT_APPLICABLE_FILE_NAME + if not not_applicable_path.exists(): + return + with not_applicable_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + assert "=" in line + item_id, reason = line.split("=", 1) + item_id = item_id.strip() + reason = reason.strip() + assert reason, f"Reason is empty for {line}" + assert item_id, f"Item ID is empty for {line}" + ids = resolve_id(item_id) + if not ids: + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for EIP {self.number}" + ) + continue + for id_covered in ids: + self.items[id_covered].not_applicable_reason = reason + + def mark_external_coverage(self): + """Read the externally covered items from the EIP.""" + if self.path is None: + return + external_coverage_path = self.path / EXTERNAL_COVERAGE_FILE_NAME + if not external_coverage_path.exists(): + return + with external_coverage_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + assert "=" in line + item_id, reason = line.split("=", 1) + item_id = item_id.strip() + reason = reason.strip() + assert item_id, f"Item ID is empty for {line}" + assert reason, f"Reason is empty for {line}" + ids = resolve_id(item_id) + if not ids: + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for EIP {self.number}" + ) + continue + for id_covered in ids: + self.items[id_covered].external_coverage_reason = reason + + def generate_filled_checklist_lines(self) -> List[str]: + """Generate the filled checklist lines for a specific EIP.""" + # Create a copy of the template content + lines = TEMPLATE_CONTENT.splitlines() + + self.mark_not_applicable() + self.mark_external_coverage() + + for checklist_item in self.items.values(): + # Find the line with this item ID + lines[checklist_item.line_number - 1] = str(checklist_item) + + lines[lines.index(PERCENTAGE_LINE)] = ( + f"| {self.total_items} | {self.covered_items} | {self.completness_emoji} " + f"{self.percentage:.2f}% |" ) + # Replace the title line with the EIP number + lines[lines.index(TITLE_LINE)] = f"# EIP-{self.number} Test Checklist" + + return lines + + def generate_filled_checklist(self, output_dir: Path) -> Path: + """Generate a filled checklist for a specific EIP.""" + lines = self.generate_filled_checklist_lines() + + output_dir = output_dir / f"eip{self.number}_checklist.md" + + # Write the filled checklist + output_dir.parent.mkdir(exist_ok=True, parents=True) + output_dir.write_text("\n".join(lines)) + + return output_dir + class EIPChecklistCollector: """Collects and manages EIP checklist items from test markers.""" def __init__(self: "EIPChecklistCollector"): """Initialize the EIP checklist collector.""" - self.eip_checklist_items: Dict[int, Dict[str, Set[Tuple[str, str]]]] = defaultdict( - lambda: defaultdict(set) - ) - self.template_path = ( - Path(__file__).parents[3] - / "docs" - / "writing_tests" - / "checklist_templates" - / "eip_testing_checklist_template.md" - ) - self.eip_paths: Dict[int, Path] = {} - - if not self.template_path.exists(): - pytest.fail(f"EIP checklist template not found at {self.template_path}") - - self.template_content = self.template_path.read_text() - self.template_items: Dict[str, Tuple[EIPItem, int]] = {} # ID -> (item, line_number) - self.all_ids: Set[str] = set() + self.eips: Dict[int, EIP] = {} - # Parse the template to extract checklist item IDs and descriptions - lines = self.template_content.splitlines() - for i, line in enumerate(lines): - # Match lines that contain checklist items with IDs in backticks - item = EIPItem.from_checklist_line(line) - if item: - self.template_items[item.id] = (item, i + 1) - self.all_ids.add(item.id) - - def extract_eip_from_path(self, test_path: Path) -> Tuple[Optional[int], Optional[Path]]: + def extract_eip_from_path(self, test_path: Path) -> Tuple[int | None, Path | None]: """Extract EIP number from test file path.""" # Look for patterns like eip1234_ or eip1234/ in the path for part_idx, part in enumerate(test_path.parts): @@ -139,7 +279,33 @@ def extract_eip_from_path(self, test_path: Path) -> Tuple[Optional[int], Optiona return eip, eip_path return None, None - def collect_from_item(self, item: pytest.Item, primary_eip: Optional[int]) -> None: + def get_eip_from_item(self, item: pytest.Item) -> EIP | None: + """Get the EIP for a test item.""" + test_path = Path(item.location[0]) + for part_idx, part in enumerate(test_path.parts): + match = re.match(r"eip(\d+)", part) + if match: + eip = int(match.group(1)) + if eip not in self.eips: + self.eips[eip] = EIP( + number=eip, + path=test_path.parents[len(test_path.parents) - part_idx - 2], + ) + else: + if self.eips[eip].path is None: + self.eips[eip].path = test_path.parents[ + len(test_path.parents) - part_idx - 2 + ] + return self.eips[eip] + return None + + def get_eip(self, eip: int) -> EIP: + """Get the EIP for a given EIP number.""" + if eip not in self.eips: + self.eips[eip] = EIP(number=eip, path=None) + return self.eips[eip] + + def collect_from_item(self, item: pytest.Item, primary_eip: EIP | None) -> None: """Collect checklist markers from a test item.""" for marker in item.iter_markers("eip_checklist"): if not marker.args: @@ -151,81 +317,27 @@ def collect_from_item(self, item: pytest.Item, primary_eip: Optional[int]) -> No if not isinstance(additional_eips, list): additional_eips = [additional_eips] - # Get the primary EIP from the test path - if primary_eip is None: - if not additional_eips: + eips: List[EIP] = [primary_eip] if primary_eip else [] + + if additional_eips: + if any(not isinstance(eip, int) for eip in additional_eips): pytest.fail( - f"Could not extract EIP number from test path: {item.path}. " - "Marker 'eip_checklist' can only be used in tests that are located in a " - "directory named after the EIP number, or with the eip keyword argument." + "EIP numbers must be integers. Found non-integer EIPs in " + f"{item.nodeid}: {additional_eips}" ) - eips = additional_eips - else: - eips = [primary_eip] + additional_eips - - if any(not isinstance(eip, int) for eip in eips): - pytest.fail( - "EIP numbers must be integers. Found non-integer EIPs in " - f"{item.nodeid}: {eips}" - ) + eips += [self.get_eip(eip) for eip in additional_eips] for item_id in marker.args: - if item_id not in self.all_ids: - # TODO: If we decide to do the starts-with matching, we have to change this. + covered_ids = resolve_id(item_id.strip()) + if not covered_ids: logger.warning( f"Item ID {item_id} not found in the checklist template, " f"for test {item.nodeid}" ) continue - for eip in eips: - self.eip_checklist_items[eip][item_id].add((item.nodeid, item.name)) - - def generate_filled_checklist_lines(self, eip: int) -> List[str]: - """Generate the filled checklist lines for a specific EIP.""" - # Get all checklist items for this EIP - eip_items = self.eip_checklist_items.get(eip, {}) - - # Create a copy of the template content - filled_content = self.template_content - lines = filled_content.splitlines() - - # Process each line in reverse order to avoid index shifting - total_items = len(self.template_items) - covered_items = 0 - for item_id, (checklist_item, line_num) in sorted( - self.template_items.items(), key=lambda x: x[1][1], reverse=True - ): - if item_id in eip_items: - # Find the line with this item ID - line_idx = line_num - 1 - checklist_item = checklist_item.with_tests( - [f"`{test_node_id}`" for test_node_id, _ in eip_items[item_id]] - ) - lines[line_idx] = str(checklist_item) - covered_items += 1 - - percentage = covered_items / total_items * 100 - completness_emoji = "๐ŸŸข" if percentage == 100 else "๐ŸŸก" if percentage > 50 else "๐Ÿ”ด" - lines[lines.index(PERCENTAGE_LINE)] = ( - f"| {total_items} | {covered_items} | {completness_emoji} {percentage:.2f}% |" - ) - - # Replace the title line with the EIP number - lines[lines.index(TITLE_LINE)] = f"# EIP-{eip} Test Checklist" - - return lines - - def generate_filled_checklist(self, eip: int, output_dir: Path) -> Path: - """Generate a filled checklist for a specific EIP.""" - lines = self.generate_filled_checklist_lines(eip) - - output_dir = output_dir / f"eip{eip}_checklist.md" - - # Write the filled checklist - output_dir.parent.mkdir(exist_ok=True, parents=True) - output_dir.write_text("\n".join(lines)) - - return output_dir + for id_covered in covered_ids: + for eip in eips: + eip.add_covered_test(id_covered, item.nodeid) @pytest.hookimpl(tryfirst=True) def pytest_runtestloop(self, session): @@ -236,47 +348,39 @@ def pytest_runtestloop(self, session): def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): """Collect checklist markers during test collection.""" for item in items: - eip, eip_path = self.extract_eip_from_path(Path(item.location[0])) - if eip_path is not None and eip is not None: - self.eip_paths[eip] = eip_path + eip = self.get_eip_from_item(item) if item.get_closest_marker("derived_test") or item.get_closest_marker("skip"): continue self.collect_from_item(item, eip) - if not self.eip_checklist_items: - return - # Check which mode we are in checklist_doc_gen = config.getoption("checklist_doc_gen", False) checklist_output = config.getoption("checklist_output", Path("checklists")) checklist_eips = config.getoption("checklist_eips", []) checklist_props = {} - - if not self.eip_checklist_items: - pytest.exit("\nNo EIPs found with checklist markers.", returncode=pytest.ExitCode.OK) - # Generate a checklist for each EIP - for eip in sorted(self.eip_checklist_items.keys()): + for eip in self.eips.values(): # Skip if specific EIPs were requested and this isn't one of them - if checklist_eips and eip not in checklist_eips: + if checklist_eips and eip.number not in checklist_eips: continue if checklist_doc_gen: - eip_path = self.eip_paths[eip] - checklist_props[eip_path / "checklist.md"] = EipChecklistPageProps( - title=f"EIP-{eip} Test Checklist", + path = eip.path + assert path is not None + checklist_props[path] = EipChecklistPageProps( + title=f"EIP-{eip.number} Test Checklist", source_code_url="", target_or_valid_fork="mainnet", - path=eip_path / "checklist.md", + path=path, pytest_node_id="", package_name="eip_checklist", - eip=eip, - lines=self.generate_filled_checklist_lines(eip), + eip=eip.number, + lines=eip.generate_filled_checklist_lines(), ) else: - checklist_path = self.generate_filled_checklist(eip, checklist_output) - print(f"\nGenerated EIP-{eip} checklist: {checklist_path}") + checklist_path = eip.generate_filled_checklist(checklist_output) + print(f"\nGenerated EIP-{eip.number} checklist: {checklist_path}") if checklist_doc_gen: config.checklist_props = checklist_props # type: ignore diff --git a/src/pytest_plugins/filler/tests/test_eip_checklist.py b/src/pytest_plugins/filler/tests/test_eip_checklist.py index 13bbfcb436a..29511751128 100644 --- a/src/pytest_plugins/filler/tests/test_eip_checklist.py +++ b/src/pytest_plugins/filler/tests/test_eip_checklist.py @@ -37,6 +37,16 @@ def test_invalid_v(state_test: StateTestFiller): """ ) ) + eip_7702_external_coverage_file = eip_7702_tests_dir.join( + "eip_checklist_external_coverage.txt" + ) + eip_7702_external_coverage_file.write( + textwrap.dedent( + """ + general/code_coverage/eels = DEBUG EXTERNAL COVERAGE REASON + """ + ) + ) berlin_tests_dir = tests_dir.mkdir("berlin") eip_2930_tests_dir = berlin_tests_dir.mkdir("eip2930_set_code_tx") @@ -56,6 +66,14 @@ def test_berlin_one(state_test: StateTestFiller): """ ) ) + test_2930_n_a_file = eip_2930_tests_dir.join("eip_checklist_not_applicable.txt") + test_2930_n_a_file.write( + textwrap.dedent( + """ + new_system_contract = DEBUG NOT APPLICABLE REASON + """ + ) + ) # Run pytest with checklist-only mode testdir.copy_example(name="pytest.ini") result = testdir.runpytest( @@ -84,9 +102,12 @@ def test_berlin_one(state_test: StateTestFiller): assert "โœ…" in content assert "test_exact_gas" in content assert "test_invalid_v" in content + assert "DEBUG EXTERNAL COVERAGE REASON" in content checklist_file = checklist_dir / "eip2930_checklist.md" assert checklist_file.exists() content = checklist_file.read() assert "โœ…" in content assert "test_invalid_v" in content + assert "N/A" in content + assert "DEBUG NOT APPLICABLE REASON" in content From 4e024da758628e1e40744d72d316dd4a20cde3b2 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Fri, 30 May 2025 23:10:58 +0000 Subject: [PATCH 10/16] Update docs page --- docs/filling_tests/eip_checklist.md | 145 +++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 4 deletions(-) diff --git a/docs/filling_tests/eip_checklist.md b/docs/filling_tests/eip_checklist.md index 65438e695c2..557c9c30fc9 100644 --- a/docs/filling_tests/eip_checklist.md +++ b/docs/filling_tests/eip_checklist.md @@ -35,6 +35,18 @@ def test_invalid_signature(state_test: StateTestFiller): pass ``` +### Partial ID Matching + +You can use partial IDs that will match all checklist items starting with that prefix: + +```python +# This will mark all items under "new_transaction_type/test/signature/invalid/" as covered +@pytest.mark.eip_checklist("new_transaction_type/test/signature/invalid/") +def test_all_invalid_signatures(state_test: StateTestFiller): + """Test covering all invalid signature scenarios.""" + pass +``` + ## Generating Checklists ### Using the Dedicated `checklist` Command @@ -59,11 +71,136 @@ checklist --eip 7702 --eip 2930 When building the documentation with `mkdocs`, checklists are automatically generated for all EIPs that have tests with checklist markers. The checklists appear in the test documentation alongside the test modules. +## External Coverage and Not Applicable Items + +### External Coverage + +For checklist items that are covered by external tests, procedures, or tools (e.g., EELS coverage), create a file named `eip_checklist_external_coverage.txt` in the EIP test directory: + +```text +# tests/prague/eip7702_set_code_tx/eip_checklist_external_coverage.txt +general/code_coverage/eels = Covered by EELS test suite +general/code_coverage/second_client = Covered by Nethermind tests +``` + +Format: `checklist_item_id = reason` + +### Not Applicable Items + +For checklist items that are not applicable to a specific EIP, create a file named `eip_checklist_not_applicable.txt` in the EIP test directory: + +```text +# tests/prague/eip7702_set_code_tx/eip_checklist_not_applicable.txt +new_system_contract = EIP-7702 does not introduce a system contract +new_precompile = EIP-7702 does not introduce a precompile +``` + +Format: `checklist_item_id = reason` + +Both files support partial ID matching, so you can mark entire sections as not applicable: + +```text +# Mark all system contract items as not applicable +new_system_contract/ = EIP does not introduce system contracts +``` + ## Output Format The generated checklist will show: -- โœ… for completed items -- Test names that implement each item (up to 3, with "..." if more) -- Empty cells for uncompleted items -- A percentage of covered checklist items +- โœ… for completed items (either by tests or external coverage) +- N/A for not applicable items +- Test names that implement each item +- External coverage reasons where applicable +- A percentage of covered checklist items (excluding N/A items) +- Color-coded completion status: ๐ŸŸข (100%), ๐ŸŸก (>50%), ๐Ÿ”ด (โ‰ค50%) + +Example output snippet: + +```markdown +# EIP-7702 Test Checklist + +## Checklist Progress Tracker + +| Total Checklist Items | Covered Checklist Items | Percentage | +| --------------------- | ----------------------- | ---------- | +| 45 | 32 | ๐ŸŸก 71.11% | + +## General + +#### Code coverage + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `general/code_coverage/eels` | Run produced tests against EELS... | โœ… | Covered by EELS test suite | +| `general/code_coverage/test_coverage` | Run coverage on the test code itself... | โœ… | `tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_txs` | + +## New Transaction Type + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `new_transaction_type/test/intrinsic_validity/gas_limit/exact` | Provide the exact intrinsic gas... | โœ… | `tests/prague/eip7702_set_code_tx/test_checklist_example.py::test_exact_intrinsic_gas` | +| `new_transaction_type/test/intrinsic_validity/gas_limit/insufficient` | Provide the exact intrinsic gas minus one... | | | + +## New System Contract + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `new_system_contract/test/deployment/missing` | Verify block execution behavior... | N/A | EIP-7702 does not introduce a system contract | +``` + +## Best Practices + +1. **Start with the checklist**: Review the checklist template before writing tests to ensure comprehensive coverage +2. **Use descriptive test names**: The test name will appear in the checklist, so make it clear what the test covers +3. **Mark items as you go**: Add `eip_checklist` markers while writing tests, not as an afterthought +4. **Document external coverage**: If items are covered by external tools/tests, document this in `eip_checklist_external_coverage.txt` +5. **Be explicit about N/A items**: Document why items are not applicable in `eip_checklist_not_applicable.txt` +6. **Use partial IDs wisely**: When a test covers multiple related items, use partial IDs to mark them all + +## Workflow Example + +1. **Create test directory structure**: + + ```bash + tests/prague/eip9999_new_feature/ + โ”œโ”€โ”€ __init__.py + โ”œโ”€โ”€ spec.py + โ”œโ”€โ”€ test_basic.py + โ”œโ”€โ”€ eip_checklist_external_coverage.txt + โ””โ”€โ”€ eip_checklist_not_applicable.txt + ``` + +2. **Mark tests as you implement them**: + + ```python + @pytest.mark.eip_checklist("new_opcode/test/gas_usage/normal") + def test_opcode_gas_consumption(state_test: StateTestFiller): + """Test normal gas consumption of the new opcode.""" + pass + ``` + +3. **Document external coverage**: + + ```text + # eip_checklist_external_coverage.txt + general/code_coverage/eels = Covered by ethereum/execution-specs PR #1234 + ``` + +4. **Mark non-applicable items**: + + ```text + # eip_checklist_not_applicable.txt + new_precompile/ = EIP-9999 introduces an opcode, not a precompile + ``` + +5. **Generate and review checklist**: + + ```bash + checklist --eip 9999 + # Review the generated checklist for completeness + ``` + +## See Also + +- [EIP Testing Checklist Template](../writing_tests/checklist_templates/eip_testing_checklist_template.md) - The full checklist template From 37cea2d1f880f3ba25d0d8485b54fb9b413ce692 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Mon, 2 Jun 2025 18:27:44 +0000 Subject: [PATCH 11/16] fix correct path --- src/pytest_plugins/filler/eip_checklist.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py index 7499cbc0935..d395c7e0a50 100644 --- a/src/pytest_plugins/filler/eip_checklist.py +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -366,15 +366,15 @@ def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytes continue if checklist_doc_gen: - path = eip.path - assert path is not None - checklist_props[path] = EipChecklistPageProps( + assert eip.path is not None + checklist_path = eip.path / "checklist.md" + checklist_props[checklist_path] = EipChecklistPageProps( title=f"EIP-{eip.number} Test Checklist", source_code_url="", target_or_valid_fork="mainnet", - path=path, + path=checklist_path, pytest_node_id="", - package_name="eip_checklist", + package_name="checklist", eip=eip.number, lines=eip.generate_filled_checklist_lines(), ) From ea3b8195bdf7ef2ec11c961e28c43551f9c2ae95 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Mon, 2 Jun 2025 20:58:31 +0000 Subject: [PATCH 12/16] docs: Move checklist guide to `writing_tests` --- docs/navigation.md | 1 + docs/{filling_tests => writing_tests}/eip_checklist.md | 0 2 files changed, 1 insertion(+) rename docs/{filling_tests => writing_tests}/eip_checklist.md (100%) diff --git a/docs/navigation.md b/docs/navigation.md index 379b206f4e3..0fb2867622b 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -21,6 +21,7 @@ * [Exception Tests](writing_tests/exception_tests.md) * [Using and Extending Fork Methods](writing_tests/fork_methods.md) * [Referencing an EIP Spec Version](writing_tests/reference_specification.md) + * [EIP Checklist Generation](writing_tests/eip_checklist.md) * [Testing Checklist Templates](writing_tests/checklist_templates/index.md) * [EIP Execution Layer Testing Checklist Template](writing_tests/checklist_templates/eip_testing_checklist_template.md) * [Post-mortems](writing_tests/post_mortems.md) diff --git a/docs/filling_tests/eip_checklist.md b/docs/writing_tests/eip_checklist.md similarity index 100% rename from docs/filling_tests/eip_checklist.md rename to docs/writing_tests/eip_checklist.md From 57323d0f407adaf56adc85a94b20daea78aeb20a Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Mon, 2 Jun 2025 20:59:56 +0000 Subject: [PATCH 13/16] fix --- docs/writing_tests/eip_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_tests/eip_checklist.md b/docs/writing_tests/eip_checklist.md index 557c9c30fc9..8562b90aedd 100644 --- a/docs/writing_tests/eip_checklist.md +++ b/docs/writing_tests/eip_checklist.md @@ -203,4 +203,4 @@ Example output snippet: ## See Also -- [EIP Testing Checklist Template](../writing_tests/checklist_templates/eip_testing_checklist_template.md) - The full checklist template +- [EIP Testing Checklist Template](./checklist_templates/eip_testing_checklist_template.md) - The full checklist template From bc4d7ddbf755037f9ce9d047be7597f06414733a Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 4 Jun 2025 16:19:09 +0200 Subject: [PATCH 14/16] docs: prefix commands with `uv run` --- docs/writing_tests/eip_checklist.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/writing_tests/eip_checklist.md b/docs/writing_tests/eip_checklist.md index 8562b90aedd..0aabc6e522a 100644 --- a/docs/writing_tests/eip_checklist.md +++ b/docs/writing_tests/eip_checklist.md @@ -55,16 +55,16 @@ To generate only checklists without filling fixtures: ```bash # Generate checklists for all EIPs -checklist +uv run checklist # Generate checklist for specific EIP -checklist --eip 7702 +uv run checklist --eip 7702 # Specify output directory -checklist --output ./my-checklists +uv run checklist --output ./my-checklists # Multiple EIPs -checklist --eip 7702 --eip 2930 +uv run checklist --eip 7702 --eip 2930 ``` ### Automatic Generation in Documentation From 5509cc688b21984fecbc4018b3693db98709155f Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 4 Jun 2025 16:21:17 +0200 Subject: [PATCH 15/16] chore(clis): fix examples in `checklist` docstring --- src/cli/pytest_commands/checklist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli/pytest_commands/checklist.py b/src/cli/pytest_commands/checklist.py index 47421730388..d53406201e3 100644 --- a/src/cli/pytest_commands/checklist.py +++ b/src/cli/pytest_commands/checklist.py @@ -50,16 +50,16 @@ def checklist(output: str, eip: tuple, **kwargs) -> None: Examples: # Generate checklists for all EIPs - eest checklist + uv run checklist # Generate checklist for specific EIP - eest checklist --eip 7702 + uv run checklist --eip 7702 # Generate checklists for specific test path - eest checklist tests/prague/eip7702* + uv run checklist tests/prague/eip7702* # Specify output directory - eest checklist --output ./my-checklists + uv run checklist --output ./my-checklists """ # Add output directory to pytest args From cefbf0e3ed101703dc195ee4d30be1b52e9d23c6 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Wed, 4 Jun 2025 16:22:33 +0200 Subject: [PATCH 16/16] docs: apply correct indentation to fix ordered-list ennumeration --- docs/writing_tests/eip_checklist.md | 52 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/writing_tests/eip_checklist.md b/docs/writing_tests/eip_checklist.md index 0aabc6e522a..888af3465bd 100644 --- a/docs/writing_tests/eip_checklist.md +++ b/docs/writing_tests/eip_checklist.md @@ -162,44 +162,44 @@ Example output snippet: 1. **Create test directory structure**: - ```bash - tests/prague/eip9999_new_feature/ - โ”œโ”€โ”€ __init__.py - โ”œโ”€โ”€ spec.py - โ”œโ”€โ”€ test_basic.py - โ”œโ”€โ”€ eip_checklist_external_coverage.txt - โ””โ”€โ”€ eip_checklist_not_applicable.txt - ``` + ```bash + tests/prague/eip9999_new_feature/ + โ”œโ”€โ”€ __init__.py + โ”œโ”€โ”€ spec.py + โ”œโ”€โ”€ test_basic.py + โ”œโ”€โ”€ eip_checklist_external_coverage.txt + โ””โ”€โ”€ eip_checklist_not_applicable.txt + ``` 2. **Mark tests as you implement them**: - ```python - @pytest.mark.eip_checklist("new_opcode/test/gas_usage/normal") - def test_opcode_gas_consumption(state_test: StateTestFiller): - """Test normal gas consumption of the new opcode.""" - pass - ``` + ```python + @pytest.mark.eip_checklist("new_opcode/test/gas_usage/normal") + def test_opcode_gas_consumption(state_test: StateTestFiller): + """Test normal gas consumption of the new opcode.""" + pass + ``` 3. **Document external coverage**: - ```text - # eip_checklist_external_coverage.txt - general/code_coverage/eels = Covered by ethereum/execution-specs PR #1234 - ``` + ```text + # eip_checklist_external_coverage.txt + general/code_coverage/eels = Covered by ethereum/execution-specs PR #1234 + ``` 4. **Mark non-applicable items**: - ```text - # eip_checklist_not_applicable.txt - new_precompile/ = EIP-9999 introduces an opcode, not a precompile - ``` + ```text + # eip_checklist_not_applicable.txt + new_precompile/ = EIP-9999 introduces an opcode, not a precompile + ``` 5. **Generate and review checklist**: - ```bash - checklist --eip 9999 - # Review the generated checklist for completeness - ``` + ```bash + checklist --eip 9999 + # Review the generated checklist for completeness + ``` ## See Also