diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 371b260da10..5a54ce07cdd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -53,6 +53,7 @@ Users can select any of the artifacts depending on their testing needs for their - ๐Ÿž Fix bug in ported-from plugin and coverage script that made PRs fail with modified tests that contained no ported tests ([#1661](https://github.com/ethereum/execution-spec-tests/pull/1661)). - ๐Ÿ”€ Refactor the `click`-based CLI interface used for pytest-based commands (`fill`, `execute`, `consume`) to make them more extensible ([#1654](https://github.com/ethereum/execution-spec-tests/pull/1654)). - ๐Ÿ”€ Split `src/ethereum_test_types/types.py` into several files to improve code organization ([#1665](https://github.com/ethereum/execution-spec-tests/pull/1665)). +- โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679)). ### ๐Ÿงช Test Cases diff --git a/docs/navigation.md b/docs/navigation.md index 379b206f4e3..0fb2867622b 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -21,6 +21,7 @@ * [Exception Tests](writing_tests/exception_tests.md) * [Using and Extending Fork Methods](writing_tests/fork_methods.md) * [Referencing an EIP Spec Version](writing_tests/reference_specification.md) + * [EIP Checklist Generation](writing_tests/eip_checklist.md) * [Testing Checklist Templates](writing_tests/checklist_templates/index.md) * [EIP Execution Layer Testing Checklist Template](writing_tests/checklist_templates/eip_testing_checklist_template.md) * [Post-mortems](writing_tests/post_mortems.md) diff --git a/docs/scripts/gen_test_case_reference.py b/docs/scripts/gen_test_case_reference.py index b3f43ae901d..d5579a850d2 100644 --- a/docs/scripts/gen_test_case_reference.py +++ b/docs/scripts/gen_test_case_reference.py @@ -43,9 +43,12 @@ "filterwarnings=ignore::pytest.PytestAssertRewriteWarning", # suppress warnings due to reload "-p", "pytest_plugins.filler.gen_test_doc.gen_test_doc", + "-p", + "pytest_plugins.filler.eip_checklist", "--gen-docs", f"--gen-docs-target-fork={TARGET_FORK}", f"--until={GENERATE_UNTIL_FORK}", + "--checklist-doc-gen", "--skip-index", "-m", "not blockchain_test_engine", diff --git a/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md b/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md index bd63ee657c2..04751ebe2c9 100644 --- a/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md +++ b/docs/writing_tests/checklist_templates/eip_testing_checklist_template.md @@ -3,6 +3,12 @@ Depending on the changes introduced by an EIP, the following template is the minimum baseline to guarantee test coverage of the Execution Layer features. +## Checklist Progress Tracker + +| Total Checklist Items | Covered Checklist Items | Percentage | +| --------------------- | ----------------------- | ---------- | +| TOTAL_CHECKLIST_ITEMS | COVERED_CHECKLIST_ITEMS | PERCENTAGE | + ## General #### Code coverage diff --git a/docs/writing_tests/eip_checklist.md b/docs/writing_tests/eip_checklist.md new file mode 100644 index 00000000000..888af3465bd --- /dev/null +++ b/docs/writing_tests/eip_checklist.md @@ -0,0 +1,206 @@ +# EIP Checklist Generation + +The EIP checklist feature helps track test coverage for EIP implementations by automatically generating filled checklists based on test markers. + +## Overview + +When implementing tests for an EIP, you can mark specific tests as covering checklist items from the [EIP testing checklist template](../writing_tests/checklist_templates/eip_testing_checklist_template.md). The framework will then generate a filled checklist showing which items have been implemented. + +## Using the `pytest.mark.eip_checklist` Marker + +To mark a test as implementing a specific checklist item: + +```python +import pytest +from ethereum_test_tools import StateTestFiller + +@pytest.mark.eip_checklist("new_transaction_type/test/intrinsic_validity/gas_limit/exact") +def test_exact_intrinsic_gas(state_test: StateTestFiller): + """Test transaction with exact intrinsic gas limit.""" + # Test implementation + pass +``` + +### Marker Parameters + +- **First positional parameter** (required): The checklist item ID from the template +- **`eip` keyword parameter** (optional): List of additional EIPs covered by the test + +Example with multiple EIPs covered by the same test: + +```python +@pytest.mark.eip_checklist("new_transaction_type/test/signature/invalid/v/0", eip=[7702, 2930]) +def test_invalid_signature(state_test: StateTestFiller): + """Test invalid signature that affects multiple EIPs.""" + pass +``` + +### Partial ID Matching + +You can use partial IDs that will match all checklist items starting with that prefix: + +```python +# This will mark all items under "new_transaction_type/test/signature/invalid/" as covered +@pytest.mark.eip_checklist("new_transaction_type/test/signature/invalid/") +def test_all_invalid_signatures(state_test: StateTestFiller): + """Test covering all invalid signature scenarios.""" + pass +``` + +## Generating Checklists + +### Using the Dedicated `checklist` Command + +To generate only checklists without filling fixtures: + +```bash +# Generate checklists for all EIPs +uv run checklist + +# Generate checklist for specific EIP +uv run checklist --eip 7702 + +# Specify output directory +uv run checklist --output ./my-checklists + +# Multiple EIPs +uv run checklist --eip 7702 --eip 2930 +``` + +### Automatic Generation in Documentation + +When building the documentation with `mkdocs`, checklists are automatically generated for all EIPs that have tests with checklist markers. The checklists appear in the test documentation alongside the test modules. + +## External Coverage and Not Applicable Items + +### External Coverage + +For checklist items that are covered by external tests, procedures, or tools (e.g., EELS coverage), create a file named `eip_checklist_external_coverage.txt` in the EIP test directory: + +```text +# tests/prague/eip7702_set_code_tx/eip_checklist_external_coverage.txt +general/code_coverage/eels = Covered by EELS test suite +general/code_coverage/second_client = Covered by Nethermind tests +``` + +Format: `checklist_item_id = reason` + +### Not Applicable Items + +For checklist items that are not applicable to a specific EIP, create a file named `eip_checklist_not_applicable.txt` in the EIP test directory: + +```text +# tests/prague/eip7702_set_code_tx/eip_checklist_not_applicable.txt +new_system_contract = EIP-7702 does not introduce a system contract +new_precompile = EIP-7702 does not introduce a precompile +``` + +Format: `checklist_item_id = reason` + +Both files support partial ID matching, so you can mark entire sections as not applicable: + +```text +# Mark all system contract items as not applicable +new_system_contract/ = EIP does not introduce system contracts +``` + +## Output Format + +The generated checklist will show: + +- โœ… for completed items (either by tests or external coverage) +- N/A for not applicable items +- Test names that implement each item +- External coverage reasons where applicable +- A percentage of covered checklist items (excluding N/A items) +- Color-coded completion status: ๐ŸŸข (100%), ๐ŸŸก (>50%), ๐Ÿ”ด (โ‰ค50%) + +Example output snippet: + +```markdown +# EIP-7702 Test Checklist + +## Checklist Progress Tracker + +| Total Checklist Items | Covered Checklist Items | Percentage | +| --------------------- | ----------------------- | ---------- | +| 45 | 32 | ๐ŸŸก 71.11% | + +## General + +#### Code coverage + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `general/code_coverage/eels` | Run produced tests against EELS... | โœ… | Covered by EELS test suite | +| `general/code_coverage/test_coverage` | Run coverage on the test code itself... | โœ… | `tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_txs` | + +## New Transaction Type + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `new_transaction_type/test/intrinsic_validity/gas_limit/exact` | Provide the exact intrinsic gas... | โœ… | `tests/prague/eip7702_set_code_tx/test_checklist_example.py::test_exact_intrinsic_gas` | +| `new_transaction_type/test/intrinsic_validity/gas_limit/insufficient` | Provide the exact intrinsic gas minus one... | | | + +## New System Contract + +| ID | Description | Status | Tests | +| -- | ----------- | ------ | ----- | +| `new_system_contract/test/deployment/missing` | Verify block execution behavior... | N/A | EIP-7702 does not introduce a system contract | +``` + +## Best Practices + +1. **Start with the checklist**: Review the checklist template before writing tests to ensure comprehensive coverage +2. **Use descriptive test names**: The test name will appear in the checklist, so make it clear what the test covers +3. **Mark items as you go**: Add `eip_checklist` markers while writing tests, not as an afterthought +4. **Document external coverage**: If items are covered by external tools/tests, document this in `eip_checklist_external_coverage.txt` +5. **Be explicit about N/A items**: Document why items are not applicable in `eip_checklist_not_applicable.txt` +6. **Use partial IDs wisely**: When a test covers multiple related items, use partial IDs to mark them all + +## Workflow Example + +1. **Create test directory structure**: + + ```bash + tests/prague/eip9999_new_feature/ + โ”œโ”€โ”€ __init__.py + โ”œโ”€โ”€ spec.py + โ”œโ”€โ”€ test_basic.py + โ”œโ”€โ”€ eip_checklist_external_coverage.txt + โ””โ”€โ”€ eip_checklist_not_applicable.txt + ``` + +2. **Mark tests as you implement them**: + + ```python + @pytest.mark.eip_checklist("new_opcode/test/gas_usage/normal") + def test_opcode_gas_consumption(state_test: StateTestFiller): + """Test normal gas consumption of the new opcode.""" + pass + ``` + +3. **Document external coverage**: + + ```text + # eip_checklist_external_coverage.txt + general/code_coverage/eels = Covered by ethereum/execution-specs PR #1234 + ``` + +4. **Mark non-applicable items**: + + ```text + # eip_checklist_not_applicable.txt + new_precompile/ = EIP-9999 introduces an opcode, not a precompile + ``` + +5. **Generate and review checklist**: + + ```bash + checklist --eip 9999 + # Review the generated checklist for completeness + ``` + +## See Also + +- [EIP Testing Checklist Template](./checklist_templates/eip_testing_checklist_template.md) - The full checklist template diff --git a/pyproject.toml b/pyproject.toml index 39ec9da6042..d1d7dfa2e7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ checkfixtures = "cli.check_fixtures:check_fixtures" check_eip_versions = "cli.pytest_commands.check_eip_versions:check_eip_versions" consume = "cli.pytest_commands.consume:consume" protec = "cli.pytest_commands.consume:consume" +checklist = "cli.pytest_commands.checklist:checklist" genindex = "cli.gen_index:generate_fixtures_index_cli" gentest = "cli.gentest:generate" eofwrap = "cli.eofwrap:eof_wrap" diff --git a/src/cli/pytest_commands/__init__.py b/src/cli/pytest_commands/__init__.py index b1496dfb7e9..0c465f49c73 100644 --- a/src/cli/pytest_commands/__init__.py +++ b/src/cli/pytest_commands/__init__.py @@ -19,6 +19,8 @@ fill --help # for example, or consume engine +# or +checklist --help ``` They can also be executed (and debugged) directly in an interactive python diff --git a/src/cli/pytest_commands/checklist.py b/src/cli/pytest_commands/checklist.py new file mode 100644 index 00000000000..d53406201e3 --- /dev/null +++ b/src/cli/pytest_commands/checklist.py @@ -0,0 +1,77 @@ +"""CLI entry point for the `checklist` pytest-based command.""" + +from typing import List + +import click + +from .base import PytestCommand + + +class ChecklistCommand(PytestCommand): + """Pytest command for generating EIP checklists.""" + + def __init__(self): + """Initialize checklist command with processors.""" + super().__init__( + config_file="pytest.ini", + ) + + def process_arguments(self, pytest_args: List[str]) -> List[str]: + """Process arguments, ensuring checklist generation is enabled.""" + processed_args = super().process_arguments(pytest_args) + + # Add collect-only flag to avoid running tests + processed_args.extend(["-p", "pytest_plugins.filler.eip_checklist"]) + + return processed_args + + +@click.command() +@click.option( + "--output", + "-o", + type=click.Path(file_okay=False, dir_okay=True, writable=True), + default="./checklists", + help="Directory to output the generated checklists (default: ./checklists)", +) +@click.option( + "--eip", + "-e", + type=int, + multiple=True, + help="Generate checklist only for specific EIP(s)", +) +def checklist(output: str, eip: tuple, **kwargs) -> None: + """ + Generate EIP test checklists based on pytest.mark.eip_checklist markers. + + This command scans test files for eip_checklist markers and generates + filled checklists showing which checklist items have been implemented. + + Examples: + # Generate checklists for all EIPs + uv run checklist + + # Generate checklist for specific EIP + uv run checklist --eip 7702 + + # Generate checklists for specific test path + uv run checklist tests/prague/eip7702* + + # Specify output directory + uv run checklist --output ./my-checklists + + """ + # Add output directory to pytest args + args = ["--checklist-output", output] + + # Add EIP filter if specified + for eip_num in eip: + args.extend(["--checklist-eip", str(eip_num)]) + + command = ChecklistCommand() + command.execute(args) + + +if __name__ == "__main__": + checklist() diff --git a/src/pytest_plugins/filler/eip_checklist.py b/src/pytest_plugins/filler/eip_checklist.py new file mode 100644 index 00000000000..d395c7e0a50 --- /dev/null +++ b/src/pytest_plugins/filler/eip_checklist.py @@ -0,0 +1,386 @@ +""" +Pytest plugin for generating EIP test completion checklists. + +This plugin collects checklist markers from tests and generates a filled checklist +for each EIP based on the template at +docs/writing_tests/checklist_templates/eip_testing_checklist_template.md +""" + +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Set, Tuple + +import pytest + +from .gen_test_doc.page_props import EipChecklistPageProps + +logger = logging.getLogger("mkdocs") + + +def pytest_addoption(parser: pytest.Parser): + """Add command-line options for checklist generation.""" + group = parser.getgroup("checklist", "EIP checklist generation options") + group.addoption( + "--checklist-output", + action="store", + dest="checklist_output", + type=Path, + default=Path("./checklists"), + help="Directory to output the generated checklists", + ) + group.addoption( + "--checklist-eip", + action="append", + dest="checklist_eips", + type=int, + default=[], + help="Generate checklist only for specific EIP(s)", + ) + group.addoption( + "--checklist-doc-gen", + action="store_true", + dest="checklist_doc_gen", + default=False, + help="Generate checklists for documentation (uses mkdocs_gen_files)", + ) + + +TITLE_LINE = "# EIP Execution Layer Testing Checklist Template" +PERCENTAGE_LINE = "| TOTAL_CHECKLIST_ITEMS | COVERED_CHECKLIST_ITEMS | PERCENTAGE |" +TEMPLATE_PATH = ( + Path(__file__).parents[3] + / "docs" + / "writing_tests" + / "checklist_templates" + / "eip_testing_checklist_template.md" +) +TEMPLATE_CONTENT = TEMPLATE_PATH.read_text() +EXTERNAL_COVERAGE_FILE_NAME = "eip_checklist_external_coverage.txt" +NOT_APPLICABLE_FILE_NAME = "eip_checklist_not_applicable.txt" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): # noqa: D103 + config.pluginmanager.register(EIPChecklistCollector(), "eip-checklist-collector") + + +@dataclass(kw_only=True) +class EIPItem: + """Represents an EIP checklist item.""" + + id: str + line_number: int + description: str + tests: Set[str] + not_applicable_reason: str = "" + external_coverage_reason: str = "" + + @classmethod + def from_checklist_line(cls, *, line: str, line_number: int) -> "EIPItem | None": + """Create an EIP item from a checklist line.""" + match = re.match(r"\|\s*`([^`]+)`\s*\|\s*([^|]+)\s*\|", line) + if not match: + return None + return cls( + id=match.group(1), + line_number=line_number, + description=match.group(2), + tests=set(), + ) + + @property + def covered(self) -> bool: + """Return True if the item is covered by at least one test.""" + return len(self.tests) > 0 or self.external_coverage + + @property + def external_coverage(self) -> bool: + """Return True if the item is covered by an external test/procedure.""" + return self.external_coverage_reason != "" + + @property + def not_applicable(self) -> bool: + """Return True if the item is not applicable.""" + return self.not_applicable_reason != "" + + def __str__(self) -> str: + """Return a string representation of the EIP item.""" + status = " " + tests = "" + if self.external_coverage: + status = "โœ…" + tests = self.external_coverage_reason + elif self.covered: + status = "โœ…" + tests = ", ".join(sorted(self.tests)) + elif self.not_applicable: + status = "N/A" + tests = self.not_applicable_reason + + return f"| `{self.id}` | {self.description} | {status} | {tests} |" + + +TEMPLATE_ITEMS: Dict[str, EIPItem] = {} +# Parse the template to extract checklist item IDs and descriptions +for i, line in enumerate(TEMPLATE_CONTENT.splitlines()): + # Match lines that contain checklist items with IDs in backticks + if item := EIPItem.from_checklist_line(line=line, line_number=i + 1): + TEMPLATE_ITEMS[item.id] = item + +ALL_IDS = set(TEMPLATE_ITEMS.keys()) + + +def resolve_id(item_id: str) -> Set[str]: + """Resolve an item ID to a set of checklist IDs.""" + covered_ids = {checklist_id for checklist_id in ALL_IDS if checklist_id.startswith(item_id)} + return covered_ids + + +@dataclass(kw_only=True) +class EIP: + """Represents an EIP and its checklist.""" + + number: int + items: Dict[str, EIPItem] = field(default_factory=TEMPLATE_ITEMS.copy) + path: Path | None = None + + def add_covered_test(self, checklist_id: str, node_id: str) -> None: + """Add a covered test to the EIP.""" + self.items[checklist_id].tests.add(node_id) + + @property + def covered_items(self) -> int: + """Return the number of covered items.""" + return sum(1 for item in self.items.values() if item.covered) + + @property + def total_items(self) -> int: + """Return the number of total items.""" + return sum(1 for item in self.items.values() if not item.not_applicable) + + @property + def percentage(self) -> float: + """Return the percentage of covered items.""" + return self.covered_items / self.total_items * 100 if self.total_items else 0 + + @property + def completness_emoji(self) -> str: + """Return the completness emoji.""" + return "๐ŸŸข" if self.percentage == 100 else "๐ŸŸก" if self.percentage > 50 else "๐Ÿ”ด" + + def mark_not_applicable(self): + """Read the not-applicable items from the EIP.""" + if self.path is None: + return + not_applicable_path = self.path / NOT_APPLICABLE_FILE_NAME + if not not_applicable_path.exists(): + return + with not_applicable_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + assert "=" in line + item_id, reason = line.split("=", 1) + item_id = item_id.strip() + reason = reason.strip() + assert reason, f"Reason is empty for {line}" + assert item_id, f"Item ID is empty for {line}" + ids = resolve_id(item_id) + if not ids: + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for EIP {self.number}" + ) + continue + for id_covered in ids: + self.items[id_covered].not_applicable_reason = reason + + def mark_external_coverage(self): + """Read the externally covered items from the EIP.""" + if self.path is None: + return + external_coverage_path = self.path / EXTERNAL_COVERAGE_FILE_NAME + if not external_coverage_path.exists(): + return + with external_coverage_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + assert "=" in line + item_id, reason = line.split("=", 1) + item_id = item_id.strip() + reason = reason.strip() + assert item_id, f"Item ID is empty for {line}" + assert reason, f"Reason is empty for {line}" + ids = resolve_id(item_id) + if not ids: + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for EIP {self.number}" + ) + continue + for id_covered in ids: + self.items[id_covered].external_coverage_reason = reason + + def generate_filled_checklist_lines(self) -> List[str]: + """Generate the filled checklist lines for a specific EIP.""" + # Create a copy of the template content + lines = TEMPLATE_CONTENT.splitlines() + + self.mark_not_applicable() + self.mark_external_coverage() + + for checklist_item in self.items.values(): + # Find the line with this item ID + lines[checklist_item.line_number - 1] = str(checklist_item) + + lines[lines.index(PERCENTAGE_LINE)] = ( + f"| {self.total_items} | {self.covered_items} | {self.completness_emoji} " + f"{self.percentage:.2f}% |" + ) + + # Replace the title line with the EIP number + lines[lines.index(TITLE_LINE)] = f"# EIP-{self.number} Test Checklist" + + return lines + + def generate_filled_checklist(self, output_dir: Path) -> Path: + """Generate a filled checklist for a specific EIP.""" + lines = self.generate_filled_checklist_lines() + + output_dir = output_dir / f"eip{self.number}_checklist.md" + + # Write the filled checklist + output_dir.parent.mkdir(exist_ok=True, parents=True) + output_dir.write_text("\n".join(lines)) + + return output_dir + + +class EIPChecklistCollector: + """Collects and manages EIP checklist items from test markers.""" + + def __init__(self: "EIPChecklistCollector"): + """Initialize the EIP checklist collector.""" + self.eips: Dict[int, EIP] = {} + + def extract_eip_from_path(self, test_path: Path) -> Tuple[int | None, Path | None]: + """Extract EIP number from test file path.""" + # Look for patterns like eip1234_ or eip1234/ in the path + for part_idx, part in enumerate(test_path.parts): + match = re.match(r"eip(\d+)", part) + if match: + eip = int(match.group(1)) + eip_path = test_path.parents[len(test_path.parents) - part_idx - 2] + return eip, eip_path + return None, None + + def get_eip_from_item(self, item: pytest.Item) -> EIP | None: + """Get the EIP for a test item.""" + test_path = Path(item.location[0]) + for part_idx, part in enumerate(test_path.parts): + match = re.match(r"eip(\d+)", part) + if match: + eip = int(match.group(1)) + if eip not in self.eips: + self.eips[eip] = EIP( + number=eip, + path=test_path.parents[len(test_path.parents) - part_idx - 2], + ) + else: + if self.eips[eip].path is None: + self.eips[eip].path = test_path.parents[ + len(test_path.parents) - part_idx - 2 + ] + return self.eips[eip] + return None + + def get_eip(self, eip: int) -> EIP: + """Get the EIP for a given EIP number.""" + if eip not in self.eips: + self.eips[eip] = EIP(number=eip, path=None) + return self.eips[eip] + + def collect_from_item(self, item: pytest.Item, primary_eip: EIP | None) -> None: + """Collect checklist markers from a test item.""" + for marker in item.iter_markers("eip_checklist"): + if not marker.args: + pytest.fail( + f"eip_checklist marker on {item.nodeid} must have at least one argument " + "(item_id)" + ) + additional_eips = marker.kwargs.get("eip", []) + if not isinstance(additional_eips, list): + additional_eips = [additional_eips] + + eips: List[EIP] = [primary_eip] if primary_eip else [] + + if additional_eips: + if any(not isinstance(eip, int) for eip in additional_eips): + pytest.fail( + "EIP numbers must be integers. Found non-integer EIPs in " + f"{item.nodeid}: {additional_eips}" + ) + eips += [self.get_eip(eip) for eip in additional_eips] + + for item_id in marker.args: + covered_ids = resolve_id(item_id.strip()) + if not covered_ids: + logger.warning( + f"Item ID {item_id} not found in the checklist template, " + f"for test {item.nodeid}" + ) + continue + for id_covered in covered_ids: + for eip in eips: + eip.add_covered_test(id_covered, item.nodeid) + + @pytest.hookimpl(tryfirst=True) + def pytest_runtestloop(self, session): + """Skip test execution, only generate checklists.""" + session.testscollected = 0 + return True + + def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): + """Collect checklist markers during test collection.""" + for item in items: + eip = self.get_eip_from_item(item) + if item.get_closest_marker("derived_test") or item.get_closest_marker("skip"): + continue + self.collect_from_item(item, eip) + + # Check which mode we are in + checklist_doc_gen = config.getoption("checklist_doc_gen", False) + checklist_output = config.getoption("checklist_output", Path("checklists")) + checklist_eips = config.getoption("checklist_eips", []) + + checklist_props = {} + # Generate a checklist for each EIP + for eip in self.eips.values(): + # Skip if specific EIPs were requested and this isn't one of them + if checklist_eips and eip.number not in checklist_eips: + continue + + if checklist_doc_gen: + assert eip.path is not None + checklist_path = eip.path / "checklist.md" + checklist_props[checklist_path] = EipChecklistPageProps( + title=f"EIP-{eip.number} Test Checklist", + source_code_url="", + target_or_valid_fork="mainnet", + path=checklist_path, + pytest_node_id="", + package_name="checklist", + eip=eip.number, + lines=eip.generate_filled_checklist_lines(), + ) + else: + checklist_path = eip.generate_filled_checklist(checklist_output) + print(f"\nGenerated EIP-{eip.number} checklist: {checklist_path}") + + if checklist_doc_gen: + config.checklist_props = checklist_props # type: ignore diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index e1a4a1256f2..7a1728b4380 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -796,12 +796,15 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): """ for test_type in BaseTest.spec_types.values(): if test_type.pytest_parameter_name() in metafunc.fixturenames: + parameters = [] + for i, format_with_or_without_label in enumerate(test_type.supported_fixture_formats): + parameter = labeled_format_parameter_set(format_with_or_without_label) + if i > 0: + parameter.marks.append(pytest.mark.derived_test) # type: ignore + parameters.append(parameter) metafunc.parametrize( [test_type.pytest_parameter_name()], - [ - labeled_format_parameter_set(format_with_or_without_label) - for format_with_or_without_label in test_type.supported_fixture_formats - ], + parameters, scope="function", indirect=True, ) diff --git a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py index aa91aeba446..52f63604ae3 100644 --- a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py +++ b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py @@ -254,7 +254,7 @@ def __init__(self, config) -> None: self.page_props: PagePropsLookup = {} @pytest.hookimpl(hookwrapper=True, trylast=True) - def pytest_collection_modifyitems(self, session, config, items): + def pytest_collection_modifyitems(self, config: pytest.Config, items: List[pytest.Item]): """Generate html doc for each test item that pytest has collected.""" yield @@ -264,11 +264,15 @@ def pytest_collection_modifyitems(self, session, config, items): for item in items: # group test case by test function functions[get_test_function_id(item)].append(item) + if hasattr(config, "checklist_props"): + checklist_props = config.checklist_props + self.page_props = {**self.page_props, **checklist_props} + # the heavy work self.create_function_page_props(functions) self.create_module_page_props() # add the pages to the page_props dict - self.page_props = {**self.function_page_props, **self.module_page_props} + self.page_props = {**self.page_props, **self.function_page_props, **self.module_page_props} # this adds pages for the intermediate directory structure (tests, tests/berlin) self.add_directory_page_props() # add other interesting pages @@ -592,4 +596,4 @@ def sort_by_fork_deployment_and_path(x: PageProps) -> Tuple[Any, ...]: def write_pages(self) -> None: """Write all pages to the target directory.""" for page in self.page_props.values(): - page.write_page(self.jinja2_env) + page.write_page(mkdocs_gen_files, self.jinja2_env) diff --git a/src/pytest_plugins/filler/gen_test_doc/page_props.py b/src/pytest_plugins/filler/gen_test_doc/page_props.py index 67a02467a9b..ec5e075fb88 100644 --- a/src/pytest_plugins/filler/gen_test_doc/page_props.py +++ b/src/pytest_plugins/filler/gen_test_doc/page_props.py @@ -14,9 +14,8 @@ from abc import abstractmethod from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Dict, List +from typing import IO, Any, ContextManager, Dict, List, Protocol -import mkdocs_gen_files # type: ignore from jinja2 import Environment from ethereum_test_tools import Opcodes @@ -79,6 +78,17 @@ def nav_path_to_sanitized_str_tuple(nav_path: Path) -> tuple: return tuple(sanitize_string_title(part) for part in nav_path.parts) +class FileOpener(Protocol): + """ + Protocol to replace `mkdocs_gen_files` so it doesn't have to be imported/installed for + unit tests. + """ + + def open(self, path: Path, mode: str) -> ContextManager[IO[Any]]: + """Open a file for writing.""" + raise NotImplementedError + + @dataclass class PagePropsBase: """ @@ -114,15 +124,38 @@ def nav_entry(self, top_level_nav_entry: str) -> tuple: path = top_level_nav_entry / Path(*self.path.parts[1:]).with_suffix("") return nav_path_to_sanitized_str_tuple(path) - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """Write the page to the target directory.""" template = jinja2_env.get_template(self.template) rendered_content = template.render(**asdict(self)) - with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + with file_opener.open(self.target_output_file, "w") as destination: for line in rendered_content.splitlines(keepends=True): destination.write(line) +@dataclass +class EipChecklistPageProps(PagePropsBase): + """Properties used to generate the EIP checklist page.""" + + eip: int + lines: List[str] + + @property + def template(self) -> str: + """Get the jinja2 template used to render this page.""" + raise Exception("EipChecklistPageProps does not have a template") + + @property + def target_output_file(self) -> Path: + """Get the target output file for this page.""" + return self.path + + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): + """Write the page to the target directory.""" + with file_opener.open(self.target_output_file, "w") as destination: + destination.write("\n".join(self.lines)) + + @dataclass class TestCase: """Properties used to define a single test case in test function parameter tables.""" @@ -164,20 +197,20 @@ def nav_entry(self, top_level_nav_entry) -> tuple: nav_path_prefix = super().nav_entry(top_level_nav_entry) # already sanitized return (*nav_path_prefix, f"{self.title}") - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """ Test functions also get a static HTML page with parametrized test cases. This is intended for easier viewing (without mkdocs styling) of the data-table that documents the parametrized test cases. """ - super().write_page(jinja2_env) + super().write_page(file_opener, jinja2_env) if not self.cases: return html_template = jinja2_env.get_template("function.html.j2") rendered_html_content = html_template.render(**asdict(self)) html_output_file = self.target_output_file.with_suffix(".html") - with mkdocs_gen_files.open(html_output_file, "w") as destination: + with file_opener.open(html_output_file, "w") as destination: for line in rendered_html_content.splitlines(keepends=True): destination.write(line) @@ -240,7 +273,7 @@ def target_output_file(self) -> Path: """Get the target output file for this page.""" return self.path - def write_page(self, jinja2_env: Environment): + def write_page(self, file_opener: FileOpener, jinja2_env: Environment): """ Write the page to the target directory. @@ -249,14 +282,20 @@ def write_page(self, jinja2_env: Environment): template = jinja2_env.get_template(self.template) rendered_content = template.render(**asdict(self)) with open(self.path, "r") as md_source: - with mkdocs_gen_files.open(self.target_output_file, "w") as destination: + with file_opener.open(self.target_output_file, "w") as destination: for line in rendered_content.splitlines(keepends=True): destination.write(line) for line in md_source: destination.write(line) -PageProps = DirectoryPageProps | ModulePageProps | FunctionPageProps | MarkdownPageProps +PageProps = ( + DirectoryPageProps + | ModulePageProps + | FunctionPageProps + | MarkdownPageProps + | EipChecklistPageProps +) PagePropsLookup = Dict[str, PageProps] ModulePagePropsLookup = Dict[str, ModulePageProps] FunctionPagePropsLookup = Dict[str, FunctionPageProps] diff --git a/src/pytest_plugins/filler/tests/test_eip_checklist.py b/src/pytest_plugins/filler/tests/test_eip_checklist.py new file mode 100644 index 00000000000..29511751128 --- /dev/null +++ b/src/pytest_plugins/filler/tests/test_eip_checklist.py @@ -0,0 +1,113 @@ +"""Test the EIP checklist plugin functionality.""" + +import textwrap + + +def test_eip_checklist_collection(testdir): + """Test that checklist markers are collected correctly.""" + # Create the test in an EIP-specific directory + tests_dir = testdir.mkdir("tests") + + prague_tests_dir = tests_dir.mkdir("prague") + eip_7702_tests_dir = prague_tests_dir.mkdir("eip7702_set_code_tx") + test_7702_module = eip_7702_tests_dir.join("test_eip7702.py") + test_7702_module.write( + textwrap.dedent( + """ + import pytest + from ethereum_test_tools import StateTestFiller + + REFERENCE_SPEC_GIT_PATH = "N/A" + REFERENCE_SPEC_VERSION = "N/A" + + @pytest.mark.valid_at("Prague") + @pytest.mark.eip_checklist( + "new_transaction_type/test/intrinsic_validity/gas_limit/exact" + ) + def test_exact_gas(state_test: StateTestFiller): + pass + + @pytest.mark.valid_at("Prague") + @pytest.mark.eip_checklist( + "new_transaction_type/test/signature/invalid/v/2", + eip=[7702, 2930] + ) + def test_invalid_v(state_test: StateTestFiller): + pass + """ + ) + ) + eip_7702_external_coverage_file = eip_7702_tests_dir.join( + "eip_checklist_external_coverage.txt" + ) + eip_7702_external_coverage_file.write( + textwrap.dedent( + """ + general/code_coverage/eels = DEBUG EXTERNAL COVERAGE REASON + """ + ) + ) + + berlin_tests_dir = tests_dir.mkdir("berlin") + eip_2930_tests_dir = berlin_tests_dir.mkdir("eip2930_set_code_tx") + test_2930_module = eip_2930_tests_dir.join("test_eip2930.py") + test_2930_module.write( + textwrap.dedent( + """ + import pytest + from ethereum_test_tools import StateTestFiller + + REFERENCE_SPEC_GIT_PATH = "N/A" + REFERENCE_SPEC_VERSION = "N/A" + + @pytest.mark.valid_at("Berlin") + def test_berlin_one(state_test: StateTestFiller): + pass + """ + ) + ) + test_2930_n_a_file = eip_2930_tests_dir.join("eip_checklist_not_applicable.txt") + test_2930_n_a_file.write( + textwrap.dedent( + """ + new_system_contract = DEBUG NOT APPLICABLE REASON + """ + ) + ) + # Run pytest with checklist-only mode + testdir.copy_example(name="pytest.ini") + result = testdir.runpytest( + "-p", + "pytest_plugins.filler.eip_checklist", + "--collect-only", + "--checklist-output", + str(testdir.tmpdir / "checklists"), + str(tests_dir), + ) + result.assert_outcomes( + passed=0, + failed=0, + skipped=0, + errors=0, + ) + + # Check that checklists were generated + checklist_dir = testdir.tmpdir / "checklists" + assert checklist_dir.exists() + checklist_file = checklist_dir / "eip7702_checklist.md" + assert checklist_file.exists() + + # Verify the checklist contains the expected markers + content = checklist_file.read() + assert "โœ…" in content + assert "test_exact_gas" in content + assert "test_invalid_v" in content + assert "DEBUG EXTERNAL COVERAGE REASON" in content + + checklist_file = checklist_dir / "eip2930_checklist.md" + assert checklist_file.exists() + content = checklist_file.read() + assert "โœ…" in content + assert "test_invalid_v" in content + assert "N/A" in content + assert "DEBUG NOT APPLICABLE REASON" in content diff --git a/src/pytest_plugins/shared/execute_fill.py b/src/pytest_plugins/shared/execute_fill.py index ae9706624c1..7eb41db754e 100644 --- a/src/pytest_plugins/shared/execute_fill.py +++ b/src/pytest_plugins/shared/execute_fill.py @@ -89,6 +89,17 @@ def pytest_configure(config: pytest.Config): "markers", "exception_test: Negative tests that include an invalid block or transaction.", ) + config.addinivalue_line( + "markers", + "eip_checklist(item_id, eip=None): Mark a test as implementing a specific checklist item. " + "The first positional parameter is the checklist item ID. " + "The optional 'eip' keyword parameter specifies additional EIPs covered by the test.", + ) + config.addinivalue_line( + "markers", + "derived_test: Mark a test as a derived test (E.g. a BlockchainTest that is derived " + "from a StateTest).", + ) @pytest.fixture(autouse=True) diff --git a/src/pytest_plugins/shared/helpers.py b/src/pytest_plugins/shared/helpers.py index 52cfa6e811d..bbe720b652f 100644 --- a/src/pytest_plugins/shared/helpers.py +++ b/src/pytest_plugins/shared/helpers.py @@ -18,6 +18,8 @@ def is_help_or_collectonly_mode(config: pytest.Config) -> bool: or config.getoption("show_ported_from", default=False) or config.getoption("links_as_filled", default=False) or config.getoption("help", default=False) + or config.pluginmanager.has_plugin("pytest_plugins.filler.eip_checklist") + or config.pluginmanager.has_plugin("pytest_plugins.filler.gen_test_doc.gen_test_doc") )