diff --git a/.github/actions/setup-env/action.yaml b/.github/actions/setup-env/action.yaml index 804bf18fa0..1344c97151 100644 --- a/.github/actions/setup-env/action.yaml +++ b/.github/actions/setup-env/action.yaml @@ -4,10 +4,10 @@ runs: using: "composite" steps: - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential rustup + rustup update --no-self-update 1.89.0 && rustup default 1.89.0 - name: Install Tox and any other packages shell: bash @@ -20,4 +20,4 @@ runs: run: | mkdir -p $GITHUB_WORKSPACE/bin $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH \ No newline at end of file + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH diff --git a/pyproject.toml b/pyproject.toml index 69eb8378b2..5407127960 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,8 @@ packages = [ "ethereum_spec_tools.evm_tools.loaders", "ethereum_spec_tools.lint", "ethereum_spec_tools.lint.lints", + "ethereum_spec_tools.new_fork", + "ethereum_spec_tools.new_fork.codemod", "ethereum", "ethereum.forks.frontier", "ethereum.forks.frontier.utils", @@ -255,6 +257,7 @@ test = [ "filelock>=3.15.1,<4", "requests", "requests-cache>=1.2.1,<2", + "libcst>=1.8,<2", ] fill = [ @@ -294,6 +297,7 @@ lint = [ ] tools = [ "platformdirs>=4.2,<5", + "libcst>=1.8,<2", ] doc = [ "docc>=0.3.0,<0.4.0", @@ -310,7 +314,7 @@ version = { attr = "ethereum.__version__" } [project.scripts] ethereum-spec-lint = "ethereum_spec_tools.lint:main" ethereum-spec-sync = "ethereum_spec_tools.sync:main" -ethereum-spec-new-fork = "ethereum_spec_tools.new_fork:main" +ethereum-spec-new-fork = "ethereum_spec_tools.new_fork.cli:main" ethereum-spec-patch = "ethereum_spec_tools.patch_tool:main" ethereum-spec-evm = "ethereum_spec_tools.evm_tools:main" check_eip_versions = "cli.pytest_commands.check_eip_versions:check_eip_versions" diff --git a/src/ethereum_spec_tools/new_fork.py b/src/ethereum_spec_tools/new_fork.py deleted file mode 100644 index 37f4bf9c21..0000000000 --- a/src/ethereum_spec_tools/new_fork.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Tool to create a new fork using the latest fork. -""" - -import argparse -import fnmatch -import os -import re -from shutil import copytree -from typing import Tuple - -DESCRIPTION = """ -Creates the base code for a new fork by using the \ -existing code from a given fork. - - -The ethereum-spec-new-fork command takes 4 arguments, 2 of which are optional - from_fork: The fork name from which the code is to be duplicated \ -Eg - "Tangerine Whistle" - to_fork: The fork name of the new fork Eg - "Spurious Dragon" - from_test(Optional): Name of the from fork within the test fixtures \ -in case it is different from fork name Eg - "EIP150" - to_test(Optional): Name of the to fork within the test fixtures \ -in case it is different from fork name Eg - "EIP158" - - -If one wants to create the spurious dragon fork from the tangerine whistle one - ethereum-spec-new-fork --from_fork="Tangerine Whistle" \ ---to_fork="Spurious Dragon" \ ---from_test=EIP150 \ ---to_test=EIP158 - -The following will have to however, be updated manually - 1. The fork number and MAINNET_FORK_BLOCK in __init__.py - 2. Any absolute package imports from other forks eg. in trie.py - 3. Package Names under setup.cfg - 4. Add the new fork to the monkey_patch() function in \ -src/ethereum_optimized/__init__.py - 5. Adjust the underline in fork/__init__.py -""" - -parser = argparse.ArgumentParser( - description=DESCRIPTION, - formatter_class=argparse.RawDescriptionHelpFormatter, -) - -parser.add_argument("--from_fork", dest="from_fork", type=str, required=True) -parser.add_argument("--to_fork", dest="to_fork", type=str, required=True) -parser.add_argument("--from_test", dest="from_test", type=str) -parser.add_argument("--to_test", dest="to_test", type=str) - - -def find_replace( - directory: str, find: str, replace: str, file_pattern: str -) -> None: - """ - Replace the occurrence of a certain text in files with a new text. - """ - for path, _, files in os.walk(directory): - for filename in fnmatch.filter(files, file_pattern): - file_path = os.path.join(path, filename) - with open(file_path, "r+b") as f: - s = f.read() - find_pattern = (r"\b" + re.escape(find) + r"\b").encode() - s = re.sub(find_pattern, replace.encode(), s) - f.seek(0) - f.write(s) - f.truncate() - - -class ForkCreator: - """ - Object to create base code for a new fork from the previous fork. - """ - - def __init__( - self, - from_fork: str, - to_fork: str, - from_test_names: str, - to_test_names: str, - ): - self.package_folder = "src/ethereum" - self.test_folder = "tests" - - # Get the fork specific data for from fork - ( - self.from_fork, - self.from_package, - self.from_path, - self.from_test_path, - ) = self.get_fork_paths(from_fork) - - # Get the fork specific data for to fork - ( - self.to_fork, - self.to_package, - self.to_path, - self.to_test_path, - ) = self.get_fork_paths(to_fork) - - self.from_test_names = from_test_names - self.to_test_names = to_test_names - - def get_fork_paths(self, fork: str) -> Tuple[str, ...]: - """ - Get the relevant paths for all folders in a particular fork. - """ - name = fork - package = name.replace(" ", "_").lower() - path = os.path.join(self.package_folder, package) - test_path = os.path.join(self.test_folder, package) - return (name, package, path, test_path) - - def duplicate_fork(self) -> None: - """ - Copy the relevant files/folders from the old fork. - """ - copytree(self.from_path, self.to_path) - copytree(self.from_test_path, self.to_test_path) - - def update_new_fork_contents(self) -> None: - """ - Replace references to the old fork with the new ones - The following however, will have to be updated manually - 1. The fork number and MAINNET_FORK_BLOCK in __init__.py - 2. Any absolute package imports from other forks eg. in trie.py - 3. Package Names under setup.cfg. - """ - # Update Source Code - find_replace(self.to_path, self.from_fork, self.to_fork, "*.py") - find_replace(self.to_path, self.from_package, self.to_package, "*.py") - find_replace( - self.to_path, self.from_fork.lower(), self.to_fork.lower(), "*.py" - ) - - # Update test files starting with the names used in the test fixtures - find_replace( - self.to_test_path, self.from_test_names, self.to_test_names, "*.py" - ) - find_replace(self.to_test_path, self.from_fork, self.to_fork, "*.py") - find_replace( - self.to_test_path, self.from_package, self.to_package, "*.py" - ) - find_replace( - self.to_test_path, - self.from_fork.lower(), - self.to_fork.lower(), - "*.py", - ) - - -def main() -> None: - """ - Create the new fork. - """ - options = parser.parse_args() - from_fork = options.from_fork - to_fork = options.to_fork - - from_test = from_fork if options.from_test is None else options.from_test - to_test = to_fork if options.to_test is None else options.to_test - - fork_creator = ForkCreator(from_fork, to_fork, from_test, to_test) - fork_creator.duplicate_fork() - fork_creator.update_new_fork_contents() - - final_help_text = """ -Fork `{fork}` has been successfully created. - -PLEASE REMEMBER TO UPDATE THE FOLLOWING MANUALLY: - 1. The fork number and MAINNET_FORK_BLOCK in __init__.py. \ -If you are proposing a new EIP, please set MAINNET_FORK_BLOCK to None. - 2. Any absolute package imports from other forks. Eg. in trie.py - 3. Package Names under setup.cfg - 4. Add the new fork to the monkey_patch() function in \ -src/ethereum_optimized/__init__.py - 5. Adjust the underline in src/{package}/__init__.py -""".format( - fork=fork_creator.to_fork, - package=fork_creator.to_package, - ) - print(final_help_text) - - -if __name__ == "__main__": - main() diff --git a/src/ethereum_spec_tools/new_fork/__init__.py b/src/ethereum_spec_tools/new_fork/__init__.py new file mode 100644 index 0000000000..66a33d9006 --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/__init__.py @@ -0,0 +1,4 @@ +""" +Command-line tool for creating a new fork package (eg. `src/ethereum/osaka`) +from an existing fork package. +""" diff --git a/src/ethereum_spec_tools/new_fork/builder.py b/src/ethereum_spec_tools/new_fork/builder.py new file mode 100644 index 0000000000..09a49ae6c9 --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/builder.py @@ -0,0 +1,424 @@ +""" +Defines `ForkBuilder`, a class that can take a template fork and transform it +into a new fork. +""" + +import json +import sys +from abc import ABC, abstractmethod +from contextlib import ExitStack, chdir +from dataclasses import dataclass, field +from pathlib import Path +from shutil import copytree, rmtree +from tempfile import TemporaryDirectory +from typing import Final, NamedTuple + +from ethereum_types.numeric import U256, Uint +from libcst.tool import main as libcst_tool +from typing_extensions import override + +from ethereum.fork_criteria import ( + ByBlockNumber, + ByTimestamp, + ForkCriteria, + Unscheduled, +) + +from ..forks import Hardfork + + +@dataclass +class CodemodArgs(ABC): + """ + Description of a libcst codemod as understood by `libcst.tool:main`. + """ + + @abstractmethod + def _to_args( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> list[list[str]]: + raise NotImplementedError + + +@dataclass +class RenameFork(CodemodArgs): + """ + Describe how to rename a fork to `libcst.tool:main`. + """ + + @override + def _to_args( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> list[list[str]]: + prefix = ".".join(fork_builder.template_fork.name.split(".")[:-1]) + + commands = [ + [ + "codemod", + "rename.RenameCommand", + "--no-format", + "--old_name", + fork_builder.template_fork.name, + "--new_name", + f"{prefix}{fork_builder.new_fork}", + str(working_directory), + ] + ] + + forks = Hardfork.discover() + before_fork = None + for fork in forks: + if fork.criteria > fork_builder.fork_criteria: + break + before_fork = fork + + if before_fork is None: + assert fork_builder.before_template_fork is None + return commands + + assert fork_builder.before_template_fork is not None + + commands.append( + [ + "codemod", + "rename.RenameCommand", + "--old_name", + fork_builder.before_template_fork.name, + "--new_name", + before_fork.name, + str(working_directory), + ] + ) + + return commands + + +class _Replacement(NamedTuple): + qualified_name: str + value: str + imports: list[tuple[str, str]] + + +@dataclass +class ReplaceValue(CodemodArgs, ABC): + """ + Base class for codemod descriptions that replace the value of an + assignment. + """ + + @abstractmethod + def _replacement( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> _Replacement: + raise NotImplementedError + + @override + def _to_args( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> list[list[str]]: + qualified_name, value, imports = self._replacement( + fork_builder, working_directory + ) + + output = fork_builder.output + assert output is not None + + fully_qualified_name = ( + f"ethereum.{fork_builder.new_fork}.{qualified_name}" + ) + + command = [ + "codemod", + "constant.SetConstantCommand", + "--no-format", + "--qualified-name", + fully_qualified_name, + "--value", + value, + str(working_directory), + ] + + for module, identifier in imports: + command.extend( + [ + "--import", + module, + identifier, + ] + ) + + return [command] + + +@dataclass +class SetConstant(ReplaceValue): + """ + Instruct `libcst.tool:main` to replace the value of a constant. + """ + + qualified_name: str + value: str + imports: list[tuple[(str, str)]] = field(default_factory=list) + + @override + def _replacement( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> _Replacement: + return _Replacement(self.qualified_name, self.value, self.imports) + + +@dataclass +class SetForkCriteria(ReplaceValue): + """ + Instruct `libcst.tool:main` to replace the value of `FORK_CRITERIA`. + """ + + @override + def _replacement( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> _Replacement: + return _Replacement( + "FORK_CRITERIA", + repr(fork_builder.fork_criteria), + [ + ("ethereum.fork_criteria", "Unscheduled"), + ("ethereum.fork_criteria", "ByBlockNumber"), + ("ethereum.fork_criteria", "ByTimestamp"), + ], + ) + + +@dataclass +class ReplaceForkName(CodemodArgs): + """ + Replace occurrences of the template fork name with the new fork's name. + """ + + @override + def _to_args( + self, fork_builder: "ForkBuilder", working_directory: Path + ) -> list[list[str]]: + common = [ + str(working_directory), + "--replace", + fork_builder.template_fork.short_name, + fork_builder.new_fork, + "--replace", + fork_builder.template_fork.title_case_name, + fork_builder.new_fork.replace("_", " ").title(), + "--replace", + fork_builder.template_fork.title_case_name.lower(), + fork_builder.new_fork.replace("_", " ").lower(), + ] + + commands = [ + [ + "codemod", + "--no-format", + "string.StringReplaceCommand", + ] + + common, + [ + "codemod", + "--no-format", + "comment.CommentReplaceCommand", + ] + + common, + ] + + return commands + + +class ForkBuilder: + """ + Takes a template fork and uses it to generate a new fork, applying source + code transformations along the way. + """ + + before_template_fork: Final[Hardfork | None] + """ + Fork immediately before `template_fork`, if one exists (else `None`). + + Necessary because some modules (notably `trie.py`) import types from the + preceding fork, and those references need to be updated. + """ + + template_fork: Final[Hardfork] + """ + Fork that is copied and modified into the new fork. + """ + + new_fork: Final[str] + """ + Name of the new fork as a Python-friendly identifier. + + For example, `"spurious_dragon"` and not `"Spurious Dragon"`. + """ + + output: Path | None + """ + Directory to place the new fork into, not including the fork itself. + + For example, to place `frontier` into `src/ethereum/frontier`, this value + would be `src/ethereum`. + + Defaults to the parent directory of `template_fork` (or `None` if one + cannot be found). Can be overwritten. + """ + + force: bool + """ + Replace the destination if `True`, otherwise error if the destination + already exists. + """ + + modifiers: list[CodemodArgs] + """ + Ordered list of code modifiers to apply while creating the new fork. + """ + + fork_criteria: ForkCriteria + """ + When the new fork is scheduled. + """ + + @property + def new_fork_path(self) -> Path: + """ + The output directory plus the new fork's short name. + """ + output = self.output + if output is None: + raise ValueError( + "no output directory found (set one with --output)" + ) + return output / self.new_fork + + def __init__( + self, + template_fork: str, + new_fork: str, + ) -> None: + self.force = False + + forks = Hardfork.discover() + + # Find the `Hardfork` object named by 'template_fork`. + before = None + found = None + for index, fork in enumerate(forks): + if fork.short_name == template_fork: + found = fork + if index > 0: + before = forks[index - 1] + break + + if found is None: + raise ValueError(f"no fork named `{template_fork}` found") + + if before: + assert before.short_name != new_fork + + self.before_template_fork = before + self.template_fork = found + self.new_fork = new_fork + + # Compute `self.output` based on `template_fork`'s location. + self.output = None + template_path = self.template_fork.path + if template_path is not None: + self.output = Path(template_path).parent + + # Try to make a sane guess for the activation criteria. + if found is forks[-1]: + self.fork_criteria = Unscheduled() + elif hasattr(found.criteria, "timestamp"): + self.fork_criteria = ByTimestamp( + U256(found.criteria.timestamp) + U256(1) + ) + elif hasattr(found.criteria, "block_number"): + self.fork_criteria = ByBlockNumber( + Uint(found.criteria.block_number) + Uint(1) + ) + else: + raise Exception(f"unknown `FORK_CRITERIA` in `{template_fork}`") + + self.modifiers = [ + RenameFork(), + SetForkCriteria(), + ReplaceForkName(), + ] + + def _create_working_directory(self) -> TemporaryDirectory: + """ + Create a temporary working directory so we don't end up in the state + where this process ~~barfs~~ abnormally terminates and we leave a + half-modified fork directory laying around. + """ + output = self.new_fork_path.parent + output.mkdir(parents=True, exist_ok=True) + return TemporaryDirectory(dir=output, prefix=".tmp-fork-") + + def _commit(self, fork_directory: Path) -> None: + if self.force: + rmtree(self.new_fork_path, ignore_errors=True) + fork_directory.rename(self.new_fork_path) + + def _copy(self, fork_directory: Path) -> None: + # TODO: Filter out __pycache__ and similar files that shouldn't be + # copied. + template_path = self.template_fork.path + if template_path is None: + raise Exception( + f"fork `{self.template_fork.short_name}` has no path" + ) + + copytree( + template_path, + fork_directory, + dirs_exist_ok=True, + ) + + def _write_config( + self, config_path: str, working_directory: TemporaryDirectory + ) -> None: + config = { + "generated_code_marker": "@generated", + "blacklist_patterns": [], + "formatter": ["cat", "-"], + "modules": [ + "libcst.codemod.commands", + "ethereum_spec_tools.new_fork.codemod", + ], + "repo_root": working_directory.name, + } + with open(Path(config_path) / ".libcst.codemod.yaml", "w") as f: + json.dump(config, f) # YAML is a superset of JSON + + def _modify(self, working_directory: TemporaryDirectory) -> None: + for modifier in self.modifiers: + for args in modifier._to_args(self, Path(working_directory.name)): + with TemporaryDirectory() as config_directory: + with chdir(config_directory): + self._write_config(config_directory, working_directory) + exit_code = libcst_tool( + "", + args, + ) + if exit_code != 0: + sys.exit(exit_code) + + def build(self) -> None: + """ + Duplicate and transform the template fork into the new fork. + """ + with ExitStack() as exit_stack: + working_directory = self._create_working_directory() + exit_stack.push(working_directory) + + package = Path(working_directory.name) / "ethereum" / self.new_fork + package.mkdir(parents=True) + + self._copy(package) + self._modify(working_directory) + self._commit(package) diff --git a/src/ethereum_spec_tools/new_fork/cli.py b/src/ethereum_spec_tools/new_fork/cli.py new file mode 100644 index 0000000000..8742c6f16d --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/cli.py @@ -0,0 +1,216 @@ +""" +Command-line interface to `ForkBuilder`. +""" + +from argparse import ArgumentParser +from pathlib import Path +from typing import Sequence + +from ethereum_types.numeric import U64, Uint + +from ethereum.fork_criteria import ByBlockNumber, ByTimestamp, Unscheduled + +from ..forks import Hardfork +from .builder import ForkBuilder, SetConstant + + +def _make_parser() -> ArgumentParser: + forks = Hardfork.discover() + fork_short_names = [f.short_name for f in forks] + + parser = ArgumentParser( + # TODO: Description / help text + ) + + parser.add_argument( + "--template-fork", + "--from_fork", + dest="template_fork", + type=str, + choices=fork_short_names, + metavar="NAME", + help="short name of the fork to use as a template", + default=fork_short_names[-1], + ) + + parser.add_argument( + "--new-fork", + "--to_fork", + dest="new_fork", + type=str, + required=True, + metavar="NAME", + help="short name (Python-friendly) of the new fork", + ) + + parser.add_argument( + "--output", + "-o", + dest="output", + type=Path, + metavar="PATH", + help="directory in which to place the generated fork", + ) + + parser.add_argument( + "--force", + "-f", + action="store_true", + help="overwrite the destination if it already exists", + default=False, + ) + + fork_criteria = parser.add_mutually_exclusive_group() + + fork_criteria.add_argument( + "--unscheduled", + action="store_const", + const=Unscheduled(), + dest="fork_criteria", + help="mark the new fork as unscheduled", + ) + + fork_criteria.add_argument( + "--at-timestamp", + type=lambda x: ByTimestamp(int(x)), + dest="fork_criteria", + help="mark the new fork as beginning at the given time", + ) + + fork_criteria.add_argument( + "--at-block", + type=lambda x: ByBlockNumber(int(x)), + dest="fork_criteria", + help="mark the new fork as beginning at the given block number", + ) + + blob_parameters = parser.add_argument_group() + + blob_parameters.add_argument( + "--target-blob-gas-per-block", + type=lambda x: U64(int(x)), + dest="target_blob_gas_per_block", + default=None, + help="Set `TARGET_BLOB_GAS_PER_BLOCK` in the generated fork", + ) + + blob_parameters.add_argument( + "--gas-per-blob", + type=lambda x: U64(int(x)), + dest="gas_per_blob", + default=None, + help="Set `GAS_PER_BLOB` in the generated fork", + ) + + blob_parameters.add_argument( + "--min-blob-gasprice", + type=lambda x: Uint(int(x)), + dest="min_blob_gasprice", + default=None, + help="Set `MIN_BLOB_GASPRICE` in the generated fork", + ) + + blob_parameters.add_argument( + "--blob-base-fee-update-fraction", + type=lambda x: Uint(int(x)), + dest="blob_base_fee_update_fraction", + default=None, + help="Set `BLOB_BASE_FEE_UPDATE_FRACTION` in the generated fork", + ) + + blob_parameters.add_argument( + "--max-blob-gas-per-block", + type=lambda x: U64(int(x)), + dest="max_blob_gas_per_block", + default=None, + help="Set `MAX_BLOB_GAS_PER_BLOCK` in the generated fork", + ) + + blob_parameters.add_argument( + "--blob-schedule-target", + type=lambda x: U64(int(x)), + dest="blob_schedule_target", + default=None, + help="Set `BLOB_SCHEDULE_TARGET` in the generated fork", + ) + + return parser + + +def main(args: Sequence[str] | None = None) -> None: + """ + Command-line entry point into `ForkBuilder`. + """ + parser = _make_parser() + options = parser.parse_args(args) + + if options.template_fork == "spurious_dragon": + raise NotImplementedError( + "An instance of 'Spurious Dragon' in a comment will get " + "incorrectly replaced; use another fork as the template." + ) + + builder = ForkBuilder(options.template_fork, options.new_fork) + + if options.output is not None: + assert isinstance(options.output, Path) + builder.output = options.output + + builder.force = options.force + + if options.fork_criteria is not None: + builder.fork_criteria = options.fork_criteria + + if options.target_blob_gas_per_block is not None: + builder.modifiers.append( + SetConstant( + "vm.gas.TARGET_BLOB_GAS_PER_BLOCK", + repr(options.target_blob_gas_per_block), + ) + ) + + if options.gas_per_blob is not None: + builder.modifiers.append( + SetConstant( + "vm.gas.GAS_PER_BLOB", + repr(options.gas_per_blob), + ) + ) + + if options.min_blob_gasprice is not None: + builder.modifiers.append( + SetConstant( + "vm.gas.MIN_BLOB_GASPRICE", + repr(options.min_blob_gasprice), + ) + ) + + if options.blob_base_fee_update_fraction is not None: + builder.modifiers.append( + SetConstant( + "vm.gas.BLOB_BASE_FEE_UPDATE_FRACTION", + repr(options.blob_base_fee_update_fraction), + ) + ) + + if options.max_blob_gas_per_block is not None: + builder.modifiers.append( + SetConstant( + "fork.MAX_BLOB_GAS_PER_BLOCK", + repr(options.max_blob_gas_per_block), + ) + ) + + if options.blob_schedule_target is not None: + builder.modifiers.append( + SetConstant( + "vm.gas.BLOB_SCHEDULE_TARGET", + repr(options.blob_schedule_target), + ) + ) + + builder.build() + + +if __name__ == "__main__": + main() diff --git a/src/ethereum_spec_tools/new_fork/codemod/__init__.py b/src/ethereum_spec_tools/new_fork/codemod/__init__.py new file mode 100644 index 0000000000..af30fc0690 --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/codemod/__init__.py @@ -0,0 +1,3 @@ +""" +libcst codemods for creating a new fork. +""" diff --git a/src/ethereum_spec_tools/new_fork/codemod/comment.py b/src/ethereum_spec_tools/new_fork/codemod/comment.py new file mode 100644 index 0000000000..df94b61e9c --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/codemod/comment.py @@ -0,0 +1,79 @@ +""" +libcst codemod that replaces text within comments. +""" + +import argparse +import dataclasses +from typing import Sequence + +import libcst as cst +from libcst import matchers as m +from libcst.codemod import CodemodCommand, CodemodContext +from typing_extensions import override + + +class CommentReplaceCommand(CodemodCommand): + """ + Replaces text within comments. + """ + + DESCRIPTION: str = "Replace text within comments." + + replacements: list[tuple[(str, str)]] + + @staticmethod + def add_args(arg_parser: argparse.ArgumentParser) -> None: + """ + Add command-line args that a user can specify for running this codemod. + """ + arg_parser.add_argument( + "--replace", + "-r", + dest="replacements", + metavar="OLD NEW", + action="append", + nargs=2, + help="Text replacement to perform.", + required=True, + ) + + def __init__( + self, + context: CodemodContext, + replacements: list[list[str]] | None, + ) -> None: + super().__init__(context) + if replacements is None: + replacements = [] + self.replacements = [(a, b) for a, b in replacements] + + @override + def transform_module_impl(self, tree: cst.Module) -> cst.Module: + """ + Transform the tree. + """ + result = m.replace( + tree, + m.Comment(), + # `isfunction` returns `False` for bound methods... + lambda a, b: self._replacement(a, b), + ) + assert isinstance(result, cst.Module) + return result + + def _replacement( + self, + node: cst.CSTNode, + extracted: dict[str, cst.CSTNode | Sequence[cst.CSTNode]], + ) -> cst.CSTNode: + del extracted + assert isinstance(node, cst.Comment) + + value = node.value + for old, new in self.replacements: + value = value.replace(old, new) + + if value == node.value: + return node + + return dataclasses.replace(node, value=value) diff --git a/src/ethereum_spec_tools/new_fork/codemod/constant.py b/src/ethereum_spec_tools/new_fork/codemod/constant.py new file mode 100644 index 0000000000..1a502d2a98 --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/codemod/constant.py @@ -0,0 +1,141 @@ +""" +libcst codemod that updates the value of a constant. +""" + +import argparse +from typing import ClassVar, Collection + +import libcst as cst +from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand +from libcst.codemod.visitors import AddImportsVisitor, RemoveImportsVisitor +from libcst.metadata import ( + FullyQualifiedNameProvider, + ParentNodeProvider, + QualifiedName, +) +from libcst.metadata.base_provider import ProviderT +from typing_extensions import override + + +class SetConstantCommand(VisitorBasedCodemodCommand): + """ + Replaces the value of a constant identified by a fully qualified name. + """ + + DESCRIPTION: str = "Replace the value of a constant." + METADATA_DEPENDENCIES: ClassVar[Collection[ProviderT]] = ( + FullyQualifiedNameProvider, + ParentNodeProvider, + ) + + qualified_name: str + value: cst.BaseExpression + imports: list[list[str]] + + _in_assign_target: bool + _matches: bool + + @staticmethod + def add_args(arg_parser: argparse.ArgumentParser) -> None: + """ + Add command-line args that a user can specify for running this codemod. + """ + arg_parser.add_argument( + "--qualified-name", + "-n", + dest="qualified_name", + metavar="NAME", + help="Qualified Python name, like ethereum.osaka.FORK_CRITERIA.", + type=str, + required=True, + ) + arg_parser.add_argument( + "--value", + dest="value", + metavar="VALUE", + help="Replacement value to assign to the qualified name.", + type=str, + required=True, + ) + arg_parser.add_argument( + "--import", + dest="imports", + metavar="FROM IDENT", + action="append", + nargs=2, + help="Additional imports to add to the module.", + ) + + def __init__( + self, + context: CodemodContext, + qualified_name: str, + value: str, + imports: list[list[str]] | None, + ) -> None: + super().__init__(context) + self.qualified_name = qualified_name + self.value = cst.parse_expression(value) + self._in_assign_target = False + self._matches = False + self.imports = imports or [] + + @override + def visit_Assign_targets(self, node: cst.Assign) -> None: # noqa: D102 + if self._in_assign_target: + raise Exception("already in assign target") + self._in_assign_target = True + + @override + def leave_Assign_targets(self, node: cst.Assign) -> None: # noqa: D102 + if not self._in_assign_target: + raise Exception("not in assign target") + self._in_assign_target = False + + @override + def visit_Assign(self, node: cst.Assign) -> None: # noqa: D102 + if self._matches or self._in_assign_target: + raise Exception("nested assign") + + @override + def leave_Assign( # noqa: D102 + self, original_node: cst.Assign, updated_node: cst.Assign + ) -> cst.Assign: + if self._in_assign_target: + raise Exception("still in assign target") + + if not self._matches: + return updated_node + + self._matches = False + + if len(original_node.targets) != 1: + raise NotImplementedError( + "cannot set value of unpacking assignment" + ) + + for module, identifier in self.imports: + AddImportsVisitor.add_needed_import( + self.context, module, identifier + ) + RemoveImportsVisitor.remove_unused_import( + self.context, module, identifier + ) + + return updated_node.with_changes(value=self.value.deep_clone()) + + @override + def visit_Name(self, node: cst.Name) -> None: # noqa: D102 + if not self._in_assign_target: + return + + qualified_names = self.get_metadata(FullyQualifiedNameProvider, node) + if not qualified_names: + return + + for qualified_name in qualified_names: + assert isinstance(qualified_name, QualifiedName) + + if qualified_name.name == self.qualified_name: + self._matches = True + break diff --git a/src/ethereum_spec_tools/new_fork/codemod/string.py b/src/ethereum_spec_tools/new_fork/codemod/string.py new file mode 100644 index 0000000000..f5fbc1f895 --- /dev/null +++ b/src/ethereum_spec_tools/new_fork/codemod/string.py @@ -0,0 +1,79 @@ +""" +libcst codemod that replaces text within strings. +""" + +import argparse +import dataclasses +from typing import Sequence + +import libcst as cst +from libcst import matchers as m +from libcst.codemod import CodemodCommand, CodemodContext +from typing_extensions import override + + +class StringReplaceCommand(CodemodCommand): + """ + Replaces text within strings. + """ + + DESCRIPTION: str = "Replace text within strings." + + replacements: list[tuple[(str, str)]] + + @staticmethod + def add_args(arg_parser: argparse.ArgumentParser) -> None: + """ + Add command-line args that a user can specify for running this codemod. + """ + arg_parser.add_argument( + "--replace", + "-r", + dest="replacements", + metavar="OLD NEW", + action="append", + nargs=2, + help="Text replacement to perform.", + required=True, + ) + + def __init__( + self, + context: CodemodContext, + replacements: list[list[str]] | None, + ) -> None: + super().__init__(context) + if replacements is None: + replacements = [] + self.replacements = [(a, b) for a, b in replacements] + + @override + def transform_module_impl(self, tree: cst.Module) -> cst.Module: + """ + Transform the tree. + """ + result = m.replace( + tree, + m.SimpleString(), + # `isfunction` returns `False` for bound methods... + lambda a, b: self._replacement(a, b), + ) + assert isinstance(result, cst.Module) + return result + + def _replacement( + self, + node: cst.CSTNode, + extracted: dict[str, cst.CSTNode | Sequence[cst.CSTNode]], + ) -> cst.CSTNode: + del extracted + assert isinstance(node, cst.SimpleString) + + value = node.value + for old, new in self.replacements: + value = value.replace(old, new) + + if value == node.value: + return node + + return dataclasses.replace(node, value=value) diff --git a/tests/json_infra/test_tools_new_fork.py b/tests/json_infra/test_tools_new_fork.py new file mode 100644 index 0000000000..0160c0a782 --- /dev/null +++ b/tests/json_infra/test_tools_new_fork.py @@ -0,0 +1,73 @@ +""" +Tests for the ethereum-spec-new-fork CLI tool. +""" + +from pathlib import Path +from tempfile import TemporaryDirectory + +from ethereum_spec_tools.new_fork.cli import main as new_fork + + +def test_end_to_end() -> None: + """ + Test that the ethereum-spec-new-fork CLI tool creates a fork from a + template, correctly modifying names, blob parameters, and imports. + """ + with TemporaryDirectory() as base_dir: + output_dir = Path(base_dir) / "ethereum" + fork_dir = output_dir / "e2e_fork" + + new_fork( + [ + "--new-fork", + "e2e_fork", + "--template-fork", + "osaka", + "--target-blob-gas-per-block", + "199", + "--blob-base-fee-update-fraction", + "750", + "--min-blob-gasprice", + "2", + "--gas-per-blob", + "1", + "--at-timestamp", + "7", + "--max-blob-gas-per-block", + "99", + "--blob-schedule-target", + "88", + "--output", + str(output_dir), + ] + ) + + with (fork_dir / "__init__.py").open("r") as f: + source = f.read() + + assert "FORK_CRITERIA = ByTimestamp(7)" in source + assert "E2E Fork" in source + assert "Osaka" not in source + + with (fork_dir / "vm" / "gas.py").open("r") as f: + source = f.read() + + expected = [ + "TARGET_BLOB_GAS_PER_BLOCK = U64(199)", + "GAS_PER_BLOB = U64(1)", + "MIN_BLOB_GASPRICE = Uint(2)", + "BLOB_BASE_FEE_UPDATE_FRACTION = Uint(750)", + "BLOB_SCHEDULE_TARGET = U64(88)", + ] + + for needle in expected: + assert needle in source + + with (fork_dir / "fork.py").open("r") as f: + assert "MAX_BLOB_GAS_PER_BLOCK = U64(99)" in f.read() + + with (fork_dir / "trie.py").open("r") as f: + assert ( + "from ethereum.forks.paris import trie as previous_trie" + in f.read() + ) diff --git a/uv.lock b/uv.lock index e24a5b8cca..6c0aad0575 100644 --- a/uv.lock +++ b/uv.lock @@ -566,6 +566,7 @@ optimized = [ test = [ { name = "filelock" }, { name = "gitpython" }, + { name = "libcst" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-xdist" }, @@ -573,6 +574,7 @@ test = [ { name = "requests-cache" }, ] tools = [ + { name = "libcst" }, { name = "platformdirs" }, ] @@ -598,6 +600,8 @@ requires-dist = [ { name = "gitpython", marker = "extra == 'fill'", specifier = ">=3.1.31,<4" }, { name = "gitpython", marker = "extra == 'test'", specifier = ">=3.1.0,<3.2" }, { name = "joblib", marker = "extra == 'fill'", specifier = ">=1.4.2" }, + { name = "libcst", marker = "extra == 'test'", specifier = ">=1.8,<2" }, + { name = "libcst", marker = "extra == 'tools'", specifier = ">=1.8,<2" }, { name = "mypy", marker = "extra == 'lint'", specifier = "==1.17.0" }, { name = "platformdirs", marker = "extra == 'tools'", specifier = ">=4.2,<5" }, { name = "py-ecc", specifier = ">=8.0.0b2,<9" }, diff --git a/vulture_whitelist.py b/vulture_whitelist.py index 0367458374..ef0e121bb3 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -19,6 +19,9 @@ GlacierForksHygiene, ) from ethereum_spec_tools.lint.lints.import_hygiene import ImportHygiene +from ethereum_spec_tools.new_fork.codemod.constant import SetConstantCommand +from ethereum_spec_tools.new_fork.codemod.string import StringReplaceCommand +from ethereum_spec_tools.new_fork.codemod.comment import CommentReplaceCommand from ethereum.trace import EvmTracer # src/ethereum/utils/hexadecimal.py @@ -110,5 +113,20 @@ ImportHygiene ImportHygiene.visit_AnnAssign +# src/ethereum_spec_tools/new_fork/codemod/constant.py +SetConstantCommand +SetConstantCommand.METADATA_DEPENDENCIES +SetConstantCommand.add_args +SetConstantCommand.visit_Assign_targets +SetConstantCommand.leave_Assign_targets +SetConstantCommand.leave_Assign + +# src/ethereum_spec_tools/new_fork/codemod/string.py +StringReplaceCommand +StringReplaceCommand.transform_module_impl + +# src/ethereum_spec_tools/new_fork/codemod/comment.py +CommentReplaceCommand +CommentReplaceCommand.transform_module_impl _children # unused attribute (src/ethereum_spec_tools/docc.py:751) diff --git a/whitelist.txt b/whitelist.txt index 5e5868bb3e..1a74485646 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -1,10 +1,12 @@ AnnAssign AsyncFunctionDef +codemod ClassDef ImportFrom radd Hash64 holiman +backtrace Bytes20 Bytes4 Bytes32 @@ -13,6 +15,11 @@ Bytes8 Bytes256 Bytes0 calldata +copy2 +copystat +pycache +dirs +codemod copyreg copytree coinbase @@ -42,10 +49,13 @@ isabstract keccak keccak256 keccak512 +libcst linter lru mutable256 mainnet +matchers +isfunction memoize memoization merkle @@ -73,6 +83,7 @@ secp256k1p secp256k1b statetest subclasses +superset iadd ispkg isub @@ -165,6 +176,7 @@ number difficulty gaslimit chainid +chdir selfbalance basefee ommer