From 550d6196f5f203ed8f228b22d2cf536836edd4f4 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Wed, 25 Jun 2025 15:53:59 +0100 Subject: [PATCH 1/8] chore: rename zkevm tests to benchmark. --- .github/configs/evm.yaml | 5 +++++ docs/CHANGELOG.md | 1 + tests/{zkevm => benchmark}/__init__.py | 0 tests/{zkevm => benchmark}/conftest.py | 0 tests/{zkevm => benchmark}/helpers.py | 0 tests/{zkevm => benchmark}/test_worst_blocks.py | 0 tests/{zkevm => benchmark}/test_worst_bytecode.py | 0 tests/{zkevm => benchmark}/test_worst_compute.py | 0 tests/{zkevm => benchmark}/test_worst_opcode.py | 0 tests/{zkevm => benchmark}/test_worst_stateful_opcodes.py | 0 10 files changed, 6 insertions(+) rename tests/{zkevm => benchmark}/__init__.py (100%) rename tests/{zkevm => benchmark}/conftest.py (100%) rename tests/{zkevm => benchmark}/helpers.py (100%) rename tests/{zkevm => benchmark}/test_worst_blocks.py (100%) rename tests/{zkevm => benchmark}/test_worst_bytecode.py (100%) rename tests/{zkevm => benchmark}/test_worst_compute.py (100%) rename tests/{zkevm => benchmark}/test_worst_opcode.py (100%) rename tests/{zkevm => benchmark}/test_worst_stateful_opcodes.py (100%) diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index ca518e94e6b..d29f8a2e01f 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -16,6 +16,11 @@ zkevm: repo: ethereum/evmone ref: master targets: ["evmone-t8n", "evmone-eofparse"] +benchmark: + impl: evmone + repo: ethereum/evmone + ref: master + targets: ["evmone-t8n", "evmone-eofparse"] eip7692: impl: evmone repo: ethereum/evmone diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ab2dff2aac1..06be6c4d8c5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -72,6 +72,7 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679), [#1718](https://github.com/ethereum/execution-spec-tests/pull/1718)). - ๐Ÿ”€ Add macOS hive development mode workaround to the docs [#1786](https://github.com/ethereum/execution-spec-tests/pull/1786). - ๐Ÿ”€ Refactor and clean up of exceptions including EOF exceptions within client specific mappers [#1803](https://github.com/ethereum/execution-spec-tests/pull/1803). +- ๐Ÿ”€ Rename zkevm tests to benchmark while maintaining the zkevm marker [#](https://github.com/ethereum/execution-spec-tests/pull/). ### ๐Ÿงช Test Cases diff --git a/tests/zkevm/__init__.py b/tests/benchmark/__init__.py similarity index 100% rename from tests/zkevm/__init__.py rename to tests/benchmark/__init__.py diff --git a/tests/zkevm/conftest.py b/tests/benchmark/conftest.py similarity index 100% rename from tests/zkevm/conftest.py rename to tests/benchmark/conftest.py diff --git a/tests/zkevm/helpers.py b/tests/benchmark/helpers.py similarity index 100% rename from tests/zkevm/helpers.py rename to tests/benchmark/helpers.py diff --git a/tests/zkevm/test_worst_blocks.py b/tests/benchmark/test_worst_blocks.py similarity index 100% rename from tests/zkevm/test_worst_blocks.py rename to tests/benchmark/test_worst_blocks.py diff --git a/tests/zkevm/test_worst_bytecode.py b/tests/benchmark/test_worst_bytecode.py similarity index 100% rename from tests/zkevm/test_worst_bytecode.py rename to tests/benchmark/test_worst_bytecode.py diff --git a/tests/zkevm/test_worst_compute.py b/tests/benchmark/test_worst_compute.py similarity index 100% rename from tests/zkevm/test_worst_compute.py rename to tests/benchmark/test_worst_compute.py diff --git a/tests/zkevm/test_worst_opcode.py b/tests/benchmark/test_worst_opcode.py similarity index 100% rename from tests/zkevm/test_worst_opcode.py rename to tests/benchmark/test_worst_opcode.py diff --git a/tests/zkevm/test_worst_stateful_opcodes.py b/tests/benchmark/test_worst_stateful_opcodes.py similarity index 100% rename from tests/zkevm/test_worst_stateful_opcodes.py rename to tests/benchmark/test_worst_stateful_opcodes.py From e0de57afe7a4618d76a41a2f6ac6d6378a2faa07 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Wed, 25 Jun 2025 16:07:22 +0100 Subject: [PATCH 2/8] chore(docs): changelog. --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 06be6c4d8c5..46b9968d79f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -72,7 +72,7 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679), [#1718](https://github.com/ethereum/execution-spec-tests/pull/1718)). - ๐Ÿ”€ Add macOS hive development mode workaround to the docs [#1786](https://github.com/ethereum/execution-spec-tests/pull/1786). - ๐Ÿ”€ Refactor and clean up of exceptions including EOF exceptions within client specific mappers [#1803](https://github.com/ethereum/execution-spec-tests/pull/1803). -- ๐Ÿ”€ Rename zkevm tests to benchmark while maintaining the zkevm marker [#](https://github.com/ethereum/execution-spec-tests/pull/). +- ๐Ÿ”€ Rename zkevm tests to benchmark while maintaining the zkevm marker [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804). ### ๐Ÿงช Test Cases From 2e98df6cfd6587d669d9097917475e1e48376457 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Jun 2025 11:48:40 +0200 Subject: [PATCH 3/8] refactor(tests,fw,tox,docs): rename zkevm marker to benchmark --- .github/workflows/tox_verify.yaml | 6 +++--- docs/getting_started/code_standards.md | 2 +- docs/running_tests/releases.md | 2 +- src/ethereum_test_specs/base.py | 4 ++-- src/pytest_plugins/shared/execute_fill.py | 2 +- tests/benchmark/conftest.py | 6 +++--- tests/benchmark/test_worst_compute.py | 2 -- tox.ini | 20 ++++++++++---------- 8 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tox_verify.yaml b/.github/workflows/tox_verify.yaml index abbd92fe54c..cb3f40b07ec 100644 --- a/.github/workflows/tox_verify.yaml +++ b/.github/workflows/tox_verify.yaml @@ -182,8 +182,8 @@ jobs: env: EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json run: uvx --with=tox-uv tox -e tests-deployed - tests_deployed_zkevm: - name: Fill zkEVM test cases, deployed, ${{ matrix.os }}, ${{ matrix.python }} + tests_deployed_benchmark: + name: Fill benchmark test cases, deployed, ${{ matrix.os }}, ${{ matrix.python }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -209,4 +209,4 @@ jobs: - name: Run tox - fill tests for deployed forks env: EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json - run: uvx --with=tox-uv tox -e tests-deployed-zkevm + run: uvx --with=tox-uv tox -e tests-deployed-benchmark diff --git a/docs/getting_started/code_standards.md b/docs/getting_started/code_standards.md index b289f58b945..63148bf9e21 100644 --- a/docs/getting_started/code_standards.md +++ b/docs/getting_started/code_standards.md @@ -12,7 +12,7 @@ Code pushed to @ethereum/execution-spec-tests must fulfill the following checks | Typecheck | `uvx --with=tox-uv tox -e typecheck` | Objects that provide typehints pass type-checking via `mypy`. | | Framework unit tests | `uvx --with=tox-uv tox -e pytest` | All framework unit tests must execute correctly. | | EL Client test cases | `uvx --with=tox-uv tox -e tests-deployed` | All client test cases for deployed forks can be generated. | -| zkEVM EL Test cases | `uvx --with=tox-uv tox -e tests-deployed-zkevm` | All client test cases specific to zkEVMs for deployed forks can be generated. | +| Benchmark EL Test cases | `uvx --with=tox-uv tox -e tests-deployed-benchmark` | All client test cases specific to benchmarks for deployed forks can be generated. | | HTML doc build | `uvx --with=tox-uv tox -e mkdocs` | Documentation generated without warnings. | | Spellcheck | `uvx --with=tox-uv tox -e spellcheck` | Code and documentation spell-check using codespell. | | Markdown lint | `uvx --with=tox-uv tox -e markdownlint` | Markdown lint (requires [additional dependency](code_standards_details.md#additional-dependencies)). | diff --git a/docs/running_tests/releases.md b/docs/running_tests/releases.md index 057bdb14b4a..6df157f17a4 100644 --- a/docs/running_tests/releases.md +++ b/docs/running_tests/releases.md @@ -46,7 +46,7 @@ These releases are tagged using the format `@vX.Y.Z`. Examples: - [`fusaka-devnet-1@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/fusaka-devnet-1%40v1.0.0) - this fixture release contains tests adhering to the [Fusaka Devnet 1 spec](https://notes.ethereum.org/@ethpandaops/fusaka-devnet-1). -- [`zkevm@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/zkevm%40v0.1.0) - this fixture release contains tests specifically aimed at testing zkEVMs. +- [`zkevm@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/zkevm%40v0.1.0) - this fixture release contains tests specifically aimed at benchmarking EVMs. Devnet releases should be treated as WIP and may not yet contain full test coverage (or even coverage for all EIPs). The coverage provided by these releases is detailed in the corresponding release notes. diff --git a/src/ethereum_test_specs/base.py b/src/ethereum_test_specs/base.py index ccd6739e1a2..dce7a5f4b66 100644 --- a/src/ethereum_test_specs/base.py +++ b/src/ethereum_test_specs/base.py @@ -161,8 +161,8 @@ def is_tx_gas_heavy_test(self) -> bool: if self._request is not None and hasattr(self._request, "node"): node = self._request.node has_slow_marker = node.get_closest_marker("slow") is not None - has_zkevm_marker = node.get_closest_marker("zkevm") is not None - return has_slow_marker or has_zkevm_marker + has_benchmark_marker = node.get_closest_marker("benchmark") is not None + return has_slow_marker or has_benchmark_marker return False def is_exception_test(self) -> bool | None: diff --git a/src/pytest_plugins/shared/execute_fill.py b/src/pytest_plugins/shared/execute_fill.py index 0c6b4106358..db0693ea6a8 100644 --- a/src/pytest_plugins/shared/execute_fill.py +++ b/src/pytest_plugins/shared/execute_fill.py @@ -83,7 +83,7 @@ def pytest_configure(config: pytest.Config): ) config.addinivalue_line( "markers", - "zkevm: Tests that are relevant to zkEVM.", + "benchmark: Tests relevant to benchmarking EVMs.", ) config.addinivalue_line( "markers", diff --git a/tests/benchmark/conftest.py b/tests/benchmark/conftest.py index 81da9c97224..4f93f813d3e 100644 --- a/tests/benchmark/conftest.py +++ b/tests/benchmark/conftest.py @@ -1,4 +1,4 @@ -"""Pytest configuration for zkEVM tests.""" +"""Pytest configuration for benchmark tests.""" from pathlib import Path @@ -6,7 +6,7 @@ def pytest_collection_modifyitems(config, items): - """Add the `zkevm` marker to all tests under `./tests/zkevm`.""" + """Add the `benchmark` marker to all tests under `./tests/benchmark`.""" for item in items: if Path(__file__).parent in Path(item.fspath).parents: - item.add_marker(pytest.mark.zkevm) + item.add_marker(pytest.mark.benchmark) diff --git a/tests/benchmark/test_worst_compute.py b/tests/benchmark/test_worst_compute.py index 512a3d9f505..e6924a93835 100644 --- a/tests/benchmark/test_worst_compute.py +++ b/tests/benchmark/test_worst_compute.py @@ -1136,7 +1136,6 @@ def test_worst_precompile_fixed_cost( ) -@pytest.mark.zkevm @pytest.mark.valid_from("Cancun") @pytest.mark.slow def test_worst_jumps(state_test: StateTestFiller, pre: Alloc): @@ -1160,7 +1159,6 @@ def test_worst_jumps(state_test: StateTestFiller, pre: Alloc): ) -@pytest.mark.zkevm @pytest.mark.valid_from("Cancun") @pytest.mark.slow def test_worst_jumpdests(state_test: StateTestFiller, pre: Alloc, fork: Fork): diff --git a/tox.ini b/tox.ini index 7a13a476550..e39e3d30b84 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ env_list = spellcheck pytest tests-deployed - tests-deployed-zkevm + tests-deployed-benchmark mkdocs [testenv] @@ -81,17 +81,17 @@ develop = Prague eip7692 = EOFv1 [testenv:tests-deployed] -description = Fill test cases in ./tests/ for deployed mainnet forks, except for slow/zkevm. +description = Fill test cases in ./tests/ for deployed mainnet forks, except for slow/benchmark tests. setenv = # Use custom EELS_RESOLUTIONS_FILE if it is set via the environment (eg, in CI) EELS_RESOLUTIONS_FILE = {env:EELS_RESOLUTIONS_FILE:} commands_pre = solc-select use {[testenv]solc_version} --always-install -commands = pytest -n auto -m "not slow and not zkevm" --skip-evm-dump --output=/tmp/fixtures-tox --clean +commands = pytest -n auto -m "not slow and not benchmark" --skip-evm-dump --output=/tmp/fixtures-tox --clean -[testenv:tests-deployed-zkevm] -description = Fill zkEVM test cases in ./tests/ for deployed mainnet forks, using evmone-t8n. +[testenv:tests-deployed-benchmark] +description = Fill benchmarking test cases in ./tests/ for deployed mainnet forks, using evmone-t8n. commands_pre = solc-select use {[testenv]solc_version} --always-install -commands = pytest -n auto -m "zkevm" --skip-evm-dump --block-gas-limit 36000000 --output=/tmp/fixtures-tox --clean --evm-bin=evmone-t8n +commands = pytest -n auto -m "benchmark" --skip-evm-dump --block-gas-limit 36000000 --output=/tmp/fixtures-tox --clean --evm-bin=evmone-t8n [testenv:tests-develop] description = Fill test cases in ./tests/ for deployed and development mainnet forks @@ -99,7 +99,7 @@ setenv = # Use custom EELS_RESOLUTIONS_FILE if it is set via the environment (eg, in CI) EELS_RESOLUTIONS_FILE = {env:EELS_RESOLUTIONS_FILE:} commands_pre = solc-select use {[testenv]solc_version} --always-install -commands = pytest -n auto --until={[forks]develop} -k "not slow and not zkevm" --skip-evm-dump --output=/tmp/fixtures-tox --clean +commands = pytest -n auto --until={[forks]develop} -k "not slow and not benchmark" --skip-evm-dump --output=/tmp/fixtures-tox --clean # ---------------------------------------------------------------------------------------------- # ALIAS ENVIRONMENTS @@ -137,18 +137,18 @@ extras = {[testenv:typecheck]extras} {[testenv:spellcheck]extras} {[testenv:tests-deployed]extras} - {[testenv:tests-deployed-zkevm]extras} + {[testenv:tests-deployed-benchmark]extras} setenv = {[testenv:pytest]setenv} commands_pre = {[testenv:tests-deployed]:commands_pre} - {[testenv:tests-deployed-zkevm]:commands_pre} + {[testenv:tests-deployed-benchmark]:commands_pre} commands = {[testenv:lint]commands} {[testenv:typecheck]commands} {[testenv:spellcheck]commands} {[testenv:tests-deployed]commands} - {[testenv:tests-deployed-zkevm]commands} + {[testenv:tests-deployed-benchmark]commands} # ALIAS that runs checks on ./docs/: spellcheck, markdownlint, mkdocs # uvx --with=tox-uv tox -e spellcheck,markdownlint,mkdocs From 7ef43443cb8fdfe5b774879ee9d5007a8586f1ad Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 26 Jun 2025 12:03:31 +0200 Subject: [PATCH 4/8] chore(tests): remove zkevm specific nomenclature from docstrings --- tests/benchmark/test_worst_blocks.py | 6 +++--- tests/benchmark/test_worst_bytecode.py | 6 +++--- tests/benchmark/test_worst_compute.py | 6 +++--- tests/benchmark/test_worst_stateful_opcodes.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/benchmark/test_worst_blocks.py b/tests/benchmark/test_worst_blocks.py index 7a2944e2bb9..fc7a5a6bca0 100644 --- a/tests/benchmark/test_worst_blocks.py +++ b/tests/benchmark/test_worst_blocks.py @@ -1,8 +1,8 @@ """ -abstract: Tests zkEVMs worst-case block scenarios. - Tests zkEVMs worst-case block scenarios. +abstract: Tests that benchmark EVMs in worst-case block scenarios. + Tests that benchmark EVMs in worst-case block scenarios. -Tests running worst-case block scenarios for zkEVMs. +Tests running worst-case block scenarios for EVMs. """ import pytest diff --git a/tests/benchmark/test_worst_bytecode.py b/tests/benchmark/test_worst_bytecode.py index ae0d2364939..90f4decd82d 100644 --- a/tests/benchmark/test_worst_bytecode.py +++ b/tests/benchmark/test_worst_bytecode.py @@ -1,8 +1,8 @@ """ -abstract: Tests for zkEVMs - Tests for zkEVMs. +abstract: Tests that benchmark EVMs in worst-case opcode scenarios. + Tests that benchmark EVMs in worst-case opcode scenarios. -Tests for zkEVMs worst-cases scenarios. +Tests that benchmark EVMs in worst-case opcode scenarios. """ import math diff --git a/tests/benchmark/test_worst_compute.py b/tests/benchmark/test_worst_compute.py index e6924a93835..a450e05a048 100644 --- a/tests/benchmark/test_worst_compute.py +++ b/tests/benchmark/test_worst_compute.py @@ -1,8 +1,8 @@ """ -abstract: Tests zkEVMs worst-case compute scenarios. - Tests zkEVMs worst-case compute scenarios. +abstract: Tests that benchmark EVMs in worst-case compute scenarios. + Tests that benchmark EVMs in worst-case compute scenarios. -Tests running worst-case compute opcodes and precompile scenarios for zkEVMs. +Tests that benchmark EVMs when running worst-case compute opcodes and precompile scenarios. """ import math diff --git a/tests/benchmark/test_worst_stateful_opcodes.py b/tests/benchmark/test_worst_stateful_opcodes.py index 9daecba5c6e..044590df31f 100644 --- a/tests/benchmark/test_worst_stateful_opcodes.py +++ b/tests/benchmark/test_worst_stateful_opcodes.py @@ -1,8 +1,8 @@ """ -abstract: Tests zkEVMs worst-case stateful opcodes. - Tests zkEVMs worst-case stateful opcodes. +abstract: Tests that benchmark EVMs for worst-case stateful opcodes. + Tests that benchmark EVMs for worst-case stateful opcodes. -Tests running worst-case stateful opcodes for zkEVMs. +Tests that benchmark EVMs for worst-case stateful opcodes. """ import math From cc42843d3f9d9515d08d24cc183221548f5a7293 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Thu, 26 Jun 2025 13:03:14 +0100 Subject: [PATCH 5/8] chore(ci|docs): rename zkevm releases to benchmark. --- .github/configs/evm.yaml | 5 ----- .github/configs/feature.yaml | 24 ++++++++++++------------ docs/CHANGELOG.md | 20 ++++++++++---------- docs/running_tests/releases.md | 2 +- whitelist.txt | 3 --- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index d29f8a2e01f..5ea0f99ce3c 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -11,11 +11,6 @@ static: repo: ethereum/evmone ref: master targets: ["evmone-t8n", "evmone-eofparse"] -zkevm: - impl: evmone - repo: ethereum/evmone - ref: master - targets: ["evmone-t8n", "evmone-eofparse"] benchmark: impl: evmone repo: ethereum/evmone diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index a960cc51054..cc65993c47a 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -11,24 +11,24 @@ static: evm-type: static fill-params: --until=Prague --fill-static-tests ./tests/static solc: 0.8.21 -zkevm_36M: - evm-type: zkevm - fill-params: --from=Cancun --until=Prague --block-gas-limit 36000000 -m zkevm ./tests +benchmark_36M: + evm-type: benchmark + fill-params: --from=Cancun --until=Prague --block-gas-limit 36000000 -m benchmark ./tests solc: 0.8.21 feature_only: true -zkevm_60M: - evm-type: zkevm - fill-params: --from=Cancun --until=Prague --block-gas-limit 60000000 -m zkevm ./tests +benchmark_60M: + evm-type: benchmark + fill-params: --from=Cancun --until=Prague --block-gas-limit 60000000 -m benchmark ./tests solc: 0.8.21 feature_only: true -zkevm_90M: - evm-type: zkevm - fill-params: --from=Cancun --until=Prague --block-gas-limit 90000000 -m zkevm ./tests +benchmark_90M: + evm-type: benchmark + fill-params: --from=Cancun --until=Prague --block-gas-limit 90000000 -m benchmark ./tests solc: 0.8.21 feature_only: true -zkevm_120M: - evm-type: zkevm - fill-params: --from=Cancun --until=Prague --block-gas-limit 120000000 -m zkevm ./tests +benchmark_120M: + evm-type: benchmark + fill-params: --from=Cancun --until=Prague --block-gas-limit 120000000 -m benchmark ./tests solc: 0.8.21 feature_only: true eip7692: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 46b9968d79f..bda0fd27054 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,9 +17,9 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e - Before: `fill` wrote fixtures into the directory specified by the `--output` flag (default: `fixtures`). This could have many unintended consequences, including unexpected errors if old or invalid fixtures existed in the directory (for details see [#1030](https://github.com/ethereum/execution-spec-tests/issues/1030)). - Now: `fill` will exit without filling any tests if the specified directory exists and is not-empty. This may be overridden by adding the `--clean` flag, which will first remove the specified directory. -#### Feature `zkevm` expanded to `zkevm_36M`, `zkevm_60M`, `zkevm_90M`, `zkevm_120M` +#### Feature `benchmark` expanded to `benchmark_36M`, `benchmark_60M`, `benchmark_90M`, `benchmark_120M` -Renames `zkevm` to `zkevm_36M` and further expands the zkEVM features to run using 60M, 90M and 120M block gas limits in `fixtures_zkevm_36M.tar.gz`, `fixtures_zkevm_60M.tar.gz`, `fixtures_zkevm_90M.tar.gz` and `fixtures_zkevm_120M.tar.gz` respectively. +Renames `benchmark` to `benchmark_36M` and further expands the benchmark features to run using 60M, 90M and 120M block gas limits in `fixtures_benchmark_36M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz` and `fixtures_benchmark_120M.tar.gz` respectively. Users can select any of the artifacts depending on their testing needs for their provers. @@ -36,9 +36,9 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Add a new pytest plugin, `ported_tests`, that lists the static fillers and PRs from `ported_from` markers for use in the coverage Github Workflow ([#1634](https://github.com/ethereum/execution-spec-tests/pull/1634)). - โœจ Enable two-phase filling of fixtures with pre-allocation groups and add a `BlockchainEngineXFixture` format ([#1706](https://github.com/ethereum/execution-spec-tests/pull/1706), [#1760](https://github.com/ethereum/execution-spec-tests/pull/1760)). - ๐Ÿ”€ Refactor: Encapsulate `fill`'s fixture output options (`--output`, `--flat-output`, `--single-fixture-per-file`) into a `FixtureOutput` class ([#1471](https://github.com/ethereum/execution-spec-tests/pull/1471),[#1612](https://github.com/ethereum/execution-spec-tests/pull/1612)). -- โœจ Don't warn about a "high Transaction gas_limit" for `zkevm` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)). +- โœจ Don't warn about a "high Transaction gas_limit" for `benchmark` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)). - ๐Ÿž `fill` no longer writes generated fixtures into an existing, non-empty output directory; it must now be empty or `--clean` must be used to delete it first ([#1608](https://github.com/ethereum/execution-spec-tests/pull/1608)). -- ๐Ÿž zkEVM marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-zkevm` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)). +- ๐Ÿž benchmark marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-benchmark` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)). - โœจ Field `postStateHash` is now added to all `blockchain_test` and `blockchain_test_engine` tests that use `exclude_full_post_state_in_output` in place of `postState`. Fixes `evmone-blockchaintest` test consumption and indirectly fixes coverage runs for these tests ([#1667](https://github.com/ethereum/execution-spec-tests/pull/1667)). - ๐Ÿ”€ Changed INVALID_DEPOSIT_EVENT_LAYOUT to a BlockException instead of a TransactionException ([#1773](https://github.com/ethereum/execution-spec-tests/pull/1773)). @@ -72,7 +72,7 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679), [#1718](https://github.com/ethereum/execution-spec-tests/pull/1718)). - ๐Ÿ”€ Add macOS hive development mode workaround to the docs [#1786](https://github.com/ethereum/execution-spec-tests/pull/1786). - ๐Ÿ”€ Refactor and clean up of exceptions including EOF exceptions within client specific mappers [#1803](https://github.com/ethereum/execution-spec-tests/pull/1803). -- ๐Ÿ”€ Rename zkevm tests to benchmark while maintaining the zkevm marker [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804). +- ๐Ÿ”€ Rename `zkevm` tests to `benchmark` while maintaining the `zkevm` marker [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804). ### ๐Ÿงช Test Cases @@ -83,7 +83,7 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ EIP-7823, EIP-7883: Add test cases for ModExp precompile gas-cost updates and input limits on Osaka ([#1579](https://github.com/ethereum/execution-spec-tests/pull/1579), [#1729](https://github.com/ethereum/execution-spec-tests/pull/1729)). - โœจ [EIP-7825](https://eips.ethereum.org/EIPS/eip-7825): Add test cases for the transaction gas limit of 30M gas ([#1711](https://github.com/ethereum/execution-spec-tests/pull/1711)). - โœจ [EIP-7951](https://eips.ethereum.org/EIPS/eip-7951): add test cases for `P256VERIFY` precompile to support secp256r1 curve [#1670](https://github.com/ethereum/execution-spec-tests/pull/1670). -- โœจ Introduce blockchain tests for ZKEVM to cover the scenario of pure ether transfers [#1742](https://github.com/ethereum/execution-spec-tests/pull/1742). +- โœจ Introduce blockchain tests for benchmark to cover the scenario of pure ether transfers [#1742](https://github.com/ethereum/execution-spec-tests/pull/1742). - โœจ [EIP-7934](https://eips.ethereum.org/EIPS/eip-7934): Add test cases for the block RLP max limit of 10MiB ([#1730](https://github.com/ethereum/execution-spec-tests/pull/1730)). - โœจ [EIP-7939](https://eips.ethereum.org/EIPS/eip-7939) Add count leading zeros (CLZ) opcode tests for Osaka ([#1733](https://github.com/ethereum/execution-spec-tests/pull/1733)). - โœจ [EIP-7918](https://eips.ethereum.org/EIPS/eip-7918): Blob base fee bounded by execution cost test cases (initial), includes some adjustments to [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) tests ([#1685](https://github.com/ethereum/execution-spec-tests/pull/1685)). @@ -110,12 +110,12 @@ A new fork `EOFv1` has additionally been created to fill and consume EOF related - โœจ Engine API updates for Osaka, add `get_blobs` rpc method ([#1510](https://github.com/ethereum/execution-spec-tests/pull/1510)). - โœจ The EIP Version checker has been moved from `fill` and `execute` to it's own command-line tool `check_eip_versions` that gets ran daily as a Github Action ([#1537](https://github.com/ethereum/execution-spec-tests/pull/1537)). - ๐Ÿ”€ Add new `tests/unscheduled` folder, move EOF from Osaka to unscheduled, add `EOFv1` fork name for EOF tests ([#1507](https://github.com/ethereum/execution-spec-tests/pull/1507)). -- โœจ CI features now contain an optional field to skip them from EEST full releases, `zkevm` and EOF features are now feature only ([#1596](https://github.com/ethereum/execution-spec-tests/pull/1596)). +- โœจ CI features now contain an optional field to skip them from EEST full releases, `benchmark` and EOF features are now feature only ([#1596](https://github.com/ethereum/execution-spec-tests/pull/1596)). - ๐Ÿž Don't attempt to install `solc` via `solc-select` on ARM (official Linux ARM builds of `solc` are not available at the time of writing, cf [ethereum/solidity#11351](https://github.com/ethereum/solidity/issues/11351)) and add a version sanity check ([#1556](https://github.com/ethereum/execution-spec-tests/pull/1556)). ### ๐Ÿงช Test Cases -- ๐Ÿ”€ Automatically apply the `zkevm` marker to all tests under `./tests/zkevm/` and `./tests/prague/eip2537_bls_12_381_precompiles/` via conftest configuration ([#1534](https://github.com/ethereum/execution-spec-tests/pull/1534)). +- ๐Ÿ”€ Automatically apply the `benchmark` marker to all tests under `./tests/benchmark/` and `./tests/prague/eip2537_bls_12_381_precompiles/` via conftest configuration ([#1534](https://github.com/ethereum/execution-spec-tests/pull/1534)). - โœจ Port [calldataload](https://github.com/ethereum/tests/blob/ae4791077e8fcf716136e70fe8392f1a1f1495fb/src/GeneralStateTestsFiller/VMTests/vmTests/calldatacopyFiller.yml) and [calldatasize](https://github.com/ethereum/tests/blob/81862e4848585a438d64f911a19b3825f0f4cd95/src/GeneralStateTestsFiller/VMTests/vmTests/calldatasizeFiller.yml) tests ([#1236](https://github.com/ethereum/execution-spec-tests/pull/1236)). ## [v4.4.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v4.4.0) - 2025-04-29 @@ -132,9 +132,9 @@ The tests have been filled using the new static test filler introduced in [#1336 Users can expect that all tests currently living in [ethereum/tests](https://github.com/ethereum/tests/tree/develop/src) should eventually make its way into [`./tests/static`](https://github.com/ethereum/execution-spec-tests/tree/main/tests/static) and can rely that these tests, filled for new forks even, will be included in `fixtures_static.tar.gz`. -#### `fixtures_zkevm` +#### `fixtures_benchmark` -Another new fixture tarball has been included in this release: `fixtures_zkevm.tar.gz`. +Another new fixture tarball has been included in this release: `fixtures_benchmark.tar.gz`. Includes tests that are tailored specifically to test the execution layer proof generators. diff --git a/docs/running_tests/releases.md b/docs/running_tests/releases.md index 6df157f17a4..c01d8df2e6a 100644 --- a/docs/running_tests/releases.md +++ b/docs/running_tests/releases.md @@ -46,7 +46,7 @@ These releases are tagged using the format `@vX.Y.Z`. Examples: - [`fusaka-devnet-1@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/fusaka-devnet-1%40v1.0.0) - this fixture release contains tests adhering to the [Fusaka Devnet 1 spec](https://notes.ethereum.org/@ethpandaops/fusaka-devnet-1). -- [`zkevm@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/zkevm%40v0.1.0) - this fixture release contains tests specifically aimed at benchmarking EVMs. +- [`benchmark@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/benchmark%40v0.1.0) - this fixture release contains tests specifically aimed at benchmarking EVMs. Devnet releases should be treated as WIP and may not yet contain full test coverage (or even coverage for all EIPs). The coverage provided by these releases is detailed in the corresponding release notes. diff --git a/whitelist.txt b/whitelist.txt index eaea73a7252..e9c43a2c6e9 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -524,8 +524,6 @@ yml yParity yul zfill -zkEVM - addoption addinivalue argname @@ -1060,7 +1058,6 @@ autoformat Typechecking groupstats SharedPreStateGroup -zkEVMs qube aspell codespell From 36b20fdcc6aa09e3327fa8063e175bb63580e4a4 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Thu, 26 Jun 2025 13:30:49 +0100 Subject: [PATCH 6/8] chore(docs): align changelog docs for future releases. --- docs/CHANGELOG.md | 12 +++++++----- docs/running_tests/releases.md | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bda0fd27054..6d7e374ef98 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,9 +17,11 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e - Before: `fill` wrote fixtures into the directory specified by the `--output` flag (default: `fixtures`). This could have many unintended consequences, including unexpected errors if old or invalid fixtures existed in the directory (for details see [#1030](https://github.com/ethereum/execution-spec-tests/issues/1030)). - Now: `fill` will exit without filling any tests if the specified directory exists and is not-empty. This may be overridden by adding the `--clean` flag, which will first remove the specified directory. -#### Feature `benchmark` expanded to `benchmark_36M`, `benchmark_60M`, `benchmark_90M`, `benchmark_120M` +#### Feature `zkevm` updated to `benchmark_36M`, `benchmark_60M`, `benchmark_90M`, `benchmark_120M` -Renames `benchmark` to `benchmark_36M` and further expands the benchmark features to run using 60M, 90M and 120M block gas limits in `fixtures_benchmark_36M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz` and `fixtures_benchmark_120M.tar.gz` respectively. +Due to the crossover between `zkevm` and `benchmark` tests, all instances of the former have been replaced with the latter nomenclature. Repository PR labels and titles are additionally updated to reflect this change. + +This update renames the `zkevm` feature release to `benchmark_36M` and further expands the latter for 60M, 90M, and 120M block gas limits in `fixtures_benchmark_36M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz`, and `fixtures_benchmark_120M.tar.gz` respectively. Users can select any of the artifacts depending on their testing needs for their provers. @@ -36,9 +38,9 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Add a new pytest plugin, `ported_tests`, that lists the static fillers and PRs from `ported_from` markers for use in the coverage Github Workflow ([#1634](https://github.com/ethereum/execution-spec-tests/pull/1634)). - โœจ Enable two-phase filling of fixtures with pre-allocation groups and add a `BlockchainEngineXFixture` format ([#1706](https://github.com/ethereum/execution-spec-tests/pull/1706), [#1760](https://github.com/ethereum/execution-spec-tests/pull/1760)). - ๐Ÿ”€ Refactor: Encapsulate `fill`'s fixture output options (`--output`, `--flat-output`, `--single-fixture-per-file`) into a `FixtureOutput` class ([#1471](https://github.com/ethereum/execution-spec-tests/pull/1471),[#1612](https://github.com/ethereum/execution-spec-tests/pull/1612)). -- โœจ Don't warn about a "high Transaction gas_limit" for `benchmark` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)). +- โœจ Don't warn about a "high Transaction gas_limit" for `zkevm` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)). - ๐Ÿž `fill` no longer writes generated fixtures into an existing, non-empty output directory; it must now be empty or `--clean` must be used to delete it first ([#1608](https://github.com/ethereum/execution-spec-tests/pull/1608)). -- ๐Ÿž benchmark marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-benchmark` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)). +- ๐Ÿž `zkevm` marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-zkevm` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)). - โœจ Field `postStateHash` is now added to all `blockchain_test` and `blockchain_test_engine` tests that use `exclude_full_post_state_in_output` in place of `postState`. Fixes `evmone-blockchaintest` test consumption and indirectly fixes coverage runs for these tests ([#1667](https://github.com/ethereum/execution-spec-tests/pull/1667)). - ๐Ÿ”€ Changed INVALID_DEPOSIT_EVENT_LAYOUT to a BlockException instead of a TransactionException ([#1773](https://github.com/ethereum/execution-spec-tests/pull/1773)). @@ -72,7 +74,7 @@ Users can select any of the artifacts depending on their testing needs for their - โœจ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679), [#1718](https://github.com/ethereum/execution-spec-tests/pull/1718)). - ๐Ÿ”€ Add macOS hive development mode workaround to the docs [#1786](https://github.com/ethereum/execution-spec-tests/pull/1786). - ๐Ÿ”€ Refactor and clean up of exceptions including EOF exceptions within client specific mappers [#1803](https://github.com/ethereum/execution-spec-tests/pull/1803). -- ๐Ÿ”€ Rename `zkevm` tests to `benchmark` while maintaining the `zkevm` marker [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804). +- ๐Ÿ”€ Rename `tests/zkevm/` to `tests/benchmark/` and replace the `zkevm` pytest mark with `benchmark` [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804). ### ๐Ÿงช Test Cases diff --git a/docs/running_tests/releases.md b/docs/running_tests/releases.md index c01d8df2e6a..e49c24beb6d 100644 --- a/docs/running_tests/releases.md +++ b/docs/running_tests/releases.md @@ -46,7 +46,7 @@ These releases are tagged using the format `@vX.Y.Z`. Examples: - [`fusaka-devnet-1@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/fusaka-devnet-1%40v1.0.0) - this fixture release contains tests adhering to the [Fusaka Devnet 1 spec](https://notes.ethereum.org/@ethpandaops/fusaka-devnet-1). -- [`benchmark@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/benchmark%40v0.1.0) - this fixture release contains tests specifically aimed at benchmarking EVMs. +- [`benchmark@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/benchmark%40v1.0.0) - this fixture release contains tests specifically aimed at benchmarking EVMs. Devnet releases should be treated as WIP and may not yet contain full test coverage (or even coverage for all EIPs). The coverage provided by these releases is detailed in the corresponding release notes. From 883bef9373443d6ce13378b716725b7907460465 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Tue, 1 Jul 2025 16:42:59 +0100 Subject: [PATCH 7/8] chore: final tweaks after rebase. --- docs/CHANGELOG.md | 2 +- tests/benchmark/test_worst_opcode.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6d7e374ef98..d46f4dac956 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,7 +17,7 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e - Before: `fill` wrote fixtures into the directory specified by the `--output` flag (default: `fixtures`). This could have many unintended consequences, including unexpected errors if old or invalid fixtures existed in the directory (for details see [#1030](https://github.com/ethereum/execution-spec-tests/issues/1030)). - Now: `fill` will exit without filling any tests if the specified directory exists and is not-empty. This may be overridden by adding the `--clean` flag, which will first remove the specified directory. -#### Feature `zkevm` updated to `benchmark_36M`, `benchmark_60M`, `benchmark_90M`, `benchmark_120M` +#### Feature `zkevm` updated to `benchmark` Due to the crossover between `zkevm` and `benchmark` tests, all instances of the former have been replaced with the latter nomenclature. Repository PR labels and titles are additionally updated to reflect this change. diff --git a/tests/benchmark/test_worst_opcode.py b/tests/benchmark/test_worst_opcode.py index 8192407f1b6..7e788c34f35 100644 --- a/tests/benchmark/test_worst_opcode.py +++ b/tests/benchmark/test_worst_opcode.py @@ -1,8 +1,8 @@ """ -abstract: Tests zkEVMs worst-case opcode scenarios. - Tests zkEVMs worst-case opcode scenarios. +abstract: Tests benchmark worst-case opcode scenarios. + Tests benchmark worst-case opcode scenarios. -Tests running worst-case opcodes scenarios for zkEVMs. +Tests running worst-case opcodes scenarios for benchmarking purposes. """ import pytest From ec03fc9d171d95085222d64d13a994614edf5d49 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Tue, 1 Jul 2025 18:41:49 +0200 Subject: [PATCH 8/8] Apply suggestions from code review --- .github/configs/feature.yaml | 4 ++-- docs/CHANGELOG.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index cc65993c47a..a2cb3181d7a 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -11,9 +11,9 @@ static: evm-type: static fill-params: --until=Prague --fill-static-tests ./tests/static solc: 0.8.21 -benchmark_36M: +benchmark_30M: evm-type: benchmark - fill-params: --from=Cancun --until=Prague --block-gas-limit 36000000 -m benchmark ./tests + fill-params: --from=Cancun --until=Prague --block-gas-limit 30000000 -m benchmark ./tests solc: 0.8.21 feature_only: true benchmark_60M: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d46f4dac956..6c7274f3e2f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -21,7 +21,7 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e Due to the crossover between `zkevm` and `benchmark` tests, all instances of the former have been replaced with the latter nomenclature. Repository PR labels and titles are additionally updated to reflect this change. -This update renames the `zkevm` feature release to `benchmark_36M` and further expands the latter for 60M, 90M, and 120M block gas limits in `fixtures_benchmark_36M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz`, and `fixtures_benchmark_120M.tar.gz` respectively. +This update renames the `zkevm` feature release to `benchmark_30M` and further expands the latter for 60M, 90M, and 120M block gas limits in `fixtures_benchmark_30M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz`, and `fixtures_benchmark_120M.tar.gz` respectively. Users can select any of the artifacts depending on their testing needs for their provers.