From 8120716480f2e40c8d39565a256d518d379e1952 Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Wed, 27 Nov 2019 17:47:40 +0200 Subject: [PATCH 01/10] setup: update to package versions that work with Python 3.8. Minimal set of dependencies that otherwise break the virtualenv, detailed here: https://github.com/ethereum/py-evm/issues/1872#issuecomment-559114127 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4dc049d403..7bc1e682a5 100644 --- a/setup.py +++ b/setup.py @@ -23,14 +23,14 @@ # Installing these libraries may make the evm perform better than # using the default fallbacks though. 'eth-extra': [ - "coincurve>=10.0.0,<11.0.0", + "coincurve>=13.0.0,<14.0.0", "eth-hash[pysha3];implementation_name=='cpython'", "eth-hash[pycryptodome];implementation_name=='pypy'", "plyvel>=1.0.5,<1.2.0", ], 'test': [ "factory-boy==2.11.1", - "hypothesis==3.69.5", + "hypothesis==4.50.6", "pexpect>=4.6, <5", "pytest>=5.1.3,<6", "pytest-asyncio>=0.10.0,<0.11", @@ -41,7 +41,7 @@ 'lint': [ "flake8==3.5.0", "flake8-bugbear==18.8.0", - "mypy==0.701", + "mypy==0.750", ], 'benchmark': [ "termcolor>=1.1.0,<2.0.0", From ead4e572de8e924ae51e5523723b5181948f5059 Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Fri, 6 Dec 2019 12:25:24 +0200 Subject: [PATCH 02/10] setup: bump eth-utils to >=1.8.0. That version added support for Python 3.8 internally: https://github.com/ethereum/eth-utils/blob/master/docs/releases.rst#eth-utils-180-2019-11-04 This is also what's being installed according to `pip freeze`. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7bc1e682a5..ce54950ac5 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ "eth-bloom>=1.0.3,<2.0.0", "eth-keys>=0.2.1,<0.3.0", "eth-typing>=2.2.0,<3.0.0", - "eth-utils>=1.7.0,<2.0.0", + "eth-utils>=1.8.0,<2.0.0", "lru-dict>=1.1.6", "mypy_extensions>=0.4.1,<1.0.0", "py-ecc>=1.4.7,<2.0.0", @@ -115,5 +115,6 @@ 'Natural Language :: English', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], ) From de0049ff08c5776fd154278e63bcf3ced5b5f66e Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Wed, 27 Nov 2019 17:50:02 +0200 Subject: [PATCH 03/10] circleci/tox: add Python 3.8 environments. --- .circleci/config.yml | 36 ++++++++++++++++++++++++++++++++++++ tox.ini | 5 +++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5df3f2bc3e..1cd4058c40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,6 +168,37 @@ jobs: environment: TOXENV: py37-lint + py38-core: + <<: *common + docker: + - image: circleci/python:3.8 + environment: + TOXENV: py38-core + py38-database: + <<: *common + docker: + - image: circleci/python:3.8 + environment: + TOXENV: py38-database + py38-transactions: + <<: *common + docker: + - image: circleci/python:3.8 + environment: + TOXENV: py38-transactions + py38-vm: + <<: *common + docker: + - image: circleci/python:3.8 + environment: + TOXENV: py38-vm + py38-lint: + <<: *common + docker: + - image: circleci/python:3.8 + environment: + TOXENV: py38-lint + workflows: version: 2 test: @@ -184,12 +215,17 @@ workflows: - py36-native-blockchain-transition - py36-vm - py37-vm + - py38-vm - py36-core - py37-core + - py38-core - py36-transactions - py37-transactions + - py38-transactions - py36-database - py37-database + - py38-database - py36-docs - py36-lint - py37-lint + - py38-lint diff --git a/tox.ini b/tox.ini index 457a62af46..8d577ef9c5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist= - py{36,37}-{core,database,transactions,vm} + py{36,37,38}-{core,database,transactions,vm} py36-benchmark py36-native-blockchain-{frontier,homestead,tangerine_whistle,spurious_dragon,byzantium,constantinople,petersburg,istanbul,metropolis,transition} - py{36,37}-lint + py{36,37,38}-lint py36-docs [flake8] @@ -41,6 +41,7 @@ deps = basepython = py36: python3.6 py37: python3.7 + py38: python3.8 [testenv:py36-docs] From 9bf0940aa378c26b7b015fc8d143d886f01c081d Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Thu, 28 Nov 2019 15:59:37 +0200 Subject: [PATCH 04/10] circleci: install `libleveldb-dev` _always_. Python package `plyvel` doesn't have pre-built wheels for Python 3.8 yet, so the CI machines need to build the wheel themselves. They need LevelDB header files for that. This change installs the headers system-wide. SQUASHED: CircleCI YAML can't handle complex map merging via anchors. So, do the blunt thing, and install the dependency on _all_ CI runs, even ones that don't need it. --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cd4058c40..0ab3fc05dd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,10 @@ version: 2.0 common: &common working_directory: ~/repo steps: + # TODO: remove this step when `plyvel` gets a Python 3.8 wheel + - run: + name: install LevelDB headers dependency system-wide + command: sudo apt-get install libleveldb-dev - checkout - run: name: checkout fixtures submodule From 3c654407ab81ad6bc75ef01cc5cba5e783d2d8d0 Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Mon, 2 Dec 2019 20:26:35 +0200 Subject: [PATCH 05/10] tests: limit read_len when reading from SlowCodeStream. ... and CodeStream, of course - it's just that SlowCodeStream comes first, so the test fails on it. A failure happens because the underlying io.BytesIO.read(size) call gets a size=9223372036854775808. That's (2^63), but it seems that the biggest size that BytesIO can read is (2^63)-1. The error surfaced after an update of package `hypothesis` from v3 to v4 (version that is latest ATM), and is possibly an issue from upstream. Or it could be ours. Anyway, limiting read_len fixes the issue. Could use (2^63)-1, too, but considering that `bytecode` max_size is limited, I don't see the point. --- tests/core/code-stream/test_code_stream.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/core/code-stream/test_code_stream.py b/tests/core/code-stream/test_code_stream.py index 6ddc79d879..ab49eac8a8 100644 --- a/tests/core/code-stream/test_code_stream.py +++ b/tests/core/code-stream/test_code_stream.py @@ -203,7 +203,10 @@ def test_new_vs_reference_code_stream_iter(bytecode): assert latest.program_counter == reference.program_counter -@given(read_len=st.integers(min_value=0), bytecode=st.binary(max_size=128)) +@given( + read_len=st.integers(min_value=0, max_value=2048), + bytecode=st.binary(max_size=128) +) def test_new_vs_reference_code_stream_read(read_len, bytecode): reference = SlowCodeStream(bytecode) latest = CodeStream(bytecode) @@ -217,7 +220,7 @@ def test_new_vs_reference_code_stream_read(read_len, bytecode): @given( read_idx=st.integers(min_value=0, max_value=10), - read_len=st.integers(min_value=0), + read_len=st.integers(min_value=0, max_value=2048), bytecode=st.binary(max_size=128), ) def test_new_vs_reference_code_stream_read_during_iter(read_idx, read_len, bytecode): From 7497337a5998e2eaac8136012fb4b641bd2d9814 Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Thu, 5 Dec 2019 16:27:33 +0200 Subject: [PATCH 06/10] lint: silence messages on "improper" f-string formatting. The messages silenced in this commit are all: On Python 3 '{}'.format(b'abc') produces "b'abc'"; use !r if this is a desired behavior Sadly, `mypy` doesn't say which particular fields are problematic; I've tried my best to single them out - hopefully, nothing extraneous got in. --- eth/chains/base.py | 8 ++++---- eth/consensus/clique/_utils.py | 6 +++--- eth/consensus/clique/datatypes.py | 2 +- eth/consensus/clique/snapshot_manager.py | 8 ++++---- eth/db/chain.py | 2 +- eth/tools/fixtures/helpers.py | 4 ++-- eth/validation.py | 2 +- eth/vm/base.py | 2 +- eth/vm/forks/frontier/validation.py | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eth/chains/base.py b/eth/chains/base.py index 228d105dd6..be557cc13d 100644 --- a/eth/chains/base.py +++ b/eth/chains/base.py @@ -225,8 +225,8 @@ def from_genesis(cls, # the computed state from the initialized state database. raise ValidationError( "The provided genesis state root does not match the computed " - f"genesis state root. Got {state.state_root}. " - f"Expected {genesis_params['state_root']}" + f"genesis state root. Got {state.state_root!r}. " + f"Expected {genesis_params['state_root']!r}" ) genesis_header = BlockHeader(**genesis_params) @@ -438,8 +438,8 @@ def import_block(self, except HeaderNotFound: raise ValidationError( f"Attempt to import block #{block.number}. " - f"Cannot import block {block.hash} before importing " - f"its parent block at {block.header.parent_hash}" + f"Cannot import block {block.hash!r} before importing " + f"its parent block at {block.header.parent_hash!r}" ) base_header_for_import = self.create_header_from_parent(parent_header) diff --git a/eth/consensus/clique/_utils.py b/eth/consensus/clique/_utils.py index be39abb9be..50f62c99a2 100644 --- a/eth/consensus/clique/_utils.py +++ b/eth/consensus/clique/_utils.py @@ -139,10 +139,10 @@ def validate_header_integrity(header: BlockHeaderAPI, epoch_length: int) -> None ) if header.nonce != NONCE_AUTH and header.nonce != NONCE_DROP: - raise ValidationError(f"Invalid nonce: {header.nonce}") + raise ValidationError(f"Invalid nonce: {header.nonce!r}") if at_checkpoint and header.nonce != NONCE_DROP: - raise ValidationError(f"Invalid checkpoint nonce: {header.nonce}") + raise ValidationError(f"Invalid checkpoint nonce: {header.nonce!r}") if len(header.extra_data) < VANITY_LENGTH: raise ValidationError("Missing vanity bytes in extra data") @@ -159,7 +159,7 @@ def validate_header_integrity(header: BlockHeaderAPI, epoch_length: int) -> None raise ValidationError("Checkpoint header must contain list of signers") if header.mix_hash != ZERO_HASH32: - raise ValidationError(f"Invalid mix hash: {header.mix_hash}") + raise ValidationError(f"Invalid mix hash: {header.mix_hash!r}") if header.uncles_hash != EMPTY_UNCLE_HASH: raise ValidationError(f"Invalid uncle hash: {header.uncle_hash}") diff --git a/eth/consensus/clique/datatypes.py b/eth/consensus/clique/datatypes.py index a0a64dc245..554d1e0c03 100644 --- a/eth/consensus/clique/datatypes.py +++ b/eth/consensus/clique/datatypes.py @@ -46,7 +46,7 @@ def validate_for(self, if not signer_is_kicked and not signer_is_nominated: raise ValidationError( "Must either kick an existing signer or nominate a new signer" - f"Subject: {subject} Current signers: {signers}" + f"Subject: {subject!r} Current signers: {signers}" ) diff --git a/eth/consensus/clique/snapshot_manager.py b/eth/consensus/clique/snapshot_manager.py index d8b46c166c..7917c3afb3 100644 --- a/eth/consensus/clique/snapshot_manager.py +++ b/eth/consensus/clique/snapshot_manager.py @@ -220,16 +220,16 @@ def get_snapshot(self, block_number: int, block_hash: Hash32) -> Snapshot: # Otherwise, we can retrieve it on the fly header = self._chain_db.get_block_header_by_hash(block_hash) except HeaderNotFound: - raise SnapshotNotFound(f"Can not get snapshot for {block_hash} at {block_number}") + raise SnapshotNotFound(f"Can not get snapshot for {block_hash!r} at {block_number}") else: if header.block_number != block_number: raise SnapshotNotFound( - f"Can not get snapshot for {block_hash} at {block_number}" + f"Can not get snapshot for {block_hash!r} at {block_number}" ) else: return self._create_snapshot_from_checkpoint_header(header) - raise SnapshotNotFound(f"Can not get snapshot for {block_hash} at {block_number}") + raise SnapshotNotFound(f"Can not get snapshot for {block_hash!r} at {block_number}") def add_snapshot(self, mutable_snapshot: MutableSnapshot) -> Snapshot: """ @@ -256,7 +256,7 @@ def get_snapshot_from_db(self, block_hash: Hash32) -> Snapshot: encoded_key = self._chain_db.db[key] except KeyError as e: raise SnapshotNotFound( - f"Can not get on-disk snapshot for {block_hash}" + f"Can not get on-disk snapshot for {block_hash!r}" ) else: return decode_snapshot(encoded_key) diff --git a/eth/db/chain.py b/eth/db/chain.py index 6e8d17d675..ef3b24f008 100644 --- a/eth/db/chain.py +++ b/eth/db/chain.py @@ -84,7 +84,7 @@ def get_block_uncles(self, uncles_hash: Hash32) -> Tuple[BlockHeaderAPI, ...]: encoded_uncles = self.db[uncles_hash] except KeyError: raise HeaderNotFound( - f"No uncles found for hash {uncles_hash}" + f"No uncles found for hash {uncles_hash!r}" ) else: return tuple(rlp.decode(encoded_uncles, sedes=rlp.sedes.CountableList(BlockHeader))) diff --git a/eth/tools/fixtures/helpers.py b/eth/tools/fixtures/helpers.py index af3433400f..351a98e7dc 100644 --- a/eth/tools/fixtures/helpers.py +++ b/eth/tools/fixtures/helpers.py @@ -76,13 +76,13 @@ def verify_state(expected_state: AccountState, state: StateAPI) -> None: if field == 'balance': error_messages.append( f"{to_normalized_address(account)}(balance) | " - f"Actual: {actual_value} | Expected: {expected_value} | " + f"Actual: {actual_value!r} | Expected: {expected_value!r} | " f"Delta: {cast(int, actual_value) - cast(int, expected_value)}" ) else: error_messages.append( f"{to_normalized_address(account)}({field}) | " - f"Actual: {actual_value} | Expected: {expected_value}" + f"Actual: {actual_value!r} | Expected: {expected_value!r}" ) raise AssertionError( f"State DB did not match expected state on {len(error_messages)} values:{new_line}" diff --git a/eth/validation.py b/eth/validation.py index 1c72600d3e..f1b7003b8b 100644 --- a/eth/validation.py +++ b/eth/validation.py @@ -111,7 +111,7 @@ def validate_lt(value: int, maximum: int, title: str="Value") -> None: def validate_canonical_address(value: Address, title: str="Value") -> None: if not isinstance(value, bytes) or not len(value) == 20: raise ValidationError( - f"{title} {value} is not a valid canonical address" + f"{title} {value!r} is not a valid canonical address" ) diff --git a/eth/vm/base.py b/eth/vm/base.py index 2e15b4bc1c..b2ce50a896 100644 --- a/eth/vm/base.py +++ b/eth/vm/base.py @@ -507,7 +507,7 @@ def validate_block(self, block: BlockAPI) -> None: if tx_root_hash != block.header.transaction_root: raise ValidationError( f"Block's transaction_root ({block.header.transaction_root}) " - f"does not match expected value: {tx_root_hash}" + f"does not match expected value: {tx_root_hash!r}" ) if len(block.uncles) > MAX_UNCLES: diff --git a/eth/vm/forks/frontier/validation.py b/eth/vm/forks/frontier/validation.py index ce2597054a..0acf6d1ec7 100644 --- a/eth/vm/forks/frontier/validation.py +++ b/eth/vm/forks/frontier/validation.py @@ -17,7 +17,7 @@ def validate_frontier_transaction(state: StateAPI, if sender_balance < gas_cost: raise ValidationError( - f"Sender {transaction.sender} cannot afford txn gas " + f"Sender {transaction.sender!r} cannot afford txn gas " f"{gas_cost} with account balance {sender_balance}" ) From 07b43985951f81a1126821cd01b827932d5e672f Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Fri, 6 Dec 2019 12:47:30 +0200 Subject: [PATCH 07/10] ABC: return same from OpcodeAPI's as_opcode() as from Opcode. 99.99% cargo-culting. I don't understand the purpose of this structuring, but the function signatures don't match, and this is the only permutation I could find between OpcodeAPI, Opcode and FRONTIER_OPCODES that stops producing an error when running `mypy`: eth/vm/opcode.py:40: error: Return type "Opcode" of "as_opcode" incompatible with return type "Type[Opcode]" in supertype "OpcodeAPI" --- eth/abc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/abc.py b/eth/abc.py index a753a26f95..ef3dd73742 100644 --- a/eth/abc.py +++ b/eth/abc.py @@ -775,9 +775,9 @@ def __call__(self, computation: 'ComputationAPI') -> None: def as_opcode(cls: Type[T], logic_fn: Callable[['ComputationAPI'], None], mnemonic: str, - gas_cost: int) -> Type[T]: + gas_cost: int) -> T: """ - Class factory method for turning vanilla functions into Opcode classes. + Class factory method for turning vanilla functions into Opcodes. """ ... From 851c141d18f9b47261ef45f47e27d5fb90d2e63c Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Fri, 6 Dec 2019 13:03:49 +0200 Subject: [PATCH 08/10] eth/vm/state: expect Hash32 (not bytes) in __init__ of BaseState. Again, 99.99% cargo-culting. This just silences a `mypy` error: eth/vm/state.py:57: error: Argument 2 to "AccountDatabaseAPI" has incompatible type "bytes"; expected "Hash32" A valid state root will definitely be a 32-byte-long hash. Question is if this is the right place to make sure of it; and whether that should be made more explicit in code than being picky about a type. --- eth/vm/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/vm/state.py b/eth/vm/state.py index 666b6f668b..ae1992125f 100644 --- a/eth/vm/state.py +++ b/eth/vm/state.py @@ -51,7 +51,7 @@ def __init__( self, db: AtomicDatabaseAPI, execution_context: ExecutionContextAPI, - state_root: bytes) -> None: + state_root: Hash32) -> None: self._db = db self.execution_context = execution_context self._account_db = self.get_account_db_class()(db, state_root) From 97cc8db2cb52d9221cb5d8ca2b5ff238391132ca Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Tue, 10 Dec 2019 20:45:07 +0200 Subject: [PATCH 09/10] ABC: add missing chain_context to VirtualMachineAPI's constructor. This silences `mypy` error: eth/chains/base.py:250: error: Unexpected keyword argument "chain_context" for "VirtualMachineAPI" eth/abc.py:2270: note: "VirtualMachineAPI" defined here --- eth/abc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth/abc.py b/eth/abc.py index ef3dd73742..ced835185d 100644 --- a/eth/abc.py +++ b/eth/abc.py @@ -2287,7 +2287,10 @@ class VirtualMachineAPI(ConfigurableAPI): extra_data_max_bytes: ClassVar[int] @abstractmethod - def __init__(self, header: BlockHeaderAPI, chaindb: ChainDatabaseAPI) -> None: + def __init__(self, + header: BlockHeaderAPI, + chaindb: ChainDatabaseAPI, + chain_context: ChainContextAPI) -> None: """ Initialize the virtual machine. """ From 7350ac395c27c448fe6136234d61d9a166bb26c1 Mon Sep 17 00:00:00 2001 From: Noel Maersk Date: Tue, 10 Dec 2019 21:15:19 +0200 Subject: [PATCH 10/10] ABC: AccountDatabaseAPI's state_root made writable. AccountDB (from eth/db/account.py) has a setter for the `state_root` property, and `BaseState` (from eth/vm/state.py) forcibly sets the state root in its revert() function. This helps silence `mypy` error: eth/vm/state.py:170: error: Property "state_root" defined in "AccountDatabaseAPI" is read-only --- eth/abc.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eth/abc.py b/eth/abc.py index ced835185d..c75ce30533 100644 --- a/eth/abc.py +++ b/eth/abc.py @@ -1633,6 +1633,16 @@ def state_root(self) -> Hash32: """ ... + @state_root.setter + def state_root(self, value: Hash32) -> None: + """ + Force-set the state root hash. + """ + # See: https://github.com/python/mypy/issues/4165 + # Since we can't also decorate this with abstract method we want to be + # sure that the setter doesn't actually get used as a noop. + raise NotImplementedError + @abstractmethod def has_root(self, state_root: bytes) -> bool: """