From f345025c67620a19019830b72b5860a689b8bc54 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 10 Sep 2021 00:09:48 -0300 Subject: [PATCH 1/9] feat(sync-v2): sync-v2 implemented, sync-v1 still default Co-authored-by: Marcelo Salhab Brogliato Co-authored-by: Pedro Ferreira --- hathor/graphviz.py | 4 + hathor/indexes/manager.py | 8 +- hathor/indexes/memory_deps_index.py | 24 +- hathor/indexes/rocksdb_deps_index.py | 18 + hathor/manager.py | 61 +- hathor/p2p/manager.py | 3 +- hathor/p2p/node_sync_v2.py | 1328 +++++++++++++++++ hathor/p2p/sync_checkpoints.py | 306 ++++ hathor/p2p/sync_mempool.py | 116 ++ hathor/p2p/sync_v2_factory.py | 34 + hathor/p2p/sync_version.py | 2 +- hathor/simulator/fake_connection.py | 11 +- hathor/simulator/simulator.py | 13 +- hathor/simulator/tx_generator.py | 15 +- hathor/transaction/base_transaction.py | 20 + hathor/transaction/block.py | 35 +- .../storage/transaction_storage.py | 11 + hathor/transaction/transaction_metadata.py | 20 +- tests/consensus/test_consensus.py | 4 +- tests/consensus/test_soft_voided.py | 4 +- tests/consensus/test_soft_voided3.py | 2 +- tests/consensus/test_soft_voided4.py | 2 +- tests/p2p/test_capabilities.py | 42 +- tests/p2p/test_protocol.py | 257 +++- tests/p2p/test_split_brain.py | 335 ++++- tests/p2p/test_split_brain2.py | 6 +- tests/p2p/test_sync.py | 273 +++- tests/p2p/test_sync_bridge.py | 82 + tests/resources/p2p/test_healthcheck.py | 2 +- tests/simulation/test_simulator.py | 4 +- tests/tx/test_tx.py | 35 + tests/unittest.py | 137 +- 32 files changed, 3046 insertions(+), 168 deletions(-) create mode 100644 hathor/p2p/node_sync_v2.py create mode 100644 hathor/p2p/sync_checkpoints.py create mode 100644 hathor/p2p/sync_mempool.py create mode 100644 hathor/p2p/sync_v2_factory.py create mode 100644 tests/p2p/test_sync_bridge.py diff --git a/hathor/graphviz.py b/hathor/graphviz.py index 0f7933c6b..a62fcf210 100644 --- a/hathor/graphviz.py +++ b/hathor/graphviz.py @@ -52,6 +52,7 @@ def __init__(self, storage: TransactionStorage, include_funds: bool = False, self.voided_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#BDC3C7') self.soft_voided_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#CCCCFF') self.conflict_attrs = dict(style='dashed,filled', penwidth='2.0', fillcolor='#BDC3C7') + self.not_fully_validated_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#F9FFAB') # Labels self.labels: Dict[bytes, str] = {} @@ -96,6 +97,9 @@ def get_node_attrs(self, tx: BaseTransaction) -> Dict[str, str]: else: node_attrs.update(self.voided_attrs) + if not meta.validation.is_fully_connected(): + node_attrs.update(self.not_fully_validated_attrs) + return node_attrs def get_edge_attrs(self, tx: BaseTransaction, neighbor_hash: bytes) -> Dict[str, str]: diff --git a/hathor/indexes/manager.py b/hathor/indexes/manager.py index bb4ccfe37..de0c2fd32 100644 --- a/hathor/indexes/manager.py +++ b/hathor/indexes/manager.py @@ -340,12 +340,12 @@ def enable_utxo_index(self) -> None: if self.utxo is None: self.utxo = MemoryUtxoIndex() - def enable_deps_index(self) -> None: + def enable_mempool_index(self) -> None: from hathor.indexes.memory_mempool_tips_index import MemoryMempoolTipsIndex if self.mempool_tips is None: self.mempool_tips = MemoryMempoolTipsIndex() - def enable_mempool_index(self) -> None: + def enable_deps_index(self) -> None: from hathor.indexes.memory_deps_index import MemoryDepsIndex if self.deps is None: self.deps = MemoryDepsIndex() @@ -394,13 +394,13 @@ def enable_utxo_index(self) -> None: if self.utxo is None: self.utxo = RocksDBUtxoIndex(self._db) - def enable_deps_index(self) -> None: + def enable_mempool_index(self) -> None: from hathor.indexes.memory_mempool_tips_index import MemoryMempoolTipsIndex if self.mempool_tips is None: # XXX: use of RocksDBMempoolTipsIndex is very slow and was suspended self.mempool_tips = MemoryMempoolTipsIndex() - def enable_mempool_index(self) -> None: + def enable_deps_index(self) -> None: from hathor.indexes.memory_deps_index import MemoryDepsIndex if self.deps is None: # XXX: use of RocksDBDepsIndex is currently suspended until it is fixed diff --git a/hathor/indexes/memory_deps_index.py b/hathor/indexes/memory_deps_index.py index 2c9d77eda..afb0d0aba 100644 --- a/hathor/indexes/memory_deps_index.py +++ b/hathor/indexes/memory_deps_index.py @@ -34,6 +34,9 @@ class MemoryDepsIndex(DepsIndex): _txs_with_deps_ready: Set[bytes] # Next to be downloaded + # - Key: hash of the tx to be downloaded + # - Value[0]: height + # - Value[1]: hash of the tx waiting for the download _needed_txs_index: Dict[bytes, Tuple[int, bytes]] def __init__(self): @@ -49,10 +52,11 @@ def force_clear(self) -> None: self._needed_txs_index = {} def add_tx(self, tx: BaseTransaction, partial: bool = True) -> None: - assert tx.hash is not None - assert tx.storage is not None validation = tx.get_metadata().validation if validation.is_fully_connected(): + # discover if new txs are ready because of this tx + self._update_new_deps_ready(tx) + # finally remove from rev deps self._del_from_deps_index(tx) elif not partial: raise ValueError('partial=False will only accept fully connected transactions') @@ -63,6 +67,18 @@ def add_tx(self, tx: BaseTransaction, partial: bool = True) -> None: def del_tx(self, tx: BaseTransaction) -> None: self._del_from_deps_index(tx) + def _update_new_deps_ready(self, tx: BaseTransaction) -> None: + """Go over the reverse dependencies of tx and check if any of them are now ready to be validated. + + This is also idempotent. + """ + assert tx.hash is not None + assert tx.storage is not None + for candidate_hash in self._rev_dep_index.get(tx.hash, []): + candidate_tx = tx.storage.get_transaction(candidate_hash) + if candidate_tx.is_ready_for_validation(): + self._txs_with_deps_ready.add(candidate_hash) + def _add_deps(self, tx: BaseTransaction) -> None: """This method is idempotent, because self.update needs it to be indempotent.""" assert tx.hash is not None @@ -139,6 +155,7 @@ def get_next_needed_tx(self) -> bytes: def _add_needed(self, tx: BaseTransaction) -> None: """This method is idempotent, because self.update needs it to be indempotent.""" + assert tx.hash is not None assert tx.storage is not None tx_storage = tx.storage @@ -153,3 +170,6 @@ def _add_needed(self, tx: BaseTransaction) -> None: if not tx_storage.transaction_exists(tx_hash): self.log.debug('tx parent is needed', tx=tx_hash.hex()) self._needed_txs_index[tx_hash] = (height, not_none(tx.hash)) + + # also, remove the given transaction from needed, because we already have it + self._needed_txs_index.pop(tx.hash, None) diff --git a/hathor/indexes/rocksdb_deps_index.py b/hathor/indexes/rocksdb_deps_index.py index 55169ba50..d5e40b788 100644 --- a/hathor/indexes/rocksdb_deps_index.py +++ b/hathor/indexes/rocksdb_deps_index.py @@ -194,6 +194,9 @@ def add_tx(self, tx: BaseTransaction, partial: bool = True) -> None: batch = rocksdb.WriteBatch() validation = tx.get_metadata().validation if validation.is_fully_connected(): + # discover if new txs are ready because of this tx + self._update_new_deps_ready(tx, batch) + # finally remove from rev deps self._del_from_deps(tx, batch) elif not partial: raise ValueError('partial=False will only accept fully connected transactions') @@ -208,6 +211,18 @@ def del_tx(self, tx: BaseTransaction) -> None: self._del_from_deps(tx, batch) self._db.write(batch) + def _update_new_deps_ready(self, tx: BaseTransaction, batch: 'rocksdb.WriteBatch') -> None: + """Go over the reverse dependencies of tx and check if any of them are now ready to be validated. + + This is also idempotent. + """ + assert tx.hash is not None + assert tx.storage is not None + for candidate_hash in self._iter_rev_deps_of(tx.hash): + candidate_tx = tx.storage.get_transaction(candidate_hash) + if candidate_tx.is_ready_for_validation(): + self._add_ready(candidate_hash, batch) + def _add_deps(self, tx: BaseTransaction, batch: 'rocksdb.WriteBatch') -> None: assert tx.hash is not None for dep in tx.get_all_dependencies(): @@ -230,6 +245,9 @@ def _add_needed(self, tx: BaseTransaction, batch: 'rocksdb.WriteBatch') -> None: self.log.debug('tx parent is needed', tx=tx.hash.hex(), tx_dep=tx_dep_hash.hex()) batch.put((self._cf, self._to_key_needed(tx_dep_hash)), self._to_value_needed(height, tx.hash)) + # also, remove the given transaction from needed, because we already have it + batch.delete((self._cf, self._to_key_needed(tx.hash))) + def remove_ready_for_validation(self, tx: bytes) -> None: self._db.delete((self._cf, self._to_key_ready(tx))) diff --git a/hathor/manager.py b/hathor/manager.py index 9406db51f..05c426009 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -473,7 +473,7 @@ def _initialize_components(self) -> None: if self.tx_storage.indexes.mempool_tips is not None: self.tx_storage.indexes.mempool_tips.update(tx) # XXX: move to indexes.update if self.tx_storage.indexes.deps is not None: - self.sync_v2_step_validations([tx]) + self.sync_v2_step_validations([tx], quiet=True) else: assert tx.validate_basic(skip_block_weight_verification=skip_block_weight_verification) self.tx_storage.save_transaction(tx, only_metadata=True) @@ -486,7 +486,8 @@ def _initialize_components(self) -> None: assert self.tx_storage.indexes is not None if self.tx_storage.indexes.mempool_tips: self.tx_storage.indexes.mempool_tips.update(tx) - self.tx_storage.add_to_indexes(tx) + if tx_meta.validation.is_fully_connected(): + self.tx_storage.add_to_indexes(tx) if tx.is_transaction and tx_meta.voided_by: self.tx_storage.del_from_indexes(tx) except (InvalidNewTransaction, TxValidationError): @@ -545,7 +546,7 @@ def _initialize_components(self) -> None: if tx.get_metadata().validation.is_final(): depended_final_txs.append(tx) if self.tx_storage.indexes.deps is not None: - self.sync_v2_step_validations(depended_final_txs) + self.sync_v2_step_validations(depended_final_txs, quiet=True) self.log.debug('pending validations finished') best_height = self.tx_storage.get_height_best_block() @@ -697,7 +698,7 @@ def _sync_v2_resume_validations(self) -> None: tx = self.tx_storage.get_transaction(tx_hash) if tx.get_metadata().validation.is_final(): depended_final_txs.append(tx) - self.sync_v2_step_validations(depended_final_txs) + self.sync_v2_step_validations(depended_final_txs, quiet=True) self.log.debug('pending validations finished') def add_listen_address(self, addr: str) -> None: @@ -1023,7 +1024,7 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non self.tx_storage.indexes.update(tx) if self.tx_storage.indexes.mempool_tips: self.tx_storage.indexes.mempool_tips.update(tx) # XXX: move to indexes.update - self.tx_fully_validated(tx) + self.tx_fully_validated(tx, quiet=quiet) elif sync_checkpoints: assert self.tx_storage.indexes.deps is not None metadata.children = self.tx_storage.indexes.deps.known_children(tx) @@ -1035,7 +1036,10 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non self.log.warn('on_new_tx(): checkpoint validation failed', tx=tx.hash_hex, exc_info=True) return False self.tx_storage.save_transaction(tx) + self.tx_storage.indexes.deps.add_tx(tx) + self.log_new_object(tx, 'new {} partially accepted while syncing checkpoints', quiet=quiet) else: + assert self.tx_storage.indexes.deps is not None if isinstance(tx, Block) and not tx.has_basic_block_parent(): if not fails_silently: raise InvalidNewTransaction('block parent needs to be at least basic-valid') @@ -1051,36 +1055,29 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non # This needs to be called right before the save because we were adding the children # in the tx parents even if the tx was invalid (failing the verifications above) # then I would have a children that was not in the storage - tx.update_initial_metadata(save=False) self.tx_storage.save_transaction(tx) + self.tx_storage.indexes.deps.add_tx(tx) + self.log_new_object(tx, 'new {} partially accepted', quiet=quiet) - if tx.is_transaction and self.tx_storage.indexes.deps is not None: + if self.tx_storage.indexes.deps is not None: self.tx_storage.indexes.deps.remove_from_needed_index(tx.hash) if self.tx_storage.indexes.deps is not None: try: - self.sync_v2_step_validations([tx]) + self.sync_v2_step_validations([tx], quiet=quiet) except (AssertionError, HathorError) as e: if not fails_silently: raise InvalidNewTransaction('step validations failed') from e self.log.warn('on_new_tx(): step validations failed', tx=tx.hash_hex, exc_info=True) return False - if not quiet: - ts_date = datetime.datetime.fromtimestamp(tx.timestamp) - now = datetime.datetime.fromtimestamp(self.reactor.seconds()) - if tx.is_block: - self.log.info('new block', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) - else: - self.log.info('new tx', tx=tx, ts_date=ts_date, time_from_now=tx.get_time_from_now(now)) - if propagate_to_peers: # Propagate to our peers. self.connections.send_tx_to_peers(tx) return True - def sync_v2_step_validations(self, txs: Iterable[BaseTransaction]) -> None: + def sync_v2_step_validations(self, txs: Iterable[BaseTransaction], *, quiet: bool) -> None: """ Step all validations until none can be stepped anymore. """ assert self.tx_storage.indexes is not None @@ -1097,7 +1094,7 @@ def sync_v2_step_validations(self, txs: Iterable[BaseTransaction]) -> None: try: # XXX: `reject_locked_reward` might not apply, partial validation is only used on sync-v2 # TODO: deal with `reject_locked_reward` on sync-v2 - assert tx.validate_full(reject_locked_reward=True) + assert tx.validate_full(reject_locked_reward=False) except (AssertionError, HathorError): # TODO raise @@ -1107,9 +1104,9 @@ def sync_v2_step_validations(self, txs: Iterable[BaseTransaction]) -> None: self.tx_storage.indexes.update(tx) if self.tx_storage.indexes.mempool_tips: self.tx_storage.indexes.mempool_tips.update(tx) # XXX: move to indexes.update - self.tx_fully_validated(tx) + self.tx_fully_validated(tx, quiet=quiet) - def tx_fully_validated(self, tx: BaseTransaction) -> None: + def tx_fully_validated(self, tx: BaseTransaction, *, quiet: bool) -> None: """ Handle operations that need to happen once the tx becomes fully validated. This might happen immediately after we receive the tx, if we have all dependencies @@ -1128,6 +1125,30 @@ def tx_fully_validated(self, tx: BaseTransaction) -> None: # TODO Remove it and use pubsub instead. self.wallet.on_new_tx(tx) + self.log_new_object(tx, 'new {}', quiet=quiet) + + def log_new_object(self, tx: BaseTransaction, message_fmt: str, *, quiet: bool) -> None: + """ A shortcut for logging additional information for block/txs. + """ + metadata = tx.get_metadata() + now = datetime.datetime.fromtimestamp(self.reactor.seconds()) + kwargs = { + 'tx': tx, + 'ts_date': datetime.datetime.fromtimestamp(tx.timestamp), + 'time_from_now': tx.get_time_from_now(now), + 'validation': metadata.validation.name, + } + if tx.is_block: + message = message_fmt.format('block') + kwargs['height'] = metadata.get_soft_height() + else: + message = message_fmt.format('tx') + if not quiet: + log_func = self.log.info + else: + log_func = self.log.debug + log_func(message, **kwargs) + def listen(self, description: str, use_ssl: Optional[bool] = None) -> None: endpoint = self.connections.listen(description, use_ssl) # XXX: endpoint: IStreamServerEndpoint does not intrinsically have a port, but in practice all concrete cases diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index 6d2061532..2c8f1a201 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -75,6 +75,7 @@ def __init__(self, reactor: Reactor, my_peer: PeerId, server_factory: 'HathorSer client_factory: 'HathorClientFactory', pubsub: PubSubManager, manager: 'HathorManager', ssl: bool, rng: Random, whitelist_only: bool, enable_sync_v1: bool, enable_sync_v2: bool) -> None: from hathor.p2p.sync_v1_factory import SyncV1Factory + from hathor.p2p.sync_v2_factory import SyncV2Factory if not (enable_sync_v1 or enable_sync_v2): raise TypeError(f'{type(self).__name__}() at least one sync version is required') @@ -139,7 +140,7 @@ def __init__(self, reactor: Reactor, my_peer: PeerId, server_factory: 'HathorSer if enable_sync_v1: self._sync_factories[SyncVersion.V1] = SyncV1Factory(self) if enable_sync_v2: - self._sync_factories[SyncVersion.V2] = SyncV1Factory(self) + self._sync_factories[SyncVersion.V2] = SyncV2Factory(self) def start(self) -> None: self.lc_reconnect.start(5, now=False) diff --git a/hathor/p2p/node_sync_v2.py b/hathor/p2p/node_sync_v2.py new file mode 100644 index 000000000..d0f97f7ef --- /dev/null +++ b/hathor/p2p/node_sync_v2.py @@ -0,0 +1,1328 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import math +import struct +from collections import OrderedDict +from enum import Enum, IntFlag +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Optional, Tuple + +from structlog import get_logger +from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.interfaces import IConsumer, IDelayedCall, IPushProducer +from twisted.internet.task import LoopingCall +from zope.interface import implementer + +from hathor.conf import HathorSettings +from hathor.p2p.messages import ProtocolMessages +from hathor.p2p.sync_checkpoints import SyncCheckpoint +from hathor.p2p.sync_manager import SyncManager +from hathor.p2p.sync_mempool import SyncMempoolManager +from hathor.transaction import BaseTransaction, Block, Transaction +from hathor.transaction.base_transaction import tx_or_block_from_bytes +from hathor.transaction.exceptions import HathorError +from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.storage.traversal import BFSWalk +from hathor.util import Reactor, verified_cast + +if TYPE_CHECKING: + from hathor.p2p.protocol import HathorProtocol + +settings = HathorSettings() +logger = get_logger() + + +class PeerState(Enum): + ERROR = 'error' + UNKNOWN = 'unknown' + SYNCING_CHECKPOINTS = 'syncing-checkpoints' + SYNCING_BLOCKS = 'syncing-blocks' + + +class StreamEnd(IntFlag): + END_HASH_REACHED = 0 + NO_MORE_BLOCKS = 1 + LIMIT_EXCEEDED = 2 + STREAM_BECAME_VOIDED = 3 # this will happen when the current chain becomes voided while it is being sent + + def __str__(self): + if self is StreamEnd.END_HASH_REACHED: + return 'end hash reached' + elif self is StreamEnd.NO_MORE_BLOCKS: + return 'end of blocks, no more blocks to download from this peer' + elif self is StreamEnd.LIMIT_EXCEEDED: + return 'streaming limit exceeded' + elif self is StreamEnd.STREAM_BECAME_VOIDED: + return 'streamed block chain became voided' + else: + raise ValueError(f'invalid StreamEnd value: {self.value}') + + +class NodeBlockSync(SyncManager): + """ An algorithm to sync the Blockchain between two peers. + """ + name: str = 'node-block-sync' + + def __init__(self, protocol: 'HathorProtocol', sync_checkpoints: SyncCheckpoint, + reactor: Optional[Reactor] = None) -> None: + """ + :param protocol: Protocol of the connection. + :type protocol: HathorProtocol + + :param reactor: Reactor to schedule later calls. (default=twisted.internet.reactor) + :type reactor: Reactor + """ + self.protocol = protocol + self.manager = protocol.node + self.tx_storage = protocol.node.tx_storage + self.sync_checkpoints = sync_checkpoints + self.state = PeerState.UNKNOWN + + if reactor is None: + from hathor.util import reactor as twisted_reactor + reactor = twisted_reactor + assert reactor is not None + self.reactor: Reactor = reactor + self._is_streaming = False + + # Create logger with context + self.log = logger.new(peer=self.protocol.get_short_peer_id()) + + # Extra + self._blk_size = 0 + self._blk_end_hash = settings.GENESIS_BLOCK_HASH + self._blk_max_quantity = 0 + + # indicates whether we're receiving a stream from the peer + self.receiving_stream = False + + # highest block where we are synced + self.synced_height = 0 + + # highest block peer has + self.peer_height = 0 + + # Latest deferred waiting for a reply. + self.deferred_by_key: Dict[str, Deferred] = {} + + # When syncing blocks we start streaming with all peers + # so the moment I get some repeated blocks, I stop the download + # because it's probably a streaming that I've just received + self.max_repeated_blocks = 10 + + # Blockchain streaming object, so I can stop anytime + self.blockchain_streaming: Optional[BlockchainStreaming] = None + + # Whether the peers are synced, i.e. our best height and best block are the same + self._synced = False + + # Indicate whether the sync manager has been started. + self._started: bool = False + + # Saves the last received block from the block streaming # this is useful to be used when running the sync of + # transactions in the case when I am downloading a side chain. Starts at the genesis, which is common to all + # peers on the network + self._last_received_block: Optional[Block] = None + + # Saves if I am in the middle of a mempool sync + # we don't execute any sync while in the middle of it + self.mempool_manager = SyncMempoolManager(self) + self._receiving_tips: Optional[List[bytes]] = None + + # Cache for get_tx calls + self._get_tx_cache: OrderedDict[bytes, BaseTransaction] = OrderedDict() + self._get_tx_cache_maxsize = 1000 + + # This exists to avoid sync-txs loop on sync-checkpoints + self._last_sync_transactions_start_hash: Optional[bytes] = None + + # Looping call of the main method + self._lc_run = LoopingCall(self.run_sync) + self._lc_run.clock = self.reactor + self._is_running = False + + # Whether we propagate transactions or not + self._is_relaying = False + + # Initial value + self._blk_height: Optional[int] = None + self._blk_end_height: Optional[int] = None + + def get_status(self) -> Dict[str, Any]: + """ Return the status of the sync. + """ + res = { + 'peer_height': self.peer_height, + 'synced_height': self.synced_height, + 'synced': self._synced, + 'state': self.state.value, + } + return res + + def is_synced(self) -> bool: + return self._synced + + def is_errored(self) -> bool: + return self.state is PeerState.ERROR + + def is_syncing_checkpoints(self) -> bool: + """True if state is SYNCING_CHECKPOINTS.""" + return self.state is PeerState.SYNCING_CHECKPOINTS + + def send_tx_to_peer_if_possible(self, tx: BaseTransaction) -> None: + if not self.is_synced(): + # XXX Should we accept any tx while I am not synced? + return + + # XXX When we start having many txs/s this become a performance issue + # Then we could change this to be a streaming of real time data with + # blocks as priorities to help miners get the blocks as fast as we can + # We decided not to implement this right now because we already have some producers + # being used in the sync algorithm and the code was becoming a bit too complex + if self._is_relaying: + self.send_data(tx) + + def is_started(self) -> bool: + return self._started + + def start(self) -> None: + """ Start sync. + """ + if self._started: + raise Exception('NodeSyncBlock is already running') + self._started = True + self._lc_run.start(5) + + def stop(self) -> None: + """ Stop sync. + """ + if not self._started: + raise Exception('NodeSyncBlock is already stopped') + self._started = False + self._lc_run.stop() + + def get_cmd_dict(self) -> Dict[ProtocolMessages, Callable[[str], None]]: + """ Return a dict of messages of the plugin. + + For further information about each message, see the RFC. + Link: https://github.com/HathorNetwork/rfcs/blob/master/text/0025-p2p-sync-v2.md#p2p-sync-protocol-messages + """ + return { + ProtocolMessages.GET_NEXT_BLOCKS: self.handle_get_next_blocks, + ProtocolMessages.GET_PREV_BLOCKS: self.handle_get_prev_blocks, + ProtocolMessages.BLOCKS: self.handle_blocks, + ProtocolMessages.BLOCKS_END: self.handle_blocks_end, + ProtocolMessages.GET_BEST_BLOCK: self.handle_get_best_block, + ProtocolMessages.BEST_BLOCK: self.handle_best_block, + ProtocolMessages.GET_BLOCK_TXS: self.handle_get_block_txs, + ProtocolMessages.TRANSACTION: self.handle_transaction, + ProtocolMessages.GET_PEER_BLOCK_HASHES: self.handle_get_peer_block_hashes, + ProtocolMessages.PEER_BLOCK_HASHES: self.handle_peer_block_hashes, + ProtocolMessages.STOP_BLOCK_STREAMING: self.handle_stop_block_streaming, + ProtocolMessages.GET_TIPS: self.handle_get_tips, + ProtocolMessages.TIPS: self.handle_tips, + ProtocolMessages.TIPS_END: self.handle_tips_end, + # XXX: overriding ReadyState.handle_error + ProtocolMessages.ERROR: self.handle_error, + ProtocolMessages.GET_DATA: self.handle_get_data, + ProtocolMessages.DATA: self.handle_data, + ProtocolMessages.RELAY: self.handle_relay, + } + + def handle_error(self, payload: str) -> None: + """ Override protocols original handle_error so we can recover a sync in progress. + """ + assert self.protocol.connections is not None + if self.sync_checkpoints.is_started() and self.sync_checkpoints.peer_syncing == self: + # Oops, we're syncing and we received an error, remove ourselves and let it recover + self.sync_checkpoints.peer_syncing = None + self.sync_checkpoints.peers_to_request.remove(self) + # forward message to overloaded handle_error: + self.protocol.handle_error(payload) + + def update_synced(self, synced: bool) -> None: + self._synced = synced + + def sync_checkpoints_finished(self) -> None: + self.log.info('finished syncing checkpoints') + self.state = PeerState.SYNCING_BLOCKS + + @inlineCallbacks + def run_sync(self) -> Generator[Any, Any, None]: + if self._is_running: + # Already running... + self.log.debug('already running') + return + self._is_running = True + try: + yield self._run_sync() + finally: + self._is_running = False + + @inlineCallbacks + def _run_sync(self) -> Generator[Any, Any, None]: + """Run sync. This is the entrypoint for the sync. + It is always safe to call this method. + """ + + if self.receiving_stream: + # If we're receiving a stream, wait for it to finish before running sync. + # If we're sending a stream, do the sync to update the peer's synced block + self.log.debug('receiving stream, try again later') + return + + if self.mempool_manager.is_running(): + # It's running a mempool sync, so we wait until it finishes + self.log.debug('running mempool sync, try again later') + return + + checkpoints = self.manager.checkpoints + bestblock = self.tx_storage.get_best_block() + meta = bestblock.get_metadata() + + self.log.debug('run sync', height=meta.height) + + assert self.protocol.connections is not None + assert self.tx_storage.indexes is not None + assert self.tx_storage.indexes.deps is not None + + if self.is_syncing_checkpoints(): + # already syncing checkpoints, nothing to do + self.log.debug('already syncing checkpoints', height=meta.height) + elif meta.height < checkpoints[-1].height: + yield self.start_sync_checkpoints() + elif self.tx_storage.indexes.deps.has_needed_tx(): + self.log.debug('needed tx exist, sync transactions') + self.update_synced(False) + # TODO: find out whether we can sync transactions from this peer to speed things up + self.run_sync_transactions() + else: + # I am already in sync with all checkpoints, sync next blocks + yield self.run_sync_blocks() + + @inlineCallbacks + def start_sync_checkpoints(self) -> Generator[Any, Any, None]: + assert self.protocol.connections is not None + # Start object to sync until last checkpoint + # and request the best block height of the peer + self.state = PeerState.SYNCING_CHECKPOINTS + self.log.debug('run sync checkpoints') + data = yield self.get_peer_best_block() + peer_best_height = data['height'] + self.peer_height = peer_best_height + self.sync_checkpoints.update_peer_height(self, peer_best_height) + self.sync_checkpoints.start() + + def run_sync_transactions(self) -> None: + from hathor.transaction.genesis import BLOCK_GENESIS + + assert self.protocol.connections is not None + assert self.tx_storage.indexes is not None + assert self.tx_storage.indexes.deps is not None + + start_hash = self.tx_storage.indexes.deps.get_next_needed_tx() + + if self.is_syncing_checkpoints(): + if start_hash == self._last_sync_transactions_start_hash: + self.log.info('sync transactions looped, skipping', start=start_hash.hex()) + self.sync_checkpoints.should_skip_sync_tx = True + self.sync_checkpoints.continue_sync() + return + self._last_sync_transactions_start_hash = start_hash + + # Start with the last received block and find the best block full validated in its chain + if self.is_syncing_checkpoints(): + block_hash = self._blk_end_hash + block_height = self._blk_end_height + else: + block = self._last_received_block + if block is None: + block = BLOCK_GENESIS + else: + while not block.get_metadata().validation.is_valid(): + block = block.get_block_parent() + assert block.hash is not None + block_hash = block.hash + block_height = block.get_metadata().get_soft_height() + + self.log.info('run sync transactions', start=start_hash.hex(), end_block_hash=block_hash.hex(), + end_block_height=block_height) + self.send_get_block_txs(start_hash, block_hash) + + @inlineCallbacks + def run_sync_blocks(self) -> Generator[Any, Any, None]: + self.state = PeerState.SYNCING_BLOCKS + + # Find my height + bestblock = self.tx_storage.get_best_block() + meta = bestblock.get_metadata() + my_height = meta.height + + self.log.debug('run sync blocks', my_height=my_height) + + # Find best block + data = yield self.get_peer_best_block() + peer_best_block = data['block'] + peer_best_height = data['height'] + self.peer_height = peer_best_height + + # find best common block + yield self.find_best_common_block(peer_best_height, peer_best_block) + self.log.debug('run_sync_blocks', peer_height=self.peer_height, synced_height=self.synced_height) + + if self.synced_height < self.peer_height: + # sync from common block + peer_block_at_height = yield self.get_peer_block_hashes([self.synced_height]) + self.run_block_sync(peer_block_at_height[0][1], self.synced_height, peer_best_block, peer_best_height) + elif my_height == self.synced_height == self.peer_height: + # we're synced and on the same height, get their mempool + self.mempool_manager.run() + else: + # we got all the peer's blocks but aren't on the same height, nothing to do + pass + + # -------------------------------------------- + # BEGIN: GET_TIPS/TIPS/TIPS_END implementation + # -------------------------------------------- + + def get_tips(self) -> Deferred[List[bytes]]: + """Async method to request the tips, returned hashes guaranteed to be new""" + key = 'tips' + deferred = self.deferred_by_key.get(key, None) + if deferred is None: + deferred = self.deferred_by_key[key] = Deferred() + self.send_get_tips() + else: + assert self._receiving_tips is not None + return deferred + + def send_get_tips(self) -> None: + self.log.debug('get tips') + self.send_message(ProtocolMessages.GET_TIPS) + self._receiving_tips = [] + + def handle_get_tips(self, payload: str) -> None: + """Handle a received GET_TIPS message.""" + assert self.tx_storage.indexes is not None + assert self.tx_storage.indexes.mempool_tips is not None + if self._is_streaming: + self.log.warn('can\'t send while streaming') # XXX: or can we? + self.send_message(ProtocolMessages.MEMPOOL_END) + return + self.log.debug('handle_get_tips') + # TODO Use a streaming of tips + for txid in self.tx_storage.indexes.mempool_tips.get(): + self.send_tips(txid) + self.send_message(ProtocolMessages.TIPS_END) + + def send_tips(self, tx_id: bytes) -> None: + """Send a TIPS message.""" + self.send_message(ProtocolMessages.TIPS, json.dumps([tx_id.hex()])) + + def handle_tips(self, payload: str) -> None: + """Handle a received TIPS message.""" + self.log.debug('tips', receiving_tips=self._receiving_tips) + if self._receiving_tips is None: + self.protocol.send_error_and_close_connection('TIPS not expected') + return + data = json.loads(payload) + data = [bytes.fromhex(x) for x in data] + # filter-out txs we already have + self._receiving_tips.extend(tx_id for tx_id in data if not self.tx_storage.transaction_exists(tx_id)) + + def handle_tips_end(self, payload: str) -> None: + """Handle a received TIPS-END message.""" + assert self._receiving_tips is not None + key = 'tips' + deferred = self.deferred_by_key.pop(key, None) + if deferred is None: + self.protocol.send_error_and_close_connection('TIPS-END not expected') + return + deferred.callback(self._receiving_tips) + self._receiving_tips = None + + # ------------------------------------------ + # END: GET_TIPS/TIPS/TIPS_END implementation + # ------------------------------------------ + + def send_relay(self) -> None: + self.log.debug('ask for relay') + self.send_message(ProtocolMessages.RELAY) + + def handle_relay(self, payload: str) -> None: + """Handle a received RELAY message.""" + # XXX: we need a way to turn this off, should we have arguments like: OFF, ALWAYS, SYNCED, ...? there is no + # specific design for this + self._is_relaying = True + + def _setup_block_streaming(self, start_hash: bytes, start_height: int, end_hash: bytes, end_height: int, + reverse: bool) -> None: + self._blk_start_hash = start_hash + self._blk_start_height = start_height + self._blk_end_hash = end_hash + self._blk_end_height = end_height + self._blk_received = 0 + self._blk_repeated = 0 + self._blk_height = start_height + raw_quantity = end_height - start_height + 1 + self._blk_max_quantity = -raw_quantity if reverse else raw_quantity + self._blk_prev_hash: Optional[bytes] = None + self._blk_stream_reverse = reverse + self._last_received_block = None + + def run_sync_between_heights(self, start_hash: bytes, start_height: int, end_hash: bytes, end_height: int) -> None: + """Called when the bestblock is between two checkpoints. + It must syncs to the left until it reaches a known block. + + We assume that we can trust in `start_hash`. + + Possible cases: + o---------------------o + o####-----------------o + + Impossible cases: + o####-----##----------o + o####---------------##o + + TODO Check len(downloads) == h(start) - h(end) + """ + self._setup_block_streaming(start_hash, start_height, end_hash, end_height, True) + quantity = start_height - end_height + self.log.info('get prev blocks', start_height=start_height, end_height=end_height, quantity=quantity, + start_hash=start_hash.hex(), end_hash=end_hash.hex()) + self.send_get_prev_blocks(start_hash, end_hash) + + def run_block_sync(self, start_hash: bytes, start_height: int, end_hash: bytes, end_height: int) -> None: + """Called when the bestblock is after all checkpoints. + It must syncs to the left until it reaches the remote's best block or the max stream limit. + """ + self._setup_block_streaming(start_hash, start_height, end_hash, end_height, False) + quantity = end_height - start_height + self.log.info('get next blocks', start_height=start_height, end_height=end_height, quantity=quantity, + start_hash=start_hash.hex(), end_hash=end_hash.hex()) + self.send_get_next_blocks(start_hash, end_hash) + + def send_message(self, cmd: ProtocolMessages, payload: Optional[str] = None) -> None: + """ Helper to send a message. + """ + assert self.protocol.state is not None + self.protocol.state.send_message(cmd, payload) + + @inlineCallbacks + def find_best_common_block(self, peer_best_height: int, peer_best_block: bytes) -> Generator[Any, Any, None]: + """ Search for the highest block/height where we're synced. + """ + assert self.tx_storage.indexes is not None + my_best_height = self.tx_storage.get_height_best_block() + + self.log.debug('find common chain', peer_height=peer_best_height, my_height=my_best_height) + + if peer_best_height <= my_best_height: + my_block = self.tx_storage.indexes.height.get(peer_best_height) + if my_block == peer_best_block: + # we have all the peer's blocks + if peer_best_height == my_best_height: + # We are in sync, ask for relay so the remote sends transactions in real time + self.update_synced(True) + self.send_relay() + else: + self.update_synced(False) + + self.log.debug('synced to the latest peer block', height=peer_best_height) + self.synced_height = peer_best_height + return + else: + # TODO peer is on a different best chain + self.log.warn('peer on different chain', peer_height=peer_best_height, + peer_block=peer_best_block.hex(), my_block=(my_block.hex() if my_block is not None else + None)) + + self.update_synced(False) + not_synced = min(peer_best_height, my_best_height) + synced = self.synced_height + + if not_synced < synced: + self.log.warn('find_best_common_block not_synced < synced', synced=synced, not_synced=not_synced) + # not_synced at this moment has the minimum of this node's or the peer's best height. If this is + # smaller than the previous synced_height, it means either this node or the peer has switched best + # chains. In this case, find the common block from checkpoint. + synced = self.manager.checkpoints[-1].height + + while not_synced - synced > 1: + self.log.debug('find_best_common_block synced not_synced', synced=synced, not_synced=not_synced) + step = math.ceil((not_synced - synced)/10) + heights = [] + height = synced + while height < not_synced: + heights.append(height) + height += step + heights.append(not_synced) + block_height_list = yield self.get_peer_block_hashes(heights) + block_height_list.reverse() + for height, block_hash in block_height_list: + # TODO initially I was checking the block_hash by height on the best chain. However, if the + # peers are not on the same chain, the sync would never move forward. I think we need to debate it + # better. Currently, the peer would show up as being in sync until the latest block I have in common + # with him, even though we're not on the same best chain + # if block_hash == self.tx_storage.get_from_block_height_index(height): + if self.tx_storage.transaction_exists(block_hash): + synced = height + break + else: + not_synced = height + + if not_synced == self.synced_height: + self.log.warn('find_best_common_block not synced to previous synced height', synced=synced, + not_synced=not_synced) + # We're not synced in our previous synced height anymore, so someone changed best chains + not_synced = min(peer_best_height, my_best_height) + synced = self.manager.checkpoints[-1].height + + self.log.debug('find_best_common_block finished synced not_synced', synced=synced, not_synced=not_synced) + self.synced_height = synced + + def get_peer_block_hashes(self, heights: List[int]) -> Deferred[List[Tuple[int, bytes]]]: + """ Returns the peer's block hashes in the given heights. + """ + key = 'peer-block-hashes' + if self.deferred_by_key.get(key, None) is not None: + raise Exception('latest_deferred is not None') + self.send_get_peer_block_hashes(heights) + deferred: Deferred[List[Tuple[int, bytes]]] = Deferred() + self.deferred_by_key[key] = deferred + return deferred + + def send_get_peer_block_hashes(self, heights: List[int]) -> None: + payload = json.dumps(heights) + self.send_message(ProtocolMessages.GET_PEER_BLOCK_HASHES, payload) + + def handle_get_peer_block_hashes(self, payload: str) -> None: + assert self.tx_storage.indexes is not None + heights = json.loads(payload) + data = [] + for h in heights: + block = self.tx_storage.indexes.height.get(h) + if block is None: + break + data.append((h, block.hex())) + payload = json.dumps(data) + self.send_message(ProtocolMessages.PEER_BLOCK_HASHES, payload) + + def handle_peer_block_hashes(self, payload: str) -> None: + data = json.loads(payload) + data = [(h, bytes.fromhex(block_hash)) for (h, block_hash) in data] + key = 'peer-block-hashes' + deferred = self.deferred_by_key.pop(key, None) + if deferred: + deferred.callback(data) + + def send_get_next_blocks(self, start_hash: bytes, end_hash: bytes) -> None: + payload = json.dumps(dict( + start_hash=start_hash.hex(), + end_hash=end_hash.hex(), + )) + self.send_message(ProtocolMessages.GET_NEXT_BLOCKS, payload) + self.receiving_stream = True + + def handle_get_next_blocks(self, payload: str) -> None: + self.log.debug('handle GET-NEXT-BLOCKS') + if self._is_streaming: + self.protocol.send_error_and_close_connection('GET-NEXT-BLOCKS received before previous one finished') + return + data = json.loads(payload) + self.send_next_blocks( + start_hash=bytes.fromhex(data['start_hash']), + end_hash=bytes.fromhex(data['end_hash']), + ) + + def send_next_blocks(self, start_hash: bytes, end_hash: bytes) -> None: + self.log.debug('start GET-NEXT-BLOCKS stream response') + # XXX If I don't have this block it will raise TransactionDoesNotExist error. Should I handle this? + blk = self.tx_storage.get_transaction(start_hash) + assert isinstance(blk, Block) + self.blockchain_streaming = BlockchainStreaming(self, blk, end_hash) + self.blockchain_streaming.start() + + def send_get_prev_blocks(self, start_hash: bytes, end_hash: bytes) -> None: + payload = json.dumps(dict( + start_hash=start_hash.hex(), + end_hash=end_hash.hex(), + )) + self.send_message(ProtocolMessages.GET_PREV_BLOCKS, payload) + self.receiving_stream = True + + def handle_get_prev_blocks(self, payload: str) -> None: + self.log.debug('handle GET-PREV-BLOCKS') + if self._is_streaming: + self.protocol.send_error_and_close_connection('GET-PREV-BLOCKS received before previous one finished') + return + data = json.loads(payload) + self.send_prev_blocks( + start_hash=bytes.fromhex(data['start_hash']), + end_hash=bytes.fromhex(data['end_hash']), + ) + + def send_prev_blocks(self, start_hash: bytes, end_hash: bytes) -> None: + self.log.debug('start GET-PREV-BLOCKS stream response') + # XXX If I don't have this block it will raise TransactionDoesNotExist error. Should I handle this? + # TODO + blk = self.tx_storage.get_transaction(start_hash) + assert isinstance(blk, Block) + self.blockchain_streaming = BlockchainStreaming(self, blk, end_hash, reverse=True) + self.blockchain_streaming.start() + + def send_blocks(self, blk: Block) -> None: + """Send a BLOCK message.""" + # self.log.debug('sending block to peer', block=blk.hash_hex) + payload = base64.b64encode(bytes(blk)).decode('ascii') + self.send_message(ProtocolMessages.BLOCKS, payload) + + def send_blocks_end(self, response_code: StreamEnd) -> None: + payload = str(int(response_code)) + self.log.debug('send BLOCKS-END', payload=payload) + self.send_message(ProtocolMessages.BLOCKS_END, payload) + + def handle_blocks_end(self, payload: str) -> None: + self.log.debug('recv BLOCKS-END', payload=payload, size=self._blk_size) + + response_code = StreamEnd(int(payload)) + self.receiving_stream = False + assert self.protocol.connections is not None + + if self.state not in [PeerState.SYNCING_BLOCKS, PeerState.SYNCING_CHECKPOINTS]: + self.log.error('unexpected BLOCKS-END', state=self.state) + self.protocol.send_error_and_close_connection('Not expecting to receive BLOCKS-END message') + return + + self.log.debug('block streaming ended', reason=str(response_code)) + + if self.is_syncing_checkpoints(): + if self._blk_height == self._blk_end_height: + # Tell the checkpoints sync that it's over and can continue + self.sync_checkpoints.on_stream_ends() + else: + self.sync_checkpoints.continue_sync() + else: + # XXX What should we do if it's in the next block sync phase? + return + + def handle_blocks(self, payload: str) -> None: + """Handle a received BLOCK message.""" + if self.state not in [PeerState.SYNCING_BLOCKS, PeerState.SYNCING_CHECKPOINTS]: + self.log.error('unexpected BLOCK', state=self.state) + self.protocol.send_error_and_close_connection('Not expecting to receive BLOCK message') + return + + assert self.protocol.connections is not None + + blk_bytes = base64.b64decode(payload) + blk = tx_or_block_from_bytes(blk_bytes) + if not isinstance(blk, Block): + # Not a block. Punish peer? + return + blk.storage = self.tx_storage + + assert blk.hash is not None + + self._blk_received += 1 + if self._blk_received > self._blk_max_quantity + 1: + self.log.warn('too many blocks received', last_block=blk.hash_hex) + # Too many blocks. Punish peer? + if self.is_syncing_checkpoints(): + # Tell the checkpoints sync to stop syncing from this peer and ban him + self.sync_checkpoints.on_sync_error() + + self.state = PeerState.ERROR + return + + if self.tx_storage.transaction_exists(blk.hash): + # We reached a block we already have. Skip it. + self._blk_prev_hash = blk.hash + self._blk_repeated += 1 + if self.receiving_stream and self._blk_repeated > self.max_repeated_blocks: + self.log.debug('repeated block received', total_repeated=self._blk_repeated) + self.handle_many_repeated_blocks() + + # basic linearity validation, crucial for correctly predicting the next block's height + if self._blk_stream_reverse: + if self._last_received_block and blk.hash != self._last_received_block.get_block_parent_hash(): + self.handle_invalid_block('received block is not parent of previous block') + return + else: + if self._last_received_block and blk.get_block_parent_hash() != self._last_received_block.hash: + self.handle_invalid_block('received block is not child of previous block') + return + + try: + # this methods takes care of checking if the block already exists, + # it will take care of doing at least a basic validation + # self.log.debug('add new block', block=blk.hash_hex) + is_syncing_checkpoints = self.is_syncing_checkpoints() + if is_syncing_checkpoints: + assert self._blk_height is not None + # XXX: maybe improve this, feels a bit hacky + blk.storage = self.tx_storage + blk.set_height(self._blk_height) + if self.manager.tx_storage.transaction_exists(blk.hash): + # XXX: early terminate? + self.log.debug('block early terminate?', blk_id=blk.hash.hex()) + else: + self.log.debug('block received', blk_id=blk.hash.hex()) + self.manager.on_new_tx(blk, propagate_to_peers=False, quiet=True, partial=True, + sync_checkpoints=is_syncing_checkpoints, + reject_locked_reward=not is_syncing_checkpoints) + except HathorError: + self.handle_invalid_block(exc_info=True) + return + else: + self._last_received_block = blk + self._blk_repeated = 0 + assert self._blk_height is not None + if self._blk_stream_reverse: + self._blk_height -= 1 + else: + self._blk_height += 1 + # XXX: debugging log, maybe add timing info + if self._blk_received % 500 == 0: + self.log.debug('block streaming in progress', blocks_received=self._blk_received, + next_height=self._blk_height) + + def handle_invalid_block(self, msg: Optional[str] = None, *, exc_info: bool = False) -> None: + """ Call this method when receiving an invalid block. + """ + kwargs: Dict[str, Any] = {} + if msg is not None: + kwargs['error'] = msg + if exc_info: + kwargs['exc_info'] = True + self.log.warn('invalid new block', **kwargs) + # Invalid block?! + if self.is_syncing_checkpoints(): + # Tell the checkpoints sync to stop syncing from this peer and ban him + assert self.protocol.connections is not None + self.sync_checkpoints.on_sync_error() + self.state = PeerState.ERROR + + def handle_many_repeated_blocks(self) -> None: + """ Method called when a block stream received many repeated blocks + so I must stop the stream and reschedule to continue the sync with this peer later + """ + self.send_stop_block_streaming() + self.receiving_stream = False + + def send_stop_block_streaming(self) -> None: + self.send_message(ProtocolMessages.STOP_BLOCK_STREAMING) + + def handle_stop_block_streaming(self, payload: str) -> None: + if not self.blockchain_streaming or not self._is_streaming: + self.log.debug('got stop streaming message with no streaming running') + return + + self.log.debug('got stop streaming message') + self.blockchain_streaming.stop() + self.blockchain_streaming = None + + def get_peer_best_block(self) -> Deferred: + key = 'best-block' + deferred = self.deferred_by_key.pop(key, None) + if self.deferred_by_key.get(key, None) is not None: + raise Exception('latest_deferred is not None') + + self.send_get_best_block() + deferred = Deferred() + self.deferred_by_key[key] = deferred + return deferred + + def send_get_best_block(self) -> None: + self.send_message(ProtocolMessages.GET_BEST_BLOCK) + + def handle_get_best_block(self, payload: str) -> None: + best_block = self.tx_storage.get_best_block() + meta = best_block.get_metadata() + data = {'block': best_block.hash_hex, 'height': meta.height} + self.send_message(ProtocolMessages.BEST_BLOCK, json.dumps(data)) + + def handle_best_block(self, payload: str) -> None: + data = json.loads(payload) + assert self.protocol.connections is not None + self.log.debug('got best block', **data) + data['block'] = bytes.fromhex(data['block']) + + key = 'best-block' + deferred = self.deferred_by_key.pop(key, None) + if deferred: + deferred.callback(data) + + def _setup_tx_streaming(self): + self._tx_received = 0 + self._tx_max_quantity = DEAFAULT_STREAMING_LIMIT # XXX: maybe this is redundant + # XXX: what else can we add for checking if everything is going well? + + def send_get_block_txs(self, child_hash: bytes, last_block_hash: bytes) -> None: + """ Request a BFS of all transactions that parent of CHILD, up to the ones first comfirmed by LAST-BLOCK. + + Note that CHILD can either be a block or a transaction. But LAST-BLOCK is always a block. + """ + self._setup_tx_streaming() + self.log.debug('send_get_block_txs', child=child_hash.hex(), last_block=last_block_hash.hex()) + payload = json.dumps(dict( + child=child_hash.hex(), + last_block=last_block_hash.hex(), + )) + self.send_message(ProtocolMessages.GET_BLOCK_TXS, payload) + self.receiving_stream = True + + def handle_get_block_txs(self, payload: str) -> None: + if self._is_streaming: + self.log.warn('already streaming') + # self.log.warn('ignore GET-BLOCK-TXS, already streaming') + # return + data = json.loads(payload) + self.log.debug('handle_get_block_txs', **data) + child_hash = bytes.fromhex(data['child']) + last_block_hash = bytes.fromhex(data['last_block']) + self.send_block_txs(child_hash, last_block_hash) + + def send_block_txs(self, child_hash: bytes, last_block_hash: bytes) -> None: + try: + tx = self.tx_storage.get_transaction(child_hash) + except TransactionDoesNotExist: + # In case the tx does not exist we send a NOT-FOUND message + self.send_message(ProtocolMessages.NOT_FOUND, child_hash.hex()) + return + if not self.tx_storage.transaction_exists(last_block_hash): + # In case the tx does not exist we send a NOT-FOUND message + self.send_message(ProtocolMessages.NOT_FOUND, last_block_hash.hex()) + return + x = TransactionsStreaming(self, tx, last_block_hash) + x.start() + + def send_transaction(self, tx: Transaction) -> None: + """Send a TRANSACTION message.""" + # payload = bytes(tx).hex() # fails for big transactions + payload = base64.b64encode(bytes(tx)).decode('ascii') + self.send_message(ProtocolMessages.TRANSACTION, payload) + + def handle_transaction(self, payload: str) -> None: + """Handle a received TRANSACTION message.""" + assert self.protocol.connections is not None + + # if self.state != PeerState.SYNCING_TXS: + # self.protocol.send_error_and_close_connection('Not expecting to receive transactions') + # return + + # tx_bytes = bytes.fromhex(payload) + tx_bytes = base64.b64decode(payload) + tx = tx_or_block_from_bytes(tx_bytes) + assert tx.hash is not None + if not isinstance(tx, Transaction): + self.log.warn('not a transaction', hash=tx.hash_hex) + # Not a transaction. Punish peer? + return + + self._tx_received += 1 + if self._tx_received > self._tx_max_quantity + 1: + self.log.warn('too many txs received') + # Too many blocks. Punish peer? + if self.is_syncing_checkpoints(): + # Tell the checkpoints sync to stop syncing from this peer and ban him + self.sync_checkpoints.on_sync_error() + + self.state = PeerState.ERROR + return + + try: + # this methods takes care of checking if the tx already exists, it will take care of doing at least + # a basic validation + # self.log.debug('add new tx', tx=tx.hash_hex) + is_syncing_checkpoints = self.is_syncing_checkpoints() + if self.manager.tx_storage.transaction_exists(tx.hash): + # XXX: early terminate? + self.log.debug('tx early terminate?', tx_id=tx.hash.hex()) + else: + self.log.debug('tx received', tx_id=tx.hash.hex()) + self.manager.on_new_tx(tx, propagate_to_peers=False, quiet=True, partial=True, + sync_checkpoints=is_syncing_checkpoints, + reject_locked_reward=not is_syncing_checkpoints) + except HathorError: + self.log.warn('invalid new tx', exc_info=True) + # Invalid block?! + # Invalid transaction?! + if self.is_syncing_checkpoints(): + assert self.protocol.connections is not None + # Tell the checkpoints sync to stop syncing from this peer and ban him + self.sync_checkpoints.on_sync_error() + # Maybe stop syncing and punish peer. + self.state = PeerState.ERROR + return + else: + # XXX: debugging log, maybe add timing info + if self._tx_received % 100 == 0: + self.log.debug('tx streaming in progress', txs_received=self._tx_received) + + # ----------------------------------- + # BEGIN: GET_DATA/DATA implementation + # ----------------------------------- + + @inlineCallbacks + def get_tx(self, tx_id: bytes) -> Generator[Deferred, Any, BaseTransaction]: + """Async method to get a transaction from the db/cache or to download it.""" + tx = self._get_tx_cache.get(tx_id) + if tx is not None: + self.log.debug('tx in cache', tx=tx_id.hex()) + return tx + try: + tx = self.tx_storage.get_transaction(tx_id) + except TransactionDoesNotExist: + tx = yield self.get_data(tx_id, 'mempool') + if tx is None: + self.log.error('failed to get tx', tx_id=tx_id.hex()) + self.protocol.send_error_and_close_connection(f'DATA mempool {tx_id.hex()} not found') + raise + # XXX: Verify tx? + # tx.verify() + return tx + + def get_data(self, tx_id: bytes, origin: str) -> Deferred: + """Async method to request a tx by id""" + # TODO: deal with stale `get_data` calls + if origin != 'mempool': + raise ValueError(f'origin={origin} not supported, only origin=mempool is supported') + key = f'{origin}:{tx_id.hex()}' + deferred = self.deferred_by_key.get(key, None) + if deferred is None: + deferred = self.deferred_by_key[key] = Deferred() + self.send_get_data(tx_id, origin=origin) + self.log.debug('get_data of new tx_id', deferred=deferred, key=key) + else: + # XXX: can we re-use deferred objects like this? + self.log.debug('get_data of same tx_id, reusing deferred', deferred=deferred, key=key) + return deferred + + def _on_get_data(self, tx: BaseTransaction, origin: str) -> None: + """Called when a requested tx is received.""" + assert tx.hash is not None + key = f'{origin}:{tx.hash_hex}' + deferred = self.deferred_by_key.pop(key, None) + if deferred is None: + # Peer sent the wrong transaction?! + # XXX: ban peer? + self.protocol.send_error_and_close_connection(f'DATA {origin}: with tx that was not requested') + return + self.log.debug('get_data fulfilled', deferred=deferred, key=key) + self._get_tx_cache[tx.hash] = tx + if len(self._get_tx_cache) > self._get_tx_cache_maxsize: + self._get_tx_cache.popitem(last=False) + deferred.callback(tx) + + def send_data(self, tx: BaseTransaction, *, origin: str = '') -> None: + """ Send a DATA message. + """ + self.log.debug('send tx', tx=tx.hash_hex) + tx_payload = base64.b64encode(tx.get_struct()).decode('ascii') + if not origin: + payload = tx_payload + else: + payload = ' '.join([origin, tx_payload]) + self.send_message(ProtocolMessages.DATA, payload) + + def send_get_data(self, txid: bytes, *, origin: Optional[str] = None) -> None: + """Send a GET-DATA message for a given txid.""" + data = { + 'txid': txid.hex(), + } + if origin is not None: + data['origin'] = origin + payload = json.dumps(data) + self.send_message(ProtocolMessages.GET_DATA, payload) + + def handle_get_data(self, payload: str) -> None: + """Handle a received GET-DATA message.""" + data = json.loads(payload) + txid_hex = data['txid'] + origin = data.get('origin', '') + # self.log.debug('handle_get_data', payload=hash_hex) + try: + tx = self.protocol.node.tx_storage.get_transaction(bytes.fromhex(txid_hex)) + self.send_data(tx, origin=origin) + except TransactionDoesNotExist: + # In case the tx does not exist we send a NOT-FOUND message + self.send_message(ProtocolMessages.NOT_FOUND, txid_hex) + + def handle_data(self, payload: str) -> None: + """ Handle a received DATA message. + """ + if not payload: + return + part1, _, part2 = payload.partition(' ') + if not part2: + origin = None + data = base64.b64decode(part1) + else: + origin = part1 + data = base64.b64decode(part2) + + try: + tx = tx_or_block_from_bytes(data) + except struct.error: + # Invalid data for tx decode + return + + if origin: + if origin != 'mempool': + # XXX: ban peer? + self.protocol.send_error_and_close_connection(f'DATA {origin}: unsupported origin') + return + assert tx is not None + self._on_get_data(tx, origin) + return + + assert tx is not None + assert tx.hash is not None + if self.protocol.node.tx_storage.get_genesis(tx.hash): + # We just got the data of a genesis tx/block. What should we do? + # Will it reduce peer reputation score? + return + + tx.storage = self.protocol.node.tx_storage + assert tx.hash is not None + + if self.manager.tx_storage.transaction_exists(tx.hash): + # transaction already added to the storage, ignore it + # XXX: maybe we could add a hash blacklist and punish peers propagating known bad txs + self.manager.tx_storage.compare_bytes_with_local_tx(tx) + return + else: + self.log.info('tx received in real time from peer', tx=tx.hash_hex, peer=self.protocol.get_peer_id()) + # If we have not requested the data, it is a new transaction being propagated + # in the network, thus, we propagate it as well. + self.manager.on_new_tx(tx, conn=self.protocol, propagate_to_peers=True) + + # --------------------------------- + # END: GET_DATA/DATA implementation + # --------------------------------- + + +DEAFAULT_STREAMING_LIMIT = 1_000 + + +@implementer(IPushProducer) +class _StreamingBase: + def __init__(self, node_sync: NodeBlockSync, *, limit: int = DEAFAULT_STREAMING_LIMIT): + self.node_sync = node_sync + self.protocol: 'HathorProtocol' = node_sync.protocol + assert self.protocol.transport is not None + self.consumer = verified_cast(IConsumer, self.protocol.transport) + + self.counter = 0 + self.limit = limit + + self.is_running: bool = False + self.is_producing: bool = False + + self.delayed_call: Optional[IDelayedCall] = None + self.log = logger.new(peer=node_sync.protocol.get_short_peer_id()) + + def schedule_if_needed(self) -> None: + """Schedule `send_next` if needed.""" + if not self.is_running: + return + + if not self.is_producing: + return + + if self.delayed_call and self.delayed_call.active(): + return + + self.delayed_call = self.node_sync.reactor.callLater(0, self.send_next) + + def start(self) -> None: + """Start pushing.""" + self.log.debug('start streaming') + assert not self.node_sync._is_streaming + self.node_sync._is_streaming = True + self.is_running = True + self.consumer.registerProducer(self, True) + self.resumeProducing() + + def stop(self) -> None: + """Stop pushing.""" + self.log.debug('stop streaming') + assert self.node_sync._is_streaming + self.is_running = False + self.pauseProducing() + self.consumer.unregisterProducer() + self.node_sync._is_streaming = False + + def send_next(self) -> None: + """Push next block to peer.""" + raise NotImplementedError + + def resumeProducing(self) -> None: + """This method is automatically called to resume pushing data.""" + self.is_producing = True + self.schedule_if_needed() + + def pauseProducing(self) -> None: + """This method is automatically called to pause pushing data.""" + self.is_producing = False + if self.delayed_call and self.delayed_call.active(): + self.delayed_call.cancel() + + def stopProducing(self) -> None: + """This method is automatically called to stop pushing data.""" + self.pauseProducing() + + +class BlockchainStreaming(_StreamingBase): + def __init__(self, node_sync: NodeBlockSync, start_block: Block, end_hash: bytes, + *, limit: int = DEAFAULT_STREAMING_LIMIT, reverse: bool = False): + super().__init__(node_sync, limit=limit) + + self.start_block = start_block + self.current_block: Optional[Block] = start_block + self.end_hash = end_hash + self.reverse = reverse + + def send_next(self) -> None: + """Push next block to peer.""" + assert self.is_running + assert self.is_producing + + cur = self.current_block + assert cur is not None + assert cur.hash is not None + + if cur.hash == self.end_hash: + # only send the last when not reverse + if not self.reverse: + self.log.debug('send next block', blk_id=cur.hash.hex()) + self.node_sync.send_blocks(cur) + self.stop() + self.node_sync.send_blocks_end(StreamEnd.END_HASH_REACHED) + return + + if self.counter >= self.limit: + # only send the last when not reverse + if not self.reverse: + self.log.debug('send next block', blk_id=cur.hash.hex()) + self.node_sync.send_blocks(cur) + self.stop() + self.node_sync.send_blocks_end(StreamEnd.LIMIT_EXCEEDED) + return + + if cur.get_metadata().voided_by: + self.stop() + self.node_sync.send_blocks_end(StreamEnd.STREAM_BECAME_VOIDED) + return + + self.counter += 1 + + self.log.debug('send next block', blk_id=cur.hash.hex()) + self.node_sync.send_blocks(cur) + + if self.reverse: + self.current_block = cur.get_block_parent() + else: + self.current_block = cur.get_next_block_best_chain() + + # XXX: don't send the genesis or the current block + if self.current_block is None or self.current_block.is_genesis: + self.stop() + self.node_sync.send_blocks_end(StreamEnd.NO_MORE_BLOCKS) + return + + self.schedule_if_needed() + + +class TransactionsStreaming(_StreamingBase): + """Streams all transactions confirmed by the given block, from right to left (decreasing timestamp). + """ + + def __init__(self, node_sync: NodeBlockSync, child: BaseTransaction, last_block_hash: bytes, + *, limit: int = DEAFAULT_STREAMING_LIMIT): + # XXX: is limit needed for tx streaming? Or let's always send all txs for + # a block? Very unlikely we'll reach this limit + super().__init__(node_sync, limit=limit) + + assert child.storage is not None + self.storage = child.storage + self.child = child + self.last_block_hash = last_block_hash + self.last_block_height = 0 + + self.bfs = BFSWalk(child.storage, is_dag_verifications=True, is_left_to_right=False) + # self.iter = self.bfs.run(child, skip_root=True) + self.iter = self.bfs.run(child, skip_root=False) + + def start(self) -> None: + super().start() + last_blk_meta = self.storage.get_metadata(self.last_block_hash) + assert last_blk_meta is not None + self.last_block_height = last_blk_meta.get_soft_height() + + # TODO: make this generic too? + def send_next(self) -> None: + """Push next transaction to peer.""" + assert self.is_running + assert self.is_producing + + try: + cur = next(self.iter) + except StopIteration: + # nothing more to send + self.stop() + self.node_sync.send_blocks_end(StreamEnd.END_HASH_REACHED) + return + + if cur.is_block: + if cur.hash == self.last_block_hash: + self.bfs.skip_neighbors(cur) + self.schedule_if_needed() + return + + assert isinstance(cur, Transaction) + assert cur.hash is not None + + cur_metadata = cur.get_metadata() + if cur_metadata.voided_by: + self.stop() + self.node_sync.send_blocks_end(StreamEnd.STREAM_BECAME_VOIDED) + return + + assert cur_metadata.first_block is not None + first_blk_meta = self.storage.get_metadata(cur_metadata.first_block) + assert first_blk_meta is not None + + confirmed_by_height = first_blk_meta.height + if confirmed_by_height <= self.last_block_height: + # got to a tx that is confirmed by the given last-block or an older block + self.log.debug('tx confirmed by block older than last_block', tx=cur.hash_hex, + confirmed_by_height=confirmed_by_height, last_block_height=self.last_block_height) + self.bfs.skip_neighbors(cur) + self.schedule_if_needed() + return + + self.log.debug('send next transaction', tx_id=cur.hash.hex()) + self.node_sync.send_transaction(cur) + + self.counter += 1 + if self.counter >= self.limit: + self.stop() + self.node_sync.send_blocks_end(StreamEnd.LIMIT_EXCEEDED) + return + + self.schedule_if_needed() diff --git a/hathor/p2p/sync_checkpoints.py b/hathor/p2p/sync_checkpoints.py new file mode 100644 index 000000000..14ca13914 --- /dev/null +++ b/hathor/p2p/sync_checkpoints.py @@ -0,0 +1,306 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import chain +from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional + +from structlog import get_logger +from twisted.internet.task import LoopingCall + +from hathor.checkpoint import Checkpoint +from hathor.transaction.block import Block +from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.util import Reactor + +if TYPE_CHECKING: + from hathor.manager import HathorManager # noqa: F401 + from hathor.p2p.node_sync_v2 import NodeBlockSync # noqa: F401 + +logger = get_logger() + + +class _SyncInterval(NamedTuple): + start_hash: bytes + start_height: int + end_hash: bytes + end_height: int + + +class SyncCheckpoint: + """This is the central manager of the sync between checkpoints among all peers. + """ + + # Looping call interval. + LC_INTERVAL: int = 5 + + def __init__(self, manager: 'HathorManager'): + # All peers that have all the checkpoints to download + self.peers_to_request: List['NodeBlockSync'] = [] + + # All peers that we are connected but don't have all the checkpoints + self.incomplete_peers: List['NodeBlockSync'] = [] + + # All peers that we tried to download but they sent wrong blocks + self.banned_peers: List['NodeBlockSync'] = [] + + # Indicate whether the checkpoint sync has been started. + self._started: bool = False + + # HathorManager object to get checkpoints and storage + self.manager: 'HathorManager' = manager + + self.reactor: Reactor = manager.reactor + + # All checkpoints that still need to sync + self.checkpoints_to_sync: List[Checkpoint] = [] + + # Previous checkpoints map + self.previous_checkpoints: Dict[Checkpoint, Checkpoint] = {} + + # The peer that is syncing (the one we are downloading the blocks from) + self.peer_syncing = None + + # If set to true next run_sync_transactions will be skipped + self.should_skip_sync_tx = False + + # Create logger with context + self.log = logger.new() + + # Looping call of the main method + self._lc_run = LoopingCall(self.run_sync) + self._lc_run.clock = self.reactor + self._is_running = False + + def is_started(self) -> bool: + return self._is_running + + def start(self) -> bool: + """Start sync between checkpoints. + """ + if self._started: + self.log.warn('already running, not starting new one') + return False + + self.log.info('start checkpoint sync') + checkpoints = self.manager.checkpoints + bestblock = self.manager.tx_storage.get_best_block() + meta = bestblock.get_metadata() + assert meta.validation.is_fully_connected() + + # Fill the previous checkpoints map + it_cps = iter(checkpoints) + prev_cp = next(it_cps) + for cp in it_cps: + self.previous_checkpoints[cp] = prev_cp + prev_cp = cp + + # Get all checkpoints to sync + self.checkpoints_to_sync = [checkpoint for checkpoint in checkpoints if checkpoint.height > meta.height] + + if not self.checkpoints_to_sync: + self.log.error('something went wrong, no checkpoints to sync') + # Should start this only if there are missing checkpoints on storage + return False + + self._started = True + self._lc_run.start(self.LC_INTERVAL) + return True + + def stop(self) -> bool: + """Stop sync between checkpoints. + """ + if not self._started: + self.log.warn('already stopped') + return False + if self.peer_syncing is not None: + if self.peer_syncing in self.peers_to_request: + self.peers_to_request.remove(self.peer_syncing) + self.peer_syncing = None + self._started = False + self._lc_run.stop() + self.log.debug('stop sync') + return True + + def _checkpoint_sync_interval(self, checkpoint: Checkpoint) -> Optional[_SyncInterval]: + """Calculate start and end point of a checkpoint.""" + start_height, start_hash = checkpoint + end_height, end_hash = self.previous_checkpoints[checkpoint] + # XXX: this could be optimized a lot, but it actually isn't that slow + while start_height > end_height: + try: + block = self.manager.tx_storage.get_transaction(start_hash) + except TransactionDoesNotExist: + break + assert isinstance(block, Block) + start_hash = block.get_block_parent_hash() # parent hash + start_height = block.get_metadata().get_soft_height() - 1 # parent height + # don't try to sync checkpoints that we already have all the blocks for + if start_height == end_height: + return None + assert start_height > end_height + return _SyncInterval(start_hash, start_height, end_hash, end_height) + + def _get_next_sync_interval(self) -> Optional[_SyncInterval]: + """Iterate over checkpoints_to_sync and find a valid interval to sync, pruning already synced intervals. + + Will only return None when there are no more intervals to sync. + """ + for checkpoint in self.checkpoints_to_sync[::]: + sync_interval = self._checkpoint_sync_interval(checkpoint) + if sync_interval is not None: + return sync_interval + else: + self.checkpoints_to_sync.remove(checkpoint) + return None + + def run_sync(self): + """Run sync. This is the entrypoint for the sync. + It is always safe to call this method. + """ + assert self._started + self.log.info('try to sync checkpoints') + + if self._is_running: + self.log.debug('still running') + return + + if self.peer_syncing: + if self.peer_syncing.protocol.aborting: + self.log.warn('syncing peer disconnected') + self.stop() + else: + self.log.debug('already syncing') + return + + self._is_running = True + try: + peer_to_request = self.get_peer_to_request() + if peer_to_request: + self.peer_syncing = peer_to_request + self._run_sync() + else: + self.log.debug('no peers to sync from, try again later') + finally: + self._is_running = False + + def get_peer_to_request(self) -> Optional['NodeBlockSync']: + """ + """ + # XXX: we could use a better peer selecting strategy here + for peer in self.peers_to_request[:]: + if peer.protocol.state is None: + self.peers_to_request.remove(peer) + else: + return peer + return None + + def _run_sync(self): + """Method that actually run the sync. + It should never be called directly. + """ + assert self._started + assert self.manager.tx_storage.indexes is not None + + if self.peer_syncing.protocol.state is None: + self.log.error('lost state, something went wrong') + self.stop() + + if self.manager.tx_storage.indexes.deps.has_needed_tx() and not self.should_skip_sync_tx: + # Streaming ended. If there are needed txs, prioritize that + return self.peer_syncing.run_sync_transactions() + + if not self.checkpoints_to_sync: + self.log.debug('no checkpoints to sync') + return + + sync_interval = self._get_next_sync_interval() + if not sync_interval: + self.log.debug('no checkpoints to sync anymore') + return + + self.peer_syncing.synced_height = sync_interval.end_height + self.peer_syncing.run_sync_between_heights(*sync_interval) + + # XXX: reset skip flag + self.should_skip_sync_tx = False + + def continue_sync(self) -> None: + """Restart peer selection and wait for the next looping call. + """ + self.peer_syncing = None + + def on_sync_error(self): + """Called by NodeBlockSync when an error occurs (e.g. receive invalid blocks or receive too many blocks).""" + self.log.debug('sync error') + # Send peer_syncing to banned_peers and start again with another peer + self.banned_peers.append(self.peer_syncing) + self.peers_to_request.remove(self.peer_syncing) + self.peer_syncing = None + + def on_stream_ends(self): + """Called by NodeBlockSync when the streaming of blocks is ended. + """ + assert self.manager.tx_storage.indexes is not None + + self.log.debug('sync stream ended') + + if self.manager.tx_storage.indexes.deps.has_needed_tx(): + self.log.debug('checkpoint sync not complete: has needed txs') + self.continue_sync() + return + + if not self.checkpoints_to_sync: + self.log.debug('checkpoint sync not complete: no checkpoints to sync') + return + + # Double check I have the first checkpoint + first_cp = self.checkpoints_to_sync[0] + + if not self.manager.tx_storage.transaction_exists(first_cp.hash): + # the sync ended but I still don't have the checkpoint + self.log.debug('checkpoint sync not complete: checkpoint not found') + self.continue_sync() + return + + # Everything went fine and I have all blocks until the next checkpoint + self.checkpoints_to_sync.remove(first_cp) + + if self.checkpoints_to_sync: + # Sync until next checkpoint again if I still have unsynced checkpoints + # self.run_sync() + self.log.debug('checkpoint sync not complete: checkpoint not found') + self.continue_sync() + return + + # All blocks are downloaded until the last checkpoint. + # So, we stop the checkpoint sync and mark all connections as checkpoint finished. + # XXX Should we execute ban for the banned_peers list? How long the ban? + self.log.debug('stop all sync-checkpoints') + for peer_sync in chain(self.peers_to_request, self.incomplete_peers, self.banned_peers): + peer_sync.sync_checkpoints_finished() + self.stop() + + def update_peer_height(self, peer: 'NodeBlockSync', height: int) -> None: + """Called by NodeBlockSync when we have updated information about a peers height.""" + if height >= self.manager.checkpoints[-1].height: + if peer in self.incomplete_peers: + self.incomplete_peers.remove(peer) + # This peer has all checkpoints + self.peers_to_request.append(peer) + else: + # XXX: Maybe this isn't possible, but just in case + if peer in self.peers_to_request: + self.peers_to_request.remove(peer) + # This peer does not have all checkpoints + self.incomplete_peers.append(peer) diff --git a/hathor/p2p/sync_mempool.py b/hathor/p2p/sync_mempool.py new file mode 100644 index 000000000..e095660f2 --- /dev/null +++ b/hathor/p2p/sync_mempool.py @@ -0,0 +1,116 @@ +# Copyright 2020 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import deque +from typing import TYPE_CHECKING, Any, Deque, Generator, List, Optional, Set + +from structlog import get_logger +from twisted.internet.defer import Deferred, inlineCallbacks + +from hathor.transaction import BaseTransaction + +if TYPE_CHECKING: + from hathor.p2p.node_sync_v2 import NodeBlockSync + +logger = get_logger() + + +class SyncMempoolManager: + """Manage the sync-v2 mempool with one peer. + """ + def __init__(self, sync_manager: 'NodeBlockSync'): + """Initialize the sync-v2 mempool manager.""" + self.log = logger.new(peer=sync_manager.protocol.get_short_peer_id()) + + # Shortcuts. + self.sync_manager = sync_manager + self.manager = self.sync_manager.manager + self.tx_storage = self.manager.tx_storage + self.reactor = self.sync_manager.reactor + + # Set of tips we know but couldn't add to the DAG yet. + self.missing_tips: Set[bytes] = set() + + # Maximum number of items in the DFS. + self.MAX_STACK_LENGTH: int = 1000 + + # Looping call of the main method + self._is_running = False + + def is_running(self) -> bool: + """Whether the sync-mempool is currently running.""" + return self._is_running + + def run(self) -> None: + """Starts _run in, won't start again if already running.""" + if self.is_running(): + self.log.warn('already started') + else: + self._is_running = True + self.reactor.callLater(0, self._run) + + @inlineCallbacks + def _run(self) -> Generator[Deferred, Any, None]: + """Run a single loop of the sync-v2 mempool.""" + if not self.missing_tips: + # No missing tips? Let's get them! + tx_hashes: List[bytes] = yield self.sync_manager.get_tips() + self.missing_tips.update(tx_hashes) + + while self.missing_tips: + self.log.debug('We have missing tips! Let\'s start!', missing_tips=[x.hex() for x in self.missing_tips]) + tx_id = next(iter(self.missing_tips)) + tx: BaseTransaction = yield self.sync_manager.get_tx(tx_id) + # Stack used by the DFS in the dependencies. + # We use a deque for performance reasons. + self.log.debug('start mempool DSF', tx=tx.hash_hex) + yield self._dfs(deque([tx])) + + # sync_manager.run_sync will start it again when needed + self._is_running = False + + @inlineCallbacks + def _dfs(self, stack: Deque[BaseTransaction]) -> Generator[Deferred, Any, None]: + """DFS method.""" + while stack: + tx = stack[-1] + self.log.debug('step mempool DSF', tx=tx.hash_hex, stack_len=len(stack)) + missing_dep = self._next_missing_dep(tx) + if missing_dep is None: + self.log.debug(r'No dependencies missing! \o/') + self._add_tx(tx) + assert tx == stack.pop() + else: + self.log.debug('Iterate in the DFS.', missing_dep=missing_dep.hex()) + tx_dep = yield self.sync_manager.get_tx(missing_dep) + stack.append(tx_dep) + if len(stack) > self.MAX_STACK_LENGTH: + stack.popleft() + + def _next_missing_dep(self, tx: BaseTransaction) -> Optional[bytes]: + """Get the first missing dependency found of tx.""" + assert not tx.is_block + for txin in tx.inputs: + if not self.tx_storage.transaction_exists(txin.tx_id): + return txin.tx_id + for parent in tx.parents: + if not self.tx_storage.transaction_exists(parent): + return parent + return None + + def _add_tx(self, tx: BaseTransaction) -> None: + """Add tx to the DAG.""" + assert tx.hash is not None + self.missing_tips.discard(tx.hash) + self.manager.on_new_tx(tx) diff --git a/hathor/p2p/sync_v2_factory.py b/hathor/p2p/sync_v2_factory.py new file mode 100644 index 000000000..93fc7eeea --- /dev/null +++ b/hathor/p2p/sync_v2_factory.py @@ -0,0 +1,34 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Optional + +from hathor.p2p.manager import ConnectionsManager +from hathor.p2p.node_sync_v2 import NodeBlockSync +from hathor.p2p.sync_checkpoints import SyncCheckpoint +from hathor.p2p.sync_factory import SyncManagerFactory +from hathor.p2p.sync_manager import SyncManager +from hathor.util import Reactor + +if TYPE_CHECKING: + from hathor.p2p.protocol import HathorProtocol + + +class SyncV2Factory(SyncManagerFactory): + def __init__(self, connections: ConnectionsManager): + # Object that handles the sync until the last checkpoint for all peers + self.sync_checkpoints = SyncCheckpoint(connections.manager) + + def create_sync_manager(self, protocol: 'HathorProtocol', reactor: Optional[Reactor] = None) -> SyncManager: + return NodeBlockSync(protocol, sync_checkpoints=self.sync_checkpoints, reactor=reactor) diff --git a/hathor/p2p/sync_version.py b/hathor/p2p/sync_version.py index 906ef8611..c92bcd7b2 100644 --- a/hathor/p2p/sync_version.py +++ b/hathor/p2p/sync_version.py @@ -23,7 +23,7 @@ class SyncVersion(Enum): # example, peers using `v2-fake` (which just uses sync-v1) will not connect to peers using `v2-alpha`, and so # on. V1 = 'v1' - V2 = 'v2-fake' # uses sync-v1 to mock sync-v2 + V2 = 'v2-alpha' def __str__(self): return f'sync-{self.value}' diff --git a/hathor/simulator/fake_connection.py b/hathor/simulator/fake_connection.py index f671a2a3b..5f3877e97 100644 --- a/hathor/simulator/fake_connection.py +++ b/hathor/simulator/fake_connection.py @@ -13,7 +13,7 @@ # limitations under the License. from collections import deque -from typing import TYPE_CHECKING, Deque +from typing import TYPE_CHECKING, Deque, Optional from OpenSSL.crypto import X509 from structlog import get_logger @@ -157,9 +157,16 @@ def run_one_step(self, debug=False, force=False): return True - def run_until_complete(self, debug=False, force=False): + def run_until_empty(self, max_steps: Optional[int] = None, debug: bool = False, force: bool = False) -> None: + """ Step until the connection reports as empty, optionally raise an assert if it takes more than `max_steps`. + """ + steps = 0 while not self.is_empty(): + steps += 1 + if max_steps is not None and steps > max_steps: + raise AssertionError('took more steps than expected') self.run_one_step(debug=debug, force=force) + self.log.debug('conn empty', steps=steps) def _deliver_message(self, proto, data, debug=False): proto.dataReceived(data) diff --git a/hathor/simulator/simulator.py b/hathor/simulator/simulator.py index e21f6f4fe..2ca4b414e 100644 --- a/hathor/simulator/simulator.py +++ b/hathor/simulator/simulator.py @@ -229,16 +229,27 @@ def _run(self, interval: float, step: float, status_interval: float) -> Generato def run_until_complete(self, max_interval: float, + min_interval: float = 0.0, step: float = DEFAULT_STEP_INTERVAL, status_interval: float = DEFAULT_STATUS_INTERVAL) -> bool: """ Will stop when all peers have synced/errored (-> True), or when max_interval is elapsed (-> False). + Optionally keep running for at least `min_interval` ignoring the stop condition. + Make sure miners/tx_generators are stopped or this will almost certainly run until max_interval. """ assert self._started + steps = 0 + interval = 0.0 + initial = self._clock.seconds() for _ in self._run(max_interval, step, status_interval): - if all(not conn.can_step() for conn in self._connections): + steps += 1 + latest_time = self._clock.seconds() + interval = latest_time - initial + if interval > min_interval and all(not conn.can_step() for conn in self._connections): + self.log.debug('run_until_complete: all done', steps=steps, dt=interval) return True + self.log.debug('run_until_complete: max steps exceeded', steps=steps, dt=interval) return False def run(self, diff --git a/hathor/simulator/tx_generator.py b/hathor/simulator/tx_generator.py index f7bd337aa..37c16676f 100644 --- a/hathor/simulator/tx_generator.py +++ b/hathor/simulator/tx_generator.py @@ -13,7 +13,7 @@ # limitations under the License. from collections import deque -from typing import TYPE_CHECKING, Deque, List +from typing import TYPE_CHECKING, Deque, Iterator, List from structlog import get_logger @@ -136,3 +136,16 @@ def new_tx_step1(self): self.tx = tx self.delayedcall = self.clock.callLater(dt, self.schedule_next_transaction) self.log.debug('randomized step: schedule next transaction', dt=dt, hash=tx.hash_hex) + + def yield_until_find_a_transaction(self, *, max_steps: int) -> Iterator[None]: + """ This method returns a dummy iterator that continuously yield until there's a tx or max_steps is reached. + + This method is useful for avoiding an infinite loop in case there's something wrong that causes a transaction + to never be generated, which is common in tests. + """ + for _ in range(max_steps): + if self.latest_transactions: + break + yield + else: + raise AssertionError('took too long to generate a transaction') diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index ce728102f..cbca7beb0 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -519,6 +519,15 @@ def validate_full(self, skip_block_weight_verification: bool = False, sync_check from hathor.transaction.transaction_metadata import ValidationState meta = self.get_metadata() + + # XXX: blocks can have been created previously with soft_height, we'll remove it and add a proper height + if self.is_block: + soft_height = meta.soft_height + meta.soft_height = None + meta.height = self.calculate_height() + if soft_height is not None: + assert soft_height == meta.height, 'A wrong height was previously set as soft height. Sync bug?' + # skip full validation when it is a checkpoint if meta.validation.is_checkpoint(): meta.validation = ValidationState.CHECKPOINT_FULL @@ -1057,6 +1066,17 @@ def clone(self) -> 'BaseTransaction': def get_token_uid(self, index: int) -> bytes: raise NotImplementedError + def is_ready_for_validation(self) -> bool: + """Check whether the transaction is ready to be validated: all dependencies exist and are fully connected.""" + assert self.storage is not None + for dep_hash in self.get_all_dependencies(): + dep_meta = self.storage.get_metadata(dep_hash) + if dep_meta is None: + return False + if not dep_meta.validation.is_fully_connected(): + return False + return True + class TxInput: _tx: BaseTransaction # XXX: used for caching on hathor.transaction.Transaction.get_spent_tx diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 75a60e66d..3fcd8878b 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -24,7 +24,6 @@ from hathor.transaction.exceptions import ( BlockWithInputs, BlockWithTokensError, - CheckpointError, InvalidBlockReward, RewardLocked, TransactionDataError, @@ -93,6 +92,26 @@ def create_from_struct(cls, struct_bytes: bytes, storage: Optional['TransactionS return blc + def set_height(self, height: int) -> None: + """This method exists to set the height metadata when we can't calculate it yet (i.e. syncing checkpoints). + + `TxValidationError` will be risen if a height already exists (which is not determined by its value but by the + validation state, that is, a height of `None` on a full validated transaction is final) and it doesn't match + the given hint. This is in place in order to prevent accidentally setting the height to a wrong value after the + value has been validated, the height must be checked when basic-validating this tx and effort should be made to + always set the correct height when using this method. + """ + from hathor.transaction import TransactionMetadata + from hathor.transaction.exceptions import TxValidationError + assert self.storage is not None + assert self.hash is not None + metadata = self.storage.get_metadata(self.hash) + if metadata is None: + metadata = TransactionMetadata(hash=self.hash, accumulated_weight=self.weight, soft_height=height) + elif metadata.validation.is_fully_connected() and metadata.height != height: + raise TxValidationError(f'wrong height hint {height}, stored height is {metadata.height}') + self._metadata = metadata + def calculate_height(self) -> int: """Return the height of the block, i.e., the number of blocks since genesis""" if self.is_genesis: @@ -267,14 +286,16 @@ def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: assert self.storage is not None meta = self.get_metadata() # XXX: it's fine to use `in` with NamedTuples - if Checkpoint(meta.height, self.hash) in checkpoints: + if Checkpoint(meta.get_soft_height(), self.hash) in checkpoints: return # otherwise at least one child must be checkpoint validated - for child_tx in map(self.storage.get_transaction, meta.children): - if child_tx.get_metadata().validation.is_checkpoint(): - return - raise CheckpointError(f'Invalid new block {self.hash_hex}: expected to reach a checkpoint but none of ' - 'its children is checkpoint-valid and its hash does not match any checkpoint') + # XXX: because we decided to remove "partial" children metadata, this won't work, we could add a new metadata + # field to track this + # for child_tx in map(self.storage.get_transaction, meta.children): + # if child_tx.get_metadata().validation.is_checkpoint(): + # return + # raise CheckpointError(f'Invalid new block {self.hash_hex}: expected to reach a checkpoint but none of ' + # 'its children is checkpoint-valid and its hash does not match any checkpoint') def verify_weight(self) -> None: """Validate minimum block difficulty.""" diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index c8b920a00..602dda4e2 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -682,6 +682,7 @@ def topological_iterator(self) -> Iterator[BaseTransaction]: assert self.indexes is not None if self._always_use_topological_dfs: + self.log.debug('force choosing DFS iterator') return self._topological_sort_dfs() db_last_started_at = self.get_last_started_at() @@ -694,8 +695,10 @@ def topological_iterator(self) -> Iterator[BaseTransaction]: iter_tx: Iterator[BaseTransaction] if can_use_timestamp_index: + self.log.debug('choosing timestamp-index iterator') iter_tx = self._topological_sort_timestamp_index() else: + self.log.debug('choosing metadata iterator') iter_tx = self._topological_sort_metadata() return iter_tx @@ -920,6 +923,14 @@ def iter_mempool_from_tx_tips(self) -> Iterator[Transaction]: assert isinstance(tx, Transaction) yield tx + def iter_mempool_tips_from_best_index(self) -> Iterator[Transaction]: + """Get tx tips in the mempool, using the best available index (mempool_tips or tx_tips)""" + assert self.indexes is not None + if self.indexes.mempool_tips is not None: + yield from self.indexes.mempool_tips.iter(self) + else: + yield from self.iter_mempool_tips_from_tx_tips() + def iter_mempool_from_best_index(self) -> Iterator[Transaction]: """Get all transactions in the mempool, using the best available index (mempool_tips or tx_tips)""" assert self.indexes is not None diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index dd87cc240..cb05539bf 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -122,7 +122,8 @@ class TransactionMetadata: _last_spent_by_hash: Optional[int] def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: Optional[bytes] = None, - accumulated_weight: float = 0, score: float = 0, height: int = 0, min_height: int = 0) -> None: + accumulated_weight: float = 0, score: float = 0, height: int = 0, min_height: int = 0, + soft_height: Optional[int] = None) -> None: from hathor.transaction.genesis import is_genesis # Hash of the transaction. @@ -175,6 +176,9 @@ def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: # Min height self.min_height = min_height + # Soft height + self.soft_height = soft_height + # Validation self.validation = ValidationState.INITIAL @@ -239,7 +243,7 @@ def __eq__(self, other: Any) -> bool: return False for field in ['hash', 'conflict_with', 'voided_by', 'received_by', 'children', 'accumulated_weight', 'twins', 'score', - 'first_block', 'validation', 'min_height']: + 'first_block', 'validation', 'min_height', 'soft_height']: if (getattr(self, field) or None) != (getattr(other, field) or None): return False @@ -278,6 +282,8 @@ def to_json(self) -> Dict[str, Any]: data['first_block'] = self.first_block.hex() else: data['first_block'] = None + if self.soft_height is not None: + data['soft_height'] = self.soft_height data['validation'] = self.validation.name.lower() return data @@ -319,6 +325,9 @@ def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata': else: meta.twins = [] + if 'soft_height' in data: + meta.soft_height = data['soft_height'] + meta.accumulated_weight = data['accumulated_weight'] meta.score = data.get('score', 0) meta.height = data.get('height', 0) # XXX: should we calculate the height if it's not defined? @@ -344,3 +353,10 @@ def clone(self) -> 'TransactionMetadata': """ # XXX: using json serialization for simplicity, should it use pickle? manual fields? other alternative? return self.create_from_json(self.to_json()) + + def get_soft_height(self) -> int: + """ Returns the soft-height, which is either the soft_height or height metadata. + """ + if self.soft_height is not None: + return self.soft_height + return self.height diff --git a/tests/consensus/test_consensus.py b/tests/consensus/test_consensus.py index 3903e8850..62af698be 100644 --- a/tests/consensus/test_consensus.py +++ b/tests/consensus/test_consensus.py @@ -94,8 +94,8 @@ def test_dont_revert_block_low_weight(self): manager = self.create_peer('testnet', tx_storage=self.tx_storage) # Mine a few blocks in a row with no transaction but the genesis - blocks = add_new_blocks(manager, 3, advance_clock=15) - add_blocks_unlock_reward(manager) + add_new_blocks(manager, 3, advance_clock=15) + blocks = add_blocks_unlock_reward(manager) # Add some transactions between blocks add_new_transactions(manager, 5, advance_clock=15) diff --git a/tests/consensus/test_soft_voided.py b/tests/consensus/test_soft_voided.py index 2a952fdc3..c0638f3ca 100644 --- a/tests/consensus/test_soft_voided.py +++ b/tests/consensus/test_soft_voided.py @@ -9,7 +9,7 @@ class BaseSoftVoidedTestCase(SimulatorTestCase): - seed_config = 5988775361793628169 + seed_config = 15574446753983525568 def assertNoParentsAreSoftVoided(self, tx): for h in tx.parents: @@ -43,7 +43,7 @@ def _run_test(self, simulator, soft_voided_tx_ids): gen_tx2 = simulator.create_tx_generator(manager2, rate=10 / 60., hashpower=1e6, ignore_no_funds=True) gen_tx2.start() - while not gen_tx2.latest_transactions: + for _ in gen_tx2.yield_until_find_a_transaction(max_steps=100): simulator.run(600) yield gen_tx2 diff --git a/tests/consensus/test_soft_voided3.py b/tests/consensus/test_soft_voided3.py index b6b7be415..dfd3bffdc 100644 --- a/tests/consensus/test_soft_voided3.py +++ b/tests/consensus/test_soft_voided3.py @@ -44,7 +44,7 @@ def _run_test(self, simulator, soft_voided_tx_ids): gen_tx2 = simulator.create_tx_generator(manager2, rate=10 / 60., hashpower=1e6, ignore_no_funds=True) gen_tx2.start() - while not gen_tx2.latest_transactions: + for _ in gen_tx2.yield_until_find_a_transaction(max_steps=100): simulator.run(300) yield gen_tx2 diff --git a/tests/consensus/test_soft_voided4.py b/tests/consensus/test_soft_voided4.py index 0d557d8f0..2897796aa 100644 --- a/tests/consensus/test_soft_voided4.py +++ b/tests/consensus/test_soft_voided4.py @@ -38,7 +38,7 @@ def _run_test(self, simulator, soft_voided_tx_ids): gen_tx2 = simulator.create_tx_generator(manager2, rate=10 / 60., hashpower=1e6, ignore_no_funds=True) gen_tx2.start() - while not gen_tx2.latest_transactions: + for _ in gen_tx2.yield_until_find_a_transaction(max_steps=100): simulator.run(600) yield gen_tx2 diff --git a/tests/p2p/test_capabilities.py b/tests/p2p/test_capabilities.py index d2e4f5737..c7d3f6a6f 100644 --- a/tests/p2p/test_capabilities.py +++ b/tests/p2p/test_capabilities.py @@ -1,4 +1,6 @@ from hathor.conf import HathorSettings +from hathor.p2p.node_sync import NodeSyncTimestamp +from hathor.p2p.node_sync_v2 import NodeBlockSync from hathor.simulator import FakeConnection from tests import unittest @@ -21,6 +23,8 @@ def test_capabilities(self): # Even if we don't have the capability we must connect because the whitelist url conf is None self.assertEqual(conn._proto1.state.state_name, 'READY') self.assertEqual(conn._proto2.state.state_name, 'READY') + self.assertIsInstance(conn._proto1.state.sync_manager, NodeSyncTimestamp) + self.assertIsInstance(conn._proto2.state.sync_manager, NodeSyncTimestamp) manager3 = self.create_peer(network, capabilities=[settings.CAPABILITY_WHITELIST]) manager4 = self.create_peer(network, capabilities=[settings.CAPABILITY_WHITELIST]) @@ -34,10 +38,46 @@ def test_capabilities(self): self.assertEqual(conn2._proto1.state.state_name, 'READY') self.assertEqual(conn2._proto2.state.state_name, 'READY') + self.assertIsInstance(conn2._proto1.state.sync_manager, NodeSyncTimestamp) + self.assertIsInstance(conn2._proto2.state.sync_manager, NodeSyncTimestamp) class SyncV2HathorCapabilitiesTestCase(unittest.SyncV2Params, unittest.TestCase): - __test__ = True + def test_capabilities(self): + network = 'testnet' + manager1 = self.create_peer(network, capabilities=[settings.CAPABILITY_WHITELIST, + settings.CAPABILITY_SYNC_VERSION]) + manager2 = self.create_peer(network, capabilities=[settings.CAPABILITY_SYNC_VERSION]) + + conn = FakeConnection(manager1, manager2) + + # Run the p2p protocol. + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Even if we don't have the capability we must connect because the whitelist url conf is None + self.assertEqual(conn._proto1.state.state_name, 'READY') + self.assertEqual(conn._proto2.state.state_name, 'READY') + self.assertIsInstance(conn._proto1.state.sync_manager, NodeBlockSync) + self.assertIsInstance(conn._proto2.state.sync_manager, NodeBlockSync) + + manager3 = self.create_peer(network, capabilities=[settings.CAPABILITY_WHITELIST, + settings.CAPABILITY_SYNC_VERSION]) + manager4 = self.create_peer(network, capabilities=[settings.CAPABILITY_WHITELIST, + settings.CAPABILITY_SYNC_VERSION]) + + conn2 = FakeConnection(manager3, manager4) + + # Run the p2p protocol. + for _ in range(100): + conn2.run_one_step(debug=True) + self.clock.advance(0.1) + + self.assertEqual(conn2._proto1.state.state_name, 'READY') + self.assertEqual(conn2._proto2.state.state_name, 'READY') + self.assertIsInstance(conn2._proto1.state.sync_manager, NodeBlockSync) + self.assertIsInstance(conn2._proto2.state.sync_manager, NodeBlockSync) # sync-bridge should behave like sync-v2 diff --git a/tests/p2p/test_protocol.py b/tests/p2p/test_protocol.py index 990056f60..268005ad0 100644 --- a/tests/p2p/test_protocol.py +++ b/tests/p2p/test_protocol.py @@ -1,4 +1,5 @@ from json import JSONDecodeError +from typing import Optional from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure @@ -25,6 +26,14 @@ def setUp(self): self.manager2 = self.create_peer(self.network, peer_id=self.peer_id2) self.conn = FakeConnection(self.manager1, self.manager2) + def assertAndStepConn(self, conn: FakeConnection, regex1: bytes, regex2: Optional[bytes] = None) -> None: + """If only one regex is given it is tested on both cons, if two are given they'll be used respectively.""" + if regex2 is None: + regex2 = regex1 + self.assertRegex(conn.peek_tr1_value(), regex1) + self.assertRegex(conn.peek_tr2_value(), regex2) + conn.run_one_step() + def assertIsConnected(self, conn=None): if conn is None: conn = self.conn @@ -157,20 +166,6 @@ def test_valid_hello(self): self.assertFalse(self.conn.tr1.disconnecting) self.assertFalse(self.conn.tr2.disconnecting) - @inlineCallbacks - def test_invalid_peer_id(self): - self.conn.run_one_step() # HELLO - self.conn.run_one_step() # PEER-ID - self.conn.run_one_step() # READY - self.conn.run_one_step() # GET-PEERS - self.conn.run_one_step() # GET-TIPS - self.conn.run_one_step() # PEERS - self.conn.run_one_step() # TIPS - invalid_payload = {'id': '123', 'entrypoints': ['tcp://localhost:1234']} - yield self._send_cmd(self.conn.proto1, 'PEER-ID', json_dumps(invalid_payload)) - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'ERROR') - self.assertTrue(self.conn.tr1.disconnecting) - def test_invalid_same_peer_id(self): manager3 = self.create_peer(self.network, peer_id=self.peer_id1) conn = FakeConnection(self.manager1, manager3) @@ -215,17 +210,16 @@ def test_invalid_same_peer_id2(self): self.conn.run_one_step() conn.run_one_step() # continue until messages stop - self.conn.run_until_complete() - conn.run_until_complete() + self.conn.run_until_empty() + conn.run_until_empty() self.run_to_completion() - # one of the peers will close the connection. We don't know which on, as it depends + # one of the peers will close the connection. We don't know which one, as it depends # on the peer ids - conn1_value = self.conn.peek_tr1_value() + self.conn.peek_tr2_value() - conn2_value = conn.peek_tr1_value() + conn.peek_tr2_value() - if b'ERROR' in conn1_value: + + if self.conn.tr1.disconnecting or self.conn.tr2.disconnecting: conn_dead = self.conn conn_alive = conn - elif b'ERROR' in conn2_value: + elif conn.tr1.disconnecting or conn.tr2.disconnecting: conn_dead = conn conn_alive = self.conn else: @@ -248,51 +242,6 @@ def test_invalid_different_network(self): self.assertTrue(conn.tr1.disconnecting) conn.run_one_step() # ERROR - def test_valid_hello_and_peer_id(self): - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'HELLO') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'HELLO') - self.conn.run_one_step() # HELLO - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'PEER-ID') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'PEER-ID') - self.conn.run_one_step() # PEER-ID - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'READY') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'READY') - self.conn.run_one_step() # READY - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'GET-PEERS') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'GET-PEERS') - self.conn.run_one_step() # GET-PEERS - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'GET-TIPS') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'GET-TIPS') - self.conn.run_one_step() # GET-TIPS - self.assertIsConnected() - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'PEERS') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'PEERS') - self.conn.run_one_step() # PEERS - self._check_result_only_cmd(self.conn.peek_tr1_value(), b'TIPS') - self._check_result_only_cmd(self.conn.peek_tr2_value(), b'TIPS') - self.conn.run_one_step() # TIPS - self.assertIsConnected() - - def test_send_ping(self): - self.conn.run_one_step() # HELLO - self.conn.run_one_step() # PEER-ID - self.conn.run_one_step() # READY - self.conn.run_one_step() # GET-PEERS - self.conn.run_one_step() # GET-TIPS - self.conn.run_one_step() # PEERS - self.conn.run_one_step() # TIPS - self.assertIsConnected() - self.clock.advance(5) - self.assertEqual(b'PING\r\n', self.conn.peek_tr1_value()) - self.assertEqual(b'PING\r\n', self.conn.peek_tr2_value()) - self.conn.run_one_step() # PING - self.conn.run_one_step() # GET-TIPS - self.assertEqual(b'PONG\r\n', self.conn.peek_tr1_value()) - self.assertEqual(b'PONG\r\n', self.conn.peek_tr2_value()) - while b'PONG\r\n' in self.conn.peek_tr1_value(): - self.conn.run_one_step() - self.assertEqual(self.clock.seconds(), self.conn.proto1.last_message) - def test_send_invalid_unicode(self): # \xff is an invalid unicode. self.conn.proto1.dataReceived(b'\xff\r\n') @@ -330,6 +279,16 @@ def test_on_disconnect_after_peer_id(self): # Peer id 2 removed from peer_storage (known_peers) after disconnection and after looping call self.assertNotIn(self.peer_id2.id, self.manager1.connections.peer_storage) + def test_idle_connection(self): + self.clock.advance(settings.PEER_IDLE_TIMEOUT - 10) + self.assertIsConnected(self.conn) + self.clock.advance(15) + self.assertIsNotConnected(self.conn) + + +class SyncV1HathorProtocolTestCase(unittest.SyncV1Params, BaseHathorProtocolTestCase): + __test__ = True + def test_two_connections(self): self.conn.run_one_step() # HELLO self.conn.run_one_step() # PEER-ID @@ -347,12 +306,6 @@ def test_two_connections(self): self._check_result_only_cmd(self.conn.peek_tr1_value(), b'PEERS') self.conn.run_one_step() - def test_idle_connection(self): - self.clock.advance(settings.PEER_IDLE_TIMEOUT - 10) - self.assertIsConnected(self.conn) - self.clock.advance(15) - self.assertIsNotConnected(self.conn) - @inlineCallbacks def test_get_data(self): self.conn.run_one_step() # HELLO @@ -368,14 +321,170 @@ def test_get_data(self): self._check_result_only_cmd(self.conn.peek_tr1_value(), b'NOT-FOUND') self.conn.run_one_step() + def test_valid_hello_and_peer_id(self): + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'HELLO') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'HELLO') + self.conn.run_one_step() # HELLO + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'PEER-ID') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'PEER-ID') + self.conn.run_one_step() # PEER-ID + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'READY') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'READY') + self.conn.run_one_step() # READY + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'GET-PEERS') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'GET-PEERS') + self.conn.run_one_step() # GET-PEERS + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'GET-TIPS') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'GET-TIPS') + self.conn.run_one_step() # GET-TIPS + self.assertIsConnected() + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'PEERS') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'PEERS') + self.conn.run_one_step() # PEERS + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'TIPS') + self._check_result_only_cmd(self.conn.peek_tr2_value(), b'TIPS') + self.conn.run_one_step() # TIPS + self.assertIsConnected() -class SyncV1HathorProtocolTestCase(unittest.SyncV1Params, BaseHathorProtocolTestCase): - __test__ = True + def test_send_ping(self): + self.conn.run_one_step() # HELLO + self.conn.run_one_step() # PEER-ID + self.conn.run_one_step() # READY + self.conn.run_one_step() # GET-PEERS + self.conn.run_one_step() # GET-TIPS + self.conn.run_one_step() # PEERS + self.conn.run_one_step() # TIPS + self.assertIsConnected() + self.clock.advance(5) + self.assertEqual(b'PING\r\n', self.conn.peek_tr1_value()) + self.assertEqual(b'PING\r\n', self.conn.peek_tr2_value()) + self.conn.run_one_step() # PING + self.conn.run_one_step() # GET-TIPS + self.assertEqual(b'PONG\r\n', self.conn.peek_tr1_value()) + self.assertEqual(b'PONG\r\n', self.conn.peek_tr2_value()) + while b'PONG\r\n' in self.conn.peek_tr1_value(): + self.conn.run_one_step() + self.assertEqual(self.clock.seconds(), self.conn.proto1.last_message) + + @inlineCallbacks + def test_invalid_peer_id(self): + self.conn.run_one_step() # HELLO + self.conn.run_one_step() # PEER-ID + self.conn.run_one_step() # READY + self.conn.run_one_step() # GET-PEERS + self.conn.run_one_step() # GET-TIPS + self.conn.run_one_step() # PEERS + self.conn.run_one_step() # TIPS + invalid_payload = {'id': '123', 'entrypoints': ['tcp://localhost:1234']} + yield self._send_cmd(self.conn.proto1, 'PEER-ID', json_dumps(invalid_payload)) + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'ERROR') + self.assertTrue(self.conn.tr1.disconnecting) class SyncV2HathorProtocolTestCase(unittest.SyncV2Params, BaseHathorProtocolTestCase): __test__ = True + def test_two_connections(self): + self.assertAndStepConn(self.conn, b'^HELLO') + self.assertAndStepConn(self.conn, b'^PEER-ID') + self.assertAndStepConn(self.conn, b'^READY') + self.assertAndStepConn(self.conn, b'^GET-PEERS') + self.assertAndStepConn(self.conn, b'^GET-BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^PEERS') + self.assertAndStepConn(self.conn, b'^BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^RELAY') + self.assertIsConnected() + + # disable timeout because we will make several steps on a new conn and this might get left behind + self.conn.disable_idle_timeout() + + manager3 = self.create_peer(self.network, enable_sync_v2=True) + conn = FakeConnection(self.manager1, manager3) + self.assertAndStepConn(conn, b'^HELLO') + self.assertAndStepConn(conn, b'^PEER-ID') + self.assertAndStepConn(conn, b'^READY') + self.assertAndStepConn(conn, b'^GET-PEERS') + + self.clock.advance(5) + self.assertIsConnected() + self.assertAndStepConn(self.conn, b'^GET-TIPS') + self.assertAndStepConn(self.conn, b'^PING') + # peer1 should now send a PEERS with the new peer that just connected + self.assertAndStepConn(self.conn, b'^PEERS', b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS', b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS', b'^TIPS-END') + self.assertAndStepConn(self.conn, b'^TIPS-END', b'^PONG') + self.assertAndStepConn(self.conn, b'^PONG', b'^$') + self.assertIsConnected() + + @inlineCallbacks + def test_get_data(self): + self.assertAndStepConn(self.conn, b'^HELLO') + self.assertAndStepConn(self.conn, b'^PEER-ID') + self.assertAndStepConn(self.conn, b'^READY') + self.assertAndStepConn(self.conn, b'^GET-PEERS') + self.assertAndStepConn(self.conn, b'^GET-BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^PEERS') + self.assertAndStepConn(self.conn, b'^BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^RELAY') + self.assertIsConnected() + missing_tx = '00000000228dfcd5dec1c9c6263f6430a5b4316bb9e3decb9441a6414bfd8697' + payload = {'child': missing_tx, 'last_block': settings.GENESIS_BLOCK_HASH.hex()} + yield self._send_cmd(self.conn.proto1, 'GET-BLOCK-TXS', json_dumps(payload)) + self._check_result_only_cmd(self.conn.peek_tr1_value(), b'NOT-FOUND') + self.conn.run_one_step() + + def test_valid_hello_and_peer_id(self): + self.assertAndStepConn(self.conn, b'^HELLO') + self.assertAndStepConn(self.conn, b'^PEER-ID') + self.assertAndStepConn(self.conn, b'^READY') + self.assertAndStepConn(self.conn, b'^GET-PEERS') + self.assertAndStepConn(self.conn, b'^GET-BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^PEERS') + self.assertAndStepConn(self.conn, b'^BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^RELAY') + + # this will tick the ping-pong mechanism and looping calls + self.clock.advance(5) + self.assertIsConnected() + self.assertAndStepConn(self.conn, b'^GET-TIPS') + self.assertAndStepConn(self.conn, b'^PING') + self.assertAndStepConn(self.conn, b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS-END') + self.assertAndStepConn(self.conn, b'^PONG') + self.assertIsConnected() + + self.clock.advance(5) + self.assertAndStepConn(self.conn, b'^PING') + self.assertAndStepConn(self.conn, b'^GET-BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^PONG') + self.assertAndStepConn(self.conn, b'^BEST-BLOCK') + self.assertIsConnected() + + def test_send_ping(self): + self.assertAndStepConn(self.conn, b'^HELLO') + self.assertAndStepConn(self.conn, b'^PEER-ID') + self.assertAndStepConn(self.conn, b'^READY') + self.assertAndStepConn(self.conn, b'^GET-PEERS') + self.assertAndStepConn(self.conn, b'^GET-BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^PEERS') + self.assertAndStepConn(self.conn, b'^BEST-BLOCK') + self.assertAndStepConn(self.conn, b'^RELAY') + + # this will tick the ping-pong mechanism and looping calls + self.clock.advance(5) + self.assertAndStepConn(self.conn, b'^GET-TIPS') + self.assertAndStepConn(self.conn, b'^PING') + self.assertAndStepConn(self.conn, b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS') + self.assertAndStepConn(self.conn, b'^TIPS-END') + self.assertEqual(b'PONG\r\n', self.conn.peek_tr1_value()) + self.assertEqual(b'PONG\r\n', self.conn.peek_tr2_value()) + while b'PONG\r\n' in self.conn.peek_tr1_value(): + self.conn.run_one_step() + self.assertEqual(self.clock.seconds(), self.conn.proto1.last_message) + # sync-bridge should behave like sync-v2 class SyncBridgeHathorProtocolTestCase(unittest.SyncBridgeParams, SyncV2HathorProtocolTestCase): diff --git a/tests/p2p/test_split_brain.py b/tests/p2p/test_split_brain.py index 547f845ee..fc5a8fadc 100644 --- a/tests/p2p/test_split_brain.py +++ b/tests/p2p/test_split_brain.py @@ -11,7 +11,7 @@ from tests.utils import add_blocks_unlock_reward, add_new_block, add_new_double_spending, add_new_transactions -class BaseHathorSplitBrainTestCase(unittest.TestCase): +class BaseHathorSyncMethodsTestCase(unittest.TestCase): __test__ = False def setUp(self): @@ -45,7 +45,7 @@ def create_peer(self, network, unlock_wallet=True): return manager @pytest.mark.slow - def test_split_brain(self): + def test_split_brain_plain(self): debug_pdf = False manager1 = self.create_peer(self.network, unlock_wallet=True) @@ -80,23 +80,17 @@ def test_split_brain(self): conn = FakeConnection(manager1, manager2) - conn.run_one_step() # HELLO - conn.run_one_step() # PEER-ID - conn.run_one_step() # READY - conn.run_one_step() # GET-PEERS - conn.run_one_step() # GET-TIPS - conn.run_one_step() # PEERS - conn.run_one_step() # TIPS - - empty_counter = 0 - for i in range(2000): - if conn.is_empty(): - empty_counter += 1 - if empty_counter > 10: - break - else: - empty_counter = 0 - + # upper limit to how many steps it definitely should be enough + for i in range(3000): + if not conn.can_step(): + break + conn.run_one_step() + self.clock.advance(0.2) + else: + # error if we fall off the loop without breaking + self.fail('took more steps than expected') + self.log.debug('steps', count=i) + for i in range(500): conn.run_one_step() self.clock.advance(0.2) @@ -107,21 +101,316 @@ def test_split_brain(self): dot2.render('dot2-post') node_sync = conn.proto1.state.sync_manager - self.assertEqual(node_sync.synced_timestamp, node_sync.peer_timestamp) + self.assertSynced(node_sync) self.assertTipsEqual(manager1, manager2) self.assertConsensusEqual(manager1, manager2) self.assertConsensusValid(manager1) self.assertConsensusValid(manager2) + @pytest.mark.slow + def test_split_brain_only_blocks_different_height(self): + manager1 = self.create_peer(self.network, unlock_wallet=True) + manager1.avg_time_between_blocks = 3 + + manager2 = self.create_peer(self.network, unlock_wallet=True) + manager2.avg_time_between_blocks = 3 + + for _ in range(10): + add_new_block(manager1, advance_clock=1) + add_blocks_unlock_reward(manager1) + add_new_block(manager2, advance_clock=1) + add_blocks_unlock_reward(manager2) + self.clock.advance(10) + + # Add one more block to manager1, so it's the winner chain + add_new_block(manager1, advance_clock=1) + + block_tip1 = manager1.tx_storage.indexes.height.get_tip() + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + conn = FakeConnection(manager1, manager2) + + empty_counter = 0 + for i in range(1000): + if conn.is_empty(): + empty_counter += 1 + if empty_counter > 10: + break + else: + empty_counter = 0 + + conn.run_one_step() + self.clock.advance(1) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + self.assertConsensusEqual(manager1, manager2) + + self.assertEqual(block_tip1, manager1.tx_storage.indexes.height.get_tip()) + self.assertEqual(block_tip1, manager2.tx_storage.indexes.height.get_tip()) + + # XXX We must decide what to do when different chains have the same score + # For now we are voiding everyone until the first common block + def test_split_brain_only_blocks_same_height(self): + manager1 = self.create_peer(self.network, unlock_wallet=True) + manager1.avg_time_between_blocks = 3 + + manager2 = self.create_peer(self.network, unlock_wallet=True) + manager2.avg_time_between_blocks = 3 + + for _ in range(10): + add_new_block(manager1, advance_clock=1) + unlock_reward_blocks1 = add_blocks_unlock_reward(manager1) + add_new_block(manager2, advance_clock=1) + unlock_reward_blocks2 = add_blocks_unlock_reward(manager2) + self.clock.advance(10) + + block_tips1 = unlock_reward_blocks1[-1].hash + block_tips2 = unlock_reward_blocks2[-1].hash + + self.assertEqual(len(manager1.tx_storage.get_best_block_tips()), 1) + self.assertCountEqual(manager1.tx_storage.get_best_block_tips(), {block_tips1}) + self.assertEqual(len(manager2.tx_storage.get_best_block_tips()), 1) + self.assertCountEqual(manager2.tx_storage.get_best_block_tips(), {block_tips2}) + + # Save winners for manager1 and manager2 + winners1 = set() + for tx1 in manager1.tx_storage.get_all_transactions(): + tx1_meta = tx1.get_metadata() + if not tx1_meta.voided_by: + winners1.add(tx1.hash) + + winners2 = set() + for tx2 in manager2.tx_storage.get_all_transactions(): + tx2_meta = tx2.get_metadata() + if not tx2_meta.voided_by: + winners2.add(tx2.hash) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + conn = FakeConnection(manager1, manager2) + + empty_counter = 0 + for i in range(1000): + if conn.is_empty(): + empty_counter += 1 + if empty_counter > 10: + break + else: + empty_counter = 0 + + conn.run_one_step() + self.clock.advance(1) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + # self.assertEqual(len(manager1.tx_storage.get_best_block_tips()), 2) + # self.assertCountEqual(manager1.tx_storage.get_best_block_tips(), {block_tips1, block_tips2}) + # self.assertEqual(len(manager2.tx_storage.get_best_block_tips()), 2) + # self.assertCountEqual(manager2.tx_storage.get_best_block_tips(), {block_tips1, block_tips2}) + + winners1_after = set() + for tx1 in manager1.tx_storage.get_all_transactions(): + tx1_meta = tx1.get_metadata() + if not tx1_meta.voided_by: + winners1_after.add(tx1.hash) + + winners2_after = set() + for tx2 in manager2.tx_storage.get_all_transactions(): + tx2_meta = tx2.get_metadata() + if not tx2_meta.voided_by: + winners2_after.add(tx2.hash) + + # Both chains have the same height and score + # so they will void all blocks and keep only the genesis (the common block and txs) + self.assertEqual(len(winners1_after), 3) + self.assertEqual(len(winners2_after), 3) + + new_block = add_new_block(manager1, advance_clock=1) + self.clock.advance(20) + + empty_counter = 0 + for i in range(500): + if conn.is_empty(): + empty_counter += 1 + if empty_counter > 10: + break + else: + empty_counter = 0 + + conn.run_one_step() + self.clock.advance(1) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + winners1_after = set() + for tx1 in manager1.tx_storage.get_all_transactions(): + tx1_meta = tx1.get_metadata() + if not tx1_meta.voided_by: + winners1_after.add(tx1.hash) + + winners2_after = set() + for tx2 in manager2.tx_storage.get_all_transactions(): + tx2_meta = tx2.get_metadata() + if not tx2_meta.voided_by: + winners2_after.add(tx2.hash) + + winners1.add(new_block.hash) + winners2.add(new_block.hash) + + if new_block.get_block_parent().hash == block_tips1: + winners = winners1 + else: + winners = winners2 + + self.assertCountEqual(winners, winners1_after) + self.assertCountEqual(winners, winners2_after) + + self.assertEqual(len(manager1.tx_storage.get_best_block_tips()), 1) + self.assertCountEqual(manager1.tx_storage.get_best_block_tips(), {new_block.hash}) + self.assertEqual(len(manager2.tx_storage.get_best_block_tips()), 1) + self.assertCountEqual(manager2.tx_storage.get_best_block_tips(), {new_block.hash}) + + def test_split_brain_only_blocks_bigger_score(self): + manager1 = self.create_peer(self.network, unlock_wallet=True) + manager1.avg_time_between_blocks = 3 + + manager2 = self.create_peer(self.network, unlock_wallet=True) + manager2.avg_time_between_blocks = 3 + + # Start with 1 because of the genesis block + manager2_blocks = 1 + for _ in range(10): + add_new_block(manager1, advance_clock=1) + add_blocks_unlock_reward(manager1) + add_new_block(manager2, advance_clock=1) + manager2_blocks += 1 + blocks2 = add_blocks_unlock_reward(manager2) + manager2_blocks += len(blocks2) + self.clock.advance(10) + + # Add two more blocks to manager1, so it's the winner chain + add_new_block(manager1, advance_clock=1) + add_new_block(manager1, advance_clock=1) + + # Propagates a block with bigger weight, so the score of the manager2 chain + # will be bigger than the other one + b = add_new_block(manager2, advance_clock=1, propagate=False) + b.weight = 5 + b.resolve() + manager2.propagate_tx(b) + manager2_blocks += 1 + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + conn = FakeConnection(manager1, manager2) + + empty_counter = 0 + for i in range(1000): + if conn.is_empty(): + empty_counter += 1 + if empty_counter > 10: + break + else: + empty_counter = 0 + + conn.run_one_step() + self.clock.advance(1) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + self.assertConsensusEqual(manager1, manager2) + + winners2_blocks = 0 + for tx2 in manager2.tx_storage.get_all_transactions(): + tx2_meta = tx2.get_metadata() + if tx2.is_block and not tx2_meta.voided_by: + winners2_blocks += 1 -class SyncV1HathorSplitBrainTestCase(unittest.SyncV1Params, BaseHathorSplitBrainTestCase): + # Assert that the consensus had the manager2 chain + self.assertEqual(winners2_blocks, manager2_blocks) + + +class SyncV1HathorSyncMethodsTestCase(unittest.SyncV1Params, BaseHathorSyncMethodsTestCase): __test__ = True -class SyncV2HathorSplitBrainTestCase(unittest.SyncV2Params, BaseHathorSplitBrainTestCase): +class SyncV2HathorSyncMethodsTestCase(unittest.SyncV2Params, BaseHathorSyncMethodsTestCase): __test__ = True + # XXX: should test_split_brain be ported to sync-v2? + + def test_split_brain_no_double_spending(self): + manager1 = self.create_peer(self.network, unlock_wallet=True) + manager1.avg_time_between_blocks = 3 + + manager2 = self.create_peer(self.network, unlock_wallet=True) + manager2.avg_time_between_blocks = 3 + + winner_blocks = 1 + winner_txs = 2 + + for _ in range(10): + add_new_block(manager1, advance_clock=1) + add_blocks_unlock_reward(manager1) + add_new_block(manager2, advance_clock=1) + winner_blocks += 1 + blocks = add_blocks_unlock_reward(manager2) + winner_blocks += len(blocks) + self.clock.advance(10) + for _ in range(random.randint(3, 10)): + add_new_transactions(manager1, random.randint(2, 4), advance_clock=1) + txs = add_new_transactions(manager2, random.randint(3, 7), advance_clock=1) + winner_txs += len(txs) + self.clock.advance(10) + + self.clock.advance(20) + + # Manager2 will be the winner because it has the biggest chain + add_new_block(manager2, advance_clock=1) + winner_blocks += 1 + self.clock.advance(20) + + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + conn = FakeConnection(manager1, manager2) + # Disable idle timeout. + conn.disable_idle_timeout() + + self.log.info('starting sync now...') + + # upper limit to how many steps it definitely should be enough + for i in range(1000): + if not conn.can_step(): + break + conn.run_one_step() + self.clock.advance(1) + else: + # error if we fall off the loop without breaking + self.fail('took more steps than expected') + + self.log.debug('steps taken', steps=i + 1) + + self.assertConsensusEqual(manager1, manager2) + self.assertConsensusValid(manager1) + self.assertConsensusValid(manager2) + + winners2 = set() + for tx in manager2.tx_storage.get_all_transactions(): + tx_meta = tx.get_metadata() + if not tx_meta.voided_by: + winners2.add(tx.hash) + + self.assertEqual(len(winners2), winner_blocks + winner_txs) + # sync-bridge should behave like sync-v2 -class SyncBridgeHathorSplitBrainTestCase(unittest.SyncBridgeParams, SyncV2HathorSplitBrainTestCase): +class SyncBridgeHathorSyncMethodsTestCase(unittest.SyncBridgeParams, SyncV2HathorSyncMethodsTestCase): pass diff --git a/tests/p2p/test_split_brain2.py b/tests/p2p/test_split_brain2.py index 9fad42242..74bef12ca 100644 --- a/tests/p2p/test_split_brain2.py +++ b/tests/p2p/test_split_brain2.py @@ -67,10 +67,8 @@ def test_split_brain(self): dot2 = GraphvizVisualizer(manager2.tx_storage, include_verifications=True).dot() dot2.render('dot2-post') - node_sync = conn12.proto1.state.sync_manager - self.assertEqual(node_sync.synced_timestamp, node_sync.peer_timestamp) - node_sync = conn12.proto2.state.sync_manager - self.assertEqual(node_sync.synced_timestamp, node_sync.peer_timestamp) + self.assertSynced(conn12.proto1.state.sync_manager) + self.assertSynced(conn12.proto2.state.sync_manager) self.assertTipsEqual(manager1, manager2) self.assertConsensusEqual(manager1, manager2) self.assertConsensusValid(manager1) diff --git a/tests/p2p/test_sync.py b/tests/p2p/test_sync.py index 9447980df..b9b04a5b4 100644 --- a/tests/p2p/test_sync.py +++ b/tests/p2p/test_sync.py @@ -2,6 +2,7 @@ from twisted.python.failure import Failure +from hathor.checkpoint import Checkpoint as cp from hathor.conf import HathorSettings from hathor.crypto.util import decode_address from hathor.p2p.protocol import PeerIdState @@ -9,6 +10,7 @@ from hathor.simulator import FakeConnection from hathor.transaction.storage.exceptions import TransactionIsNotABlock from tests import unittest +from tests.utils import add_blocks_unlock_reward settings = HathorSettings() @@ -76,7 +78,6 @@ def test_get_blocks_before(self): genesis_block = self.genesis_blocks[0] result = self.manager1.tx_storage.get_blocks_before(genesis_block.hash) self.assertEqual(0, len(result)) - genesis_tx = [tx for tx in self.genesis if not tx.is_block][0] with self.assertRaises(TransactionIsNotABlock): self.manager1.tx_storage.get_blocks_before(genesis_tx.hash) @@ -217,8 +218,9 @@ def test_tx_propagation_nat_peers(self): self._add_new_transactions(1) - for _ in range(1000): - if self.conn1.is_empty() and self.conn2.is_empty(): + for i in range(1000): + # XXX: give it at least 100 steps before checking for emptyness + if i > 100 and self.conn1.is_empty() and self.conn2.is_empty(): break self.conn1.run_one_step() self.conn2.run_one_step() @@ -477,6 +479,271 @@ def test_downloader_disconnect(self): class SyncV2HathorSyncMethodsTestCase(unittest.SyncV2Params, BaseHathorSyncMethodsTestCase): __test__ = True + def test_sync_metadata(self): + # test if the synced peer will build all tx metadata correctly + + height = 0 + # add a mix of blocks and transactions + height += len(self._add_new_blocks(8)) + height += len(add_blocks_unlock_reward(self.manager1)) + self._add_new_transactions(2) + height += len(self._add_new_blocks(1)) + self._add_new_transactions(4) + height += len(self._add_new_blocks(2)) + self._add_new_transactions(2) + + manager2 = self.create_peer(self.network) + self.assertEqual(manager2.state, manager2.NodeState.READY) + conn = FakeConnection(self.manager1, manager2) + + for _ in range(100): + if conn.is_empty(): + break + conn.run_one_step(debug=True) + self.clock.advance(1) + + # check they have the same consensus + node_sync1 = conn.proto1.state.sync_manager + node_sync2 = conn.proto2.state.sync_manager + self.assertEqual(node_sync1.peer_height, height) + self.assertEqual(node_sync1.synced_height, height) + self.assertEqual(node_sync2.peer_height, height) + # 3 genesis + blocks + 8 txs + self.assertEqual(self.manager1.tx_storage.get_vertices_count(), height + 11) + self.assertEqual(manager2.tx_storage.get_vertices_count(), height + 11) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + self.assertConsensusEqual(self.manager1, manager2) + + # Nodes are synced. Make sure manager2 has the correct metadata. + for tx in self.manager1.tx_storage.topological_iterator(): + meta1 = tx.get_metadata() + meta2 = manager2.tx_storage.get_metadata(tx.hash) + self.assertCountEqual(meta1.children or [], meta2.children or []) + self.assertCountEqual(meta1.voided_by or [], meta2.voided_by or []) + self.assertCountEqual(meta1.conflict_with or [], meta2.conflict_with or []) + self.assertCountEqual(meta1.twins or [], meta2.twins or []) + + def test_tx_propagation_nat_peers(self): + super().test_tx_propagation_nat_peers() + + node_sync1 = self.conn1.proto1.state.sync_manager + self.assertEqual(self.manager1.tx_storage.latest_timestamp, self.manager2.tx_storage.latest_timestamp) + self.assertEqual(node_sync1.peer_height, node_sync1.synced_height) + self.assertEqual(node_sync1.peer_height, self.manager1.tx_storage.get_height_best_block()) + + node_sync2 = self.conn2.proto1.state.sync_manager + self.assertEqual(self.manager2.tx_storage.latest_timestamp, self.manager3.tx_storage.latest_timestamp) + self.assertEqual(node_sync2.peer_height, node_sync2.synced_height) + self.assertEqual(node_sync2.peer_height, self.manager2.tx_storage.get_height_best_block()) + + def test_block_sync_new_blocks_and_txs(self): + self._add_new_blocks(25) + self._add_new_transactions(3) + self._add_new_blocks(4) + self._add_new_transactions(5) + + manager2 = self.create_peer(self.network) + self.assertEqual(manager2.state, manager2.NodeState.READY) + + conn = FakeConnection(self.manager1, manager2) + + for _ in range(1000): + conn.run_one_step() + self.clock.advance(0.1) + + # dot1 = self.manager1.tx_storage.graphviz(format='pdf') + # dot1.render('dot1') + + # dot2 = manager2.tx_storage.graphviz(format='pdf') + # dot2.render('dot2') + + node_sync = conn.proto1.state.sync_manager + self.assertEqual(self.manager1.tx_storage.latest_timestamp, manager2.tx_storage.latest_timestamp) + self.assertEqual(node_sync.peer_height, node_sync.synced_height) + self.assertEqual(node_sync.peer_height, self.manager1.tx_storage.get_height_best_block()) + self.assertConsensusEqual(self.manager1, manager2) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + + def test_block_sync_many_new_blocks(self): + self._add_new_blocks(150) + + manager2 = self.create_peer(self.network) + self.assertEqual(manager2.state, manager2.NodeState.READY) + + conn = FakeConnection(self.manager1, manager2) + + for _ in range(1000): + if conn.is_empty(): + break + conn.run_one_step(debug=True) + self.clock.advance(1) + + node_sync = conn.proto1.state.sync_manager + self.assertEqual(node_sync.peer_height, node_sync.synced_height) + self.assertEqual(node_sync.peer_height, self.manager1.tx_storage.get_height_best_block()) + self.assertConsensusEqual(self.manager1, manager2) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + + def test_block_sync_new_blocks(self): + self._add_new_blocks(15) + + manager2 = self.create_peer(self.network) + self.assertEqual(manager2.state, manager2.NodeState.READY) + + conn = FakeConnection(self.manager1, manager2) + + for _ in range(1000): + if conn.is_empty(): + break + conn.run_one_step(debug=True) + self.clock.advance(1) + + node_sync = conn.proto1.state.sync_manager + self.assertEqual(node_sync.peer_height, node_sync.synced_height) + self.assertEqual(node_sync.peer_height, self.manager1.tx_storage.get_height_best_block()) + self.assertConsensusEqual(self.manager1, manager2) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + + def test_full_sync(self): + # 10 blocks + blocks = self._add_new_blocks(10) + # N blocks to unlock the reward + unlock_reward_blocks = add_blocks_unlock_reward(self.manager1) + len_reward_unlock = len(unlock_reward_blocks) + # 3 transactions still before the last checkpoint + self._add_new_transactions(3) + # 5 more blocks and the last one is the last checkpoint + new_blocks = self._add_new_blocks(5) + + LAST_CHECKPOINT = len(blocks) + len_reward_unlock + len(new_blocks) + FIRST_CHECKPOINT = LAST_CHECKPOINT // 2 + cps = [ + cp(0, self.genesis_blocks[0].hash), + cp(FIRST_CHECKPOINT, (blocks + unlock_reward_blocks + new_blocks)[FIRST_CHECKPOINT - 1].hash), + cp(LAST_CHECKPOINT, (blocks + unlock_reward_blocks + new_blocks)[LAST_CHECKPOINT - 1].hash) + ] + + # 5 blocks after the last checkpoint + self._add_new_blocks(5) + # 3 transactions + self._add_new_transactions(3) + # 5 more blocks + self._add_new_blocks(5) + + # Add transactions to the mempool + self._add_new_transactions(2) + + self.manager1.checkpoints = cps + + manager2 = self.create_peer(self.network) + manager2.checkpoints = cps + self.assertEqual(manager2.state, manager2.NodeState.READY) + + total_count = 36 + len_reward_unlock + + self.assertEqual(self.manager1.tx_storage.get_vertices_count(), total_count) + self.assertEqual(manager2.tx_storage.get_vertices_count(), 3) + + conn = FakeConnection(self.manager1, manager2) + for i in range(300): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + conn.run_until_empty(1000) + + # node_sync = conn.proto1.state.sync_manager + # self.assertEqual(node_sync.synced_timestamp, node_sync.peer_timestamp) + # self.assertTipsEqual(self.manager1, manager2) + common_height = 25 + len_reward_unlock + + self.assertEqual(self.manager1.tx_storage.get_height_best_block(), common_height) + self.assertEqual(manager2.tx_storage.get_height_best_block(), common_height) + + node_sync1 = conn.proto1.state.sync_manager + node_sync2 = conn.proto2.state.sync_manager + self.assertEqual(node_sync1.peer_height, common_height) + self.assertEqual(node_sync1.synced_height, common_height) + self.assertEqual(node_sync2.peer_height, common_height) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + self.assertConsensusEqual(self.manager1, manager2) + + # 3 genesis + # 25 blocks + # Unlock reward blocks + # 8 txs + self.assertEqual(self.manager1.tx_storage.get_vertices_count(), total_count) + self.assertEqual(manager2.tx_storage.get_vertices_count(), total_count) + self.assertEqual(len(manager2.tx_storage.indexes.mempool_tips.get()), 1) + self.assertEqual(len(self.manager1.tx_storage.indexes.mempool_tips.get()), 1) + + def test_block_sync_checkpoints(self): + TOTAL_BLOCKS = 30 + LAST_CHECKPOINT = 15 + FIRST_CHECKPOINT = LAST_CHECKPOINT // 2 + blocks = self._add_new_blocks(TOTAL_BLOCKS, propagate=False) + cps = [ + cp(0, self.genesis_blocks[0].hash), + cp(FIRST_CHECKPOINT, blocks[FIRST_CHECKPOINT - 1].hash), + cp(LAST_CHECKPOINT, blocks[LAST_CHECKPOINT - 1].hash) + ] + self.manager1.checkpoints = cps + + manager2 = self.create_peer(self.network) + manager2.checkpoints = cps + self.assertEqual(manager2.state, manager2.NodeState.READY) + + conn = FakeConnection(self.manager1, manager2) + + # initial connection setup + for _ in range(100): + conn.run_one_step(debug=False) + self.clock.advance(0.1) + + # find synced timestamp + self.clock.advance(5) + for _ in range(600): + conn.run_one_step(debug=False) + self.clock.advance(0.1) + + self.assertEqual(self.manager1.tx_storage.get_best_block().get_metadata().height, TOTAL_BLOCKS) + self.assertEqual(manager2.tx_storage.get_best_block().get_metadata().height, TOTAL_BLOCKS) + + node_sync1 = conn.proto1.state.sync_manager + node_sync2 = conn.proto2.state.sync_manager + + self.assertEqual(node_sync1.peer_height, TOTAL_BLOCKS) + self.assertEqual(node_sync1.synced_height, TOTAL_BLOCKS) + self.assertEqual(node_sync2.peer_height, len(blocks)) + self.assertConsensusValid(self.manager1) + self.assertConsensusValid(manager2) + + def test_block_sync_only_genesis(self): + manager2 = self.create_peer(self.network) + self.assertEqual(manager2.state, manager2.NodeState.READY) + + conn = FakeConnection(self.manager1, manager2) + + genesis_tx = [tx for tx in self.genesis if not tx.is_block][0] + with self.assertRaises(TransactionIsNotABlock): + self.manager1.tx_storage.get_blocks_before(genesis_tx.hash) + + for _ in range(100): + if conn.is_empty(): + break + conn.run_one_step(debug=True) + self.clock.advance(1) + + node_sync = conn.proto1.state.sync_manager + self.assertEqual(node_sync.synced_height, 0) + self.assertEqual(node_sync.peer_height, 0) + + self.assertEqual(self.manager1.tx_storage.get_vertices_count(), 3) + self.assertEqual(manager2.tx_storage.get_vertices_count(), 3) + # TODO: an equivalent test to test_downloader, could be something like test_checkpoint_sync diff --git a/tests/p2p/test_sync_bridge.py b/tests/p2p/test_sync_bridge.py new file mode 100644 index 000000000..cdf000627 --- /dev/null +++ b/tests/p2p/test_sync_bridge.py @@ -0,0 +1,82 @@ +from hathor.simulator import FakeConnection +from tests.simulation.base import SimulatorTestCase + + +class MixedSyncRandomSimulatorTestCase(SimulatorTestCase): + __test__ = True + + def test_the_three_transacting_miners(self): + manager1 = self.create_peer(enable_sync_v1=True, enable_sync_v2=False) + manager2 = self.create_peer(enable_sync_v1=True, enable_sync_v2=True) + manager3 = self.create_peer(enable_sync_v1=False, enable_sync_v2=True) + + managers = [manager1, manager2, manager3] + all_managers = managers + miners = [] + tx_gens = [] + + for manager in managers: + miner = self.simulator.create_miner(manager, hashpower=100e6) + miner.start() + miners.append(miner) + tx_gen = self.simulator.create_tx_generator(manager, rate=2 / 60., hashpower=1e6, ignore_no_funds=True) + tx_gen.start() + tx_gens.append(tx_gen) + + self.simulator.run(2000) + + self.simulator.add_connection(FakeConnection(manager1, manager2, latency=0.300)) + self.simulator.add_connection(FakeConnection(manager1, manager3, latency=0.300)) + self.simulator.add_connection(FakeConnection(manager2, manager3, latency=0.300)) + + for tx_gen in tx_gens: + tx_gen.stop() + for miner in miners: + miner.stop() + + self.simulator.run_until_complete(2000, 600) + + for idx, node in enumerate(all_managers): + self.log.debug(f'checking node {idx}') + self.assertConsensusValid(manager) + + for manager_a, manager_b in zip(all_managers[:-1], all_managers[1:]): + # sync-v2 consensus test is more lenient (if sync-v1 assert passes sync-v2 assert will pass too) + self.assertConsensusEqualSyncV2(manager_a, manager_b, strict_sync_v2_indexes=False) + + def test_bridge_with_late_v2(self): + manager1 = self.create_peer(enable_sync_v1=True, enable_sync_v2=False) + manager2 = self.create_peer(enable_sync_v1=True, enable_sync_v2=True) + manager3 = self.create_peer(enable_sync_v1=False, enable_sync_v2=True) + + managers = [manager1, manager2] + all_managers = [manager1, manager2, manager3] + miners = [] + tx_gens = [] + + for manager in managers: + miner = self.simulator.create_miner(manager, hashpower=100e6) + miner.start() + miners.append(miner) + tx_gen = self.simulator.create_tx_generator(manager, rate=2 / 60., hashpower=1e6, ignore_no_funds=True) + tx_gen.start() + tx_gens.append(tx_gen) + + self.simulator.add_connection(FakeConnection(manager1, manager2, latency=0.300)) + self.simulator.run(2000) + + for tx_gen in tx_gens: + tx_gen.stop() + for miner in miners: + miner.stop() + + self.simulator.add_connection(FakeConnection(manager2, manager3, latency=0.300)) + self.simulator.run_until_complete(2000, 600) + + for idx, node in enumerate(all_managers): + self.log.debug(f'checking node {idx}') + self.assertConsensusValid(manager) + + for manager_a, manager_b in zip(all_managers[:-1], all_managers[1:]): + # sync-v2 consensus test is more lenient (if sync-v1 assert passes sync-v2 assert will pass too) + self.assertConsensusEqualSyncV2(manager_a, manager_b, strict_sync_v2_indexes=False) diff --git a/tests/resources/p2p/test_healthcheck.py b/tests/resources/p2p/test_healthcheck.py index 521612897..90bf1e260 100644 --- a/tests/resources/p2p/test_healthcheck.py +++ b/tests/resources/p2p/test_healthcheck.py @@ -72,7 +72,7 @@ def test_get_ready(self): add_new_blocks(self.manager, 5) # This will make sure the peers are synced - while not self.conn1.is_empty(): + for _ in range(600): self.conn1.run_one_step(debug=True) self.clock.advance(0.1) diff --git a/tests/simulation/test_simulator.py b/tests/simulation/test_simulator.py index c8002b7f2..fefca39cf 100644 --- a/tests/simulation/test_simulator.py +++ b/tests/simulation/test_simulator.py @@ -27,6 +27,7 @@ def test_one_node(self): # FIXME: the setup above produces 0 new blocks and transactions # self.assertGreater(manager1.tx_storage.get_vertices_count(), 3) + @pytest.mark.flaky(max_runs=5, min_passes=1) def test_two_nodes(self): manager1 = self.create_peer() manager2 = self.create_peer() @@ -61,6 +62,7 @@ def test_two_nodes(self): self.assertTrue(conn12.is_connected) self.assertTipsEqual(manager1, manager2) + @pytest.mark.flaky(max_runs=5, min_passes=1) def test_many_miners_since_beginning(self): nodes = [] miners = [] @@ -132,7 +134,7 @@ def test_new_syncing_peer(self): for miner in miners: miner.stop() - self.simulator.run_until_complete(600) + self.simulator.run_until_complete(2000) for idx, node in enumerate(nodes): self.log.debug(f'checking node {idx}') diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 7f35923ff..7d463a5d0 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -96,6 +96,8 @@ def test_validation(self): self.assertEqual(block_from_chain.get_next_block_best_chain(), None) def test_checkpoint_validation(self): + from hathor.checkpoint import Checkpoint + from hathor.exception import InvalidNewTransaction from hathor.transaction.transaction_metadata import ValidationState # manually validate with sync_checkpoints=True @@ -111,6 +113,34 @@ def test_checkpoint_validation(self): self.assertTrue(self.manager.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False)) self.assertEqual(block2.get_metadata().validation, ValidationState.CHECKPOINT_FULL) + # initialize a new manager and try to add a checkpoint block before its parents + block2_height = block2.get_metadata().height + manager2 = self.create_peer('testnet', checkpoints=[Checkpoint(block2_height, block2.hash)]) + del block2._metadata + block2.storage = manager2.tx_storage + block2.set_height(block2_height) + # it should fail if we don't pass sync_checkpoints=True + with self.assertRaises(InvalidNewTransaction): + manager2.on_new_tx(block2, partial=True, fails_silently=False) + # also test if it fails silently + self.assertFalse(manager2.on_new_tx(block2, partial=True, fails_silently=True)) + # otherwise it should be accepted + self.assertTrue(manager2.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False)) + self.assertEqual(block2.get_metadata().validation, ValidationState.CHECKPOINT) + + # XXX: this stopped failing because we removed the partial children metadata which simplified the validation + # and the expected expection is not raised anymore + # # otherwise it should fail if it's not in the checkpoints + # manager3 = self.create_peer('testnet', checkpoints=[]) + # del block2._metadata + # block2.storage = manager3.tx_storage + # block2.set_height(block2_height) + # # failing silently should not raise an exception, but should return False + # self.assertFalse(manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=True)) + # # otherwise it should raise the correct exception + # with self.assertRaises(InvalidNewTransaction): + # manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False) + def test_script(self): genesis_block = self.genesis_blocks[0] @@ -1111,6 +1141,11 @@ def test_sigops_input_multi_below_limit(self) -> None: class SyncV1TransactionTest(unittest.SyncV1Params, BaseTransactionTest): __test__ = True + # XXX: this test is disabled here because it's specific to sync-v2, on the future we could move it to the SyncV2... + # class, but doing it like this minimizes the git diff + def test_checkpoint_validation(self): + pass + class SyncV2TransactionTest(unittest.SyncV2Params, BaseTransactionTest): __test__ = True diff --git a/tests/unittest.py b/tests/unittest.py index f0efcafc3..ecd9db23c 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -132,15 +132,7 @@ def create_peer(self, network, peer_id=None, wallet=None, tx_storage=None, unloc capabilities=None, full_verification=True, enable_sync_v1=None, enable_sync_v2=None, checkpoints=None, utxo_index=False, event_manager=None, use_memory_index=None, start_manager=True, pubsub=None): - if enable_sync_v1 is None: - assert hasattr(self, '_enable_sync_v1'), ('`_enable_sync_v1` has no default by design, either set one on ' - 'the test class or pass `enable_sync_v1` by argument') - enable_sync_v1 = self._enable_sync_v1 - if enable_sync_v2 is None: - assert hasattr(self, '_enable_sync_v2'), ('`_enable_sync_v2` has no default by design, either set one on ' - 'the test class or pass `enable_sync_v2` by argument') - enable_sync_v2 = self._enable_sync_v2 - assert enable_sync_v1 or enable_sync_v2, 'enable at least one sync version' + enable_sync_v1, enable_sync_v2 = self._syncVersionFlags(enable_sync_v1, enable_sync_v2) if peer_id is None: peer_id = PeerId() @@ -225,21 +217,86 @@ def assertIsTopological(self, tx_sequence: Iterator[BaseTransaction], message: O self.assertIn(dep, valid_deps, message) valid_deps.add(tx.hash) + def _syncVersionFlags(self, enable_sync_v1=None, enable_sync_v2=None): + """Internal: use this to check and get the flags and optionally provide override values.""" + if enable_sync_v1 is None: + assert hasattr(self, '_enable_sync_v1'), ('`_enable_sync_v1` has no default by design, either set one on ' + 'the test class or pass `enable_sync_v1` by argument') + enable_sync_v1 = self._enable_sync_v1 + if enable_sync_v2 is None: + assert hasattr(self, '_enable_sync_v2'), ('`_enable_sync_v2` has no default by design, either set one on ' + 'the test class or pass `enable_sync_v2` by argument') + enable_sync_v2 = self._enable_sync_v2 + assert enable_sync_v1 or enable_sync_v2, 'enable at least one sync version' + return enable_sync_v1, enable_sync_v2 + def assertTipsEqual(self, manager1, manager2): + _, enable_sync_v2 = self._syncVersionFlags() + if enable_sync_v2: + self.assertTipsEqualSyncV2(manager1, manager2) + else: + self.assertTipsEqualSyncV1(manager1, manager2) + + def assertTipsNotEqual(self, manager1, manager2): s1 = set(manager1.tx_storage.get_all_tips()) s2 = set(manager2.tx_storage.get_all_tips()) - self.assertEqual(s1, s2) + self.assertNotEqual(s1, s2) - s1 = set(manager1.tx_storage.get_tx_tips()) - s2 = set(manager2.tx_storage.get_tx_tips()) + def assertTipsEqualSyncV1(self, manager1, manager2): + # tx tips + tips1 = {tx.hash for tx in manager1.tx_storage.iter_mempool_tips_from_tx_tips()} + tips2 = {tx.hash for tx in manager2.tx_storage.iter_mempool_tips_from_tx_tips()} + self.log.debug('tx tips1', len=len(tips1), list=shorten_hash(tips1)) + self.log.debug('tx tips2', len=len(tips2), list=shorten_hash(tips2)) + self.assertEqual(tips1, tips2) + + # best block + s1 = set(manager1.tx_storage.get_best_block_tips()) + s2 = set(manager2.tx_storage.get_best_block_tips()) self.assertEqual(s1, s2) - def assertTipsNotEqual(self, manager1, manager2): + # best block (from height index) + b1 = manager1.tx_storage.indexes.height.get_tip() + b2 = manager2.tx_storage.indexes.height.get_tip() + self.assertEqual(b1, b2) + + # all tips must be equal (this check should be removed together with the index) s1 = set(manager1.tx_storage.get_all_tips()) s2 = set(manager2.tx_storage.get_all_tips()) - self.assertNotEqual(s1, s2) + self.assertEqual(s1, s2) + + def assertTipsEqualSyncV2(self, manager1, manager2, *, strict_sync_v2_indexes=True): + # tx tips + if strict_sync_v2_indexes: + tips1 = manager1.tx_storage.indexes.mempool_tips.get() + tips2 = manager2.tx_storage.indexes.mempool_tips.get() + else: + tips1 = {tx.hash for tx in manager1.tx_storage.iter_mempool_tips_from_best_index()} + tips2 = {tx.hash for tx in manager2.tx_storage.iter_mempool_tips_from_best_index()} + self.log.debug('tx tips1', len=len(tips1), list=shorten_hash(tips1)) + self.log.debug('tx tips2', len=len(tips2), list=shorten_hash(tips2)) + self.assertEqual(tips1, tips2) + + # best block + s1 = set(manager1.tx_storage.get_best_block_tips()) + s2 = set(manager2.tx_storage.get_best_block_tips()) + self.log.debug('block tips1', len=len(s1), list=shorten_hash(s1)) + self.log.debug('block tips2', len=len(s2), list=shorten_hash(s2)) + self.assertEqual(s1, s2) + + # best block (from height index) + b1 = manager1.tx_storage.indexes.height.get_tip() + b2 = manager2.tx_storage.indexes.height.get_tip() + self.assertEqual(b1, b2) def assertConsensusEqual(self, manager1, manager2): + _, enable_sync_v2 = self._syncVersionFlags() + if enable_sync_v2: + self.assertConsensusEqualSyncV2(manager1, manager2) + else: + self.assertConsensusEqualSyncV1(manager1, manager2) + + def assertConsensusEqualSyncV1(self, manager1, manager2): self.assertEqual(manager1.tx_storage.get_vertices_count(), manager2.tx_storage.get_vertices_count()) for tx1 in manager1.tx_storage.get_all_transactions(): tx2 = manager2.tx_storage.get_transaction(tx1.hash) @@ -258,6 +315,44 @@ def assertConsensusEqual(self, manager1, manager2): # Hard verification # self.assertEqual(tx1_meta.voided_by, tx2_meta.voided_by) + def assertConsensusEqualSyncV2(self, manager1, manager2, *, strict_sync_v2_indexes=True): + # The current sync algorithm does not propagate voided blocks/txs + # so the count might be different even though the consensus is equal + # One peer might have voided txs that the other does not have + + # to start off, both nodes must have the same tips + self.assertTipsEqualSyncV2(manager1, manager2, strict_sync_v2_indexes=strict_sync_v2_indexes) + + # the following is specific to sync-v2 + + # helper function: + def get_all_executed_or_voided(tx_storage): + """Get all txs separated into three sets: executed, voided, partial""" + tx_executed = set() + tx_voided = set() + tx_partial = set() + for tx in tx_storage.get_all_transactions(): + assert tx.hash is not None + tx_meta = tx.get_metadata() + if not tx_meta.validation.is_fully_connected(): + tx_partial.add(tx.hash) + elif not tx_meta.voided_by: + tx_executed.add(tx.hash) + else: + tx_voided.add(tx.hash) + return tx_executed, tx_voided, tx_partial + + # extract all the transactions from each node, split into three sets + tx_executed1, tx_voided1, tx_partial1 = get_all_executed_or_voided(manager1.tx_storage) + tx_executed2, tx_voided2, tx_partial2 = get_all_executed_or_voided(manager2.tx_storage) + + # both must have the exact same executed set + self.assertEqual(tx_executed1, tx_executed2) + + # XXX: the rest actually doesn't matter + self.log.debug('node1 rest', len_voided=len(tx_voided1), len_partial=len(tx_partial1)) + self.log.debug('node2 rest', len_voided=len(tx_voided2), len_partial=len(tx_partial2)) + def assertConsensusValid(self, manager): for tx in manager.tx_storage.get_all_transactions(): if tx.is_block: @@ -309,6 +404,20 @@ def assertTransactionConsensusValid(self, tx): self.assertTrue(meta.voided_by) self.assertTrue(parent_meta.voided_by.issubset(meta.voided_by)) + def assertSynced(self, node_sync): + """Check "synced" status of p2p-manager, uses self._enable_sync_vX to choose which check to run.""" + enable_sync_v1, enable_sync_v2 = self._syncVersionFlags() + if enable_sync_v2: + self.assertV2Synced(node_sync) + elif enable_sync_v1: + self.assertV1Synced(node_sync) + + def assertV1Synced(self, node_sync): + self.assertEqual(node_sync.synced_timestamp, node_sync.peer_timestamp) + + def assertV2Synced(self, node_sync): + self.assertEqual(node_sync.synced_height, node_sync.peer_height) + def clean_tmpdirs(self): for tmpdir in self.tmpdirs: shutil.rmtree(tmpdir) From 5be9752fb1697f051f5328c845e47d7aff937a26 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Tue, 17 Jan 2023 12:54:47 -0300 Subject: [PATCH 2/9] review changes 4: partially validated voided_by marker --- hathor/conf/settings.py | 5 ++- hathor/consensus.py | 5 +++ hathor/manager.py | 4 +- hathor/transaction/base_transaction.py | 39 +++++++++++++++++++ .../storage/transaction_storage.py | 22 +++++++++-- hathor/transaction/transaction.py | 2 + hathor/transaction/transaction_metadata.py | 9 ++--- tests/resources/transaction/test_tx.py | 11 ++++++ tests/tx/test_cache_storage.py | 2 + tests/tx/test_tx.py | 3 ++ tests/wallet/test_wallet.py | 4 ++ tests/wallet/test_wallet_hd.py | 4 ++ 12 files changed, 100 insertions(+), 10 deletions(-) diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index f43300a16..d87db532e 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -358,9 +358,12 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: # List of soft voided transaction. SOFT_VOIDED_TX_IDS: List[bytes] = [] - # Identifier used in metadata's voided_by. + # Identifier used in metadata's voided_by to mark a tx as soft-voided. SOFT_VOIDED_ID: bytes = b'tx-non-grata' + # Identifier used in metadata's voided_by to mark a tx as partially validated. + PARTIALLY_VALIDATED_ID: bytes = b'in-security-check' # XXX: better name? + ENABLE_EVENT_QUEUE_FEATURE: bool = False EVENT_API_DEFAULT_BATCH_SIZE: int = 100 diff --git a/hathor/consensus.py b/hathor/consensus.py index 08c084649..55c544a28 100644 --- a/hathor/consensus.py +++ b/hathor/consensus.py @@ -103,6 +103,11 @@ def update(self, base: BaseTransaction) -> None: """Run a consensus update with its own context, indexes will be updated accordingly.""" from hathor.transaction import Block, Transaction + # XXX: first make sure we can run the consensus update on this tx: + meta = base.get_metadata() + assert meta.voided_by is None or (settings.PARTIALLY_VALIDATED_ID not in meta.voided_by) + assert meta.validation.is_fully_connected() + # this context instance will live only while this update is running context = self.create_context() diff --git a/hathor/manager.py b/hathor/manager.py index 05c426009..b7e8a95d4 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -466,8 +466,10 @@ def _initialize_components(self) -> None: if self._full_verification: # TODO: deal with invalid tx if tx.can_validate_full(): - self.tx_storage.add_to_indexes(tx) + if tx.is_genesis: + assert tx.validate_checkpoint(self.checkpoints) assert tx.validate_full(skip_block_weight_verification=skip_block_weight_verification) + self.tx_storage.add_to_indexes(tx) self.consensus_algorithm.update(tx) self.tx_storage.indexes.update(tx) if self.tx_storage.indexes.mempool_tips is not None: diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index cbca7beb0..b9266b234 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -494,6 +494,7 @@ def validate_checkpoint(self, checkpoints: List[Checkpoint]) -> bool: self.verify_checkpoint(checkpoints) meta.validation = ValidationState.CHECKPOINT + self.mark_partially_validated() return True def validate_basic(self, skip_block_weight_verification: bool = False) -> bool: @@ -508,6 +509,7 @@ def validate_basic(self, skip_block_weight_verification: bool = False) -> bool: self.verify_basic(skip_block_weight_verification=skip_block_weight_verification) meta.validation = ValidationState.BASIC + self.mark_partially_validated() return True def validate_full(self, skip_block_weight_verification: bool = False, sync_checkpoints: bool = False, @@ -531,7 +533,10 @@ def validate_full(self, skip_block_weight_verification: bool = False, sync_check # skip full validation when it is a checkpoint if meta.validation.is_checkpoint(): meta.validation = ValidationState.CHECKPOINT_FULL + # at last, remove the partially validated mark + self.unmark_partially_validated() return True + # XXX: in some cases it might be possible that this transaction is verified by a checkpoint but we went # directly into trying a full validation so we should check it here to make sure the validation states # ends up being CHECKPOINT_FULL instead of FULL @@ -544,8 +549,36 @@ def validate_full(self, skip_block_weight_verification: bool = False, sync_check meta.validation = ValidationState.CHECKPOINT_FULL else: meta.validation = ValidationState.FULL + + # at last, remove the partially validated mark + self.unmark_partially_validated() return True + def mark_partially_validated(self) -> None: + """ This function is used to add the partially-validated mark from the voided-by metadata. + + It is idempotent: calling it multiple time has the same effect as calling it once. But it must only be called + when the validation state is *NOT* "fully connected", otherwise it'll raise an assertion error. + """ + tx_meta = self.get_metadata() + assert not tx_meta.validation.is_fully_connected() + if tx_meta.voided_by is None: + tx_meta.voided_by = set() + tx_meta.voided_by.add(settings.PARTIALLY_VALIDATED_ID) + + def unmark_partially_validated(self) -> None: + """ This function is used to remove the partially-validated mark from the voided-by metadata. + + It is idempotent: calling it multiple time has the same effect as calling it once. But it must only be called + when the validation state is "fully connected", otherwise it'll raise an assertion error. + """ + tx_meta = self.get_metadata() + assert tx_meta.validation.is_fully_connected() + if tx_meta.voided_by is not None: + tx_meta.voided_by.discard(settings.PARTIALLY_VALIDATED_ID) + if not tx_meta.voided_by: + tx_meta.voided_by = None + @abstractmethod def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: """Check that this tx is a known checkpoint or is parent of another checkpoint-valid tx/block. @@ -711,6 +744,9 @@ def resolve(self, update_time: bool = True) -> bool: if hash_bytes: self.hash = hash_bytes + metadata = getattr(self, '_metadata', None) + if metadata is not None and metadata.hash is not None: + metadata.hash = hash_bytes return True else: return False @@ -859,12 +895,15 @@ def reset_metadata(self) -> None: """ Reset transaction's metadata. It is used when a node is initializing and recalculating all metadata. """ + from hathor.transaction.transaction_metadata import ValidationState assert self.storage is not None score = self.weight if self.is_genesis else 0 self._metadata = TransactionMetadata(hash=self.hash, score=score, accumulated_weight=self.weight, height=self.calculate_height(), min_height=self.calculate_min_height()) + self._metadata.validation = ValidationState.INITIAL + self._metadata.voided_by = {settings.PARTIALLY_VALIDATED_ID} self._metadata._tx_ref = weakref.ref(self) self.storage.save_transaction(self, only_metadata=True) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 602dda4e2..13c2a271c 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -349,10 +349,26 @@ def save_transaction(self: 'TransactionStorage', tx: BaseTransaction, *, only_me """ assert tx.hash is not None meta = tx.get_metadata() + self.pre_save_validation(tx, meta) - # XXX: we can only add to cache and publish txs that are fully connected (which also implies it's valid) - if not meta.validation.is_fully_connected(): - return + def pre_save_validation(self, tx: BaseTransaction, tx_meta: TransactionMetadata) -> None: + """ Must be run before every save, only raises AssertionError. + + A failure means there is a bug in the code that allowed the condition to reach the "save" code. This is a last + second measure to prevent persisting a bad transaction/metadata. + + This method receives the transaction AND the metadata in order to avoid calling ".get_metadata()" which could + potentially create a fresh metadata. + """ + assert tx.hash is not None + assert tx_meta.hash is not None + assert tx.hash == tx_meta.hash, f'{tx.hash.hex()} != {tx_meta.hash.hex()}' + voided_by = tx_meta.voided_by or set() + # XXX: PARTIALLY_VALIDATED_ID must be included if the tx is fully connected and must not included otherwise + has_partially_validated_marker = settings.PARTIALLY_VALIDATED_ID in voided_by + validation_is_fully_connected = tx_meta.validation.is_fully_connected() + assert (not has_partially_validated_marker) == validation_is_fully_connected, \ + f'{has_partially_validated_marker} == {validation_is_fully_connected}' @abstractmethod def remove_transaction(self, tx: BaseTransaction) -> None: diff --git a/hathor/transaction/transaction.py b/hathor/transaction/transaction.py index c282de45a..6a218c9ab 100644 --- a/hathor/transaction/transaction.py +++ b/hathor/transaction/transaction.py @@ -274,6 +274,8 @@ def verify_basic(self, skip_block_weight_verification: bool = False) -> None: def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: assert self.storage is not None + if self.is_genesis: + return meta = self.get_metadata() # at least one child must be checkpoint validated for child_tx in map(self.storage.get_transaction, meta.children): diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index cb05539bf..1ba8bc11e 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -139,12 +139,11 @@ def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: # Hash of the transactions that conflicts with this transaction. self.conflict_with = None - # Hash of the transactions that void this transaction. - # - # When a transaction has a conflict and is voided because of this conflict, its own hash is added to + # - Hashes of the transactions that void this transaction. + # - When a transaction has a conflict and is voided because of this conflict, its own hash is added to # voided_by. The logic is that the transaction is voiding itself. - # - # When a block is voided, its own hash is added to voided_by. + # - When a block is voided, its own hash is added to voided_by. + # - When it is constructed it will be voided by "partially validated" until it is validated self.voided_by = None self._last_voided_by_hash = None diff --git a/tests/resources/transaction/test_tx.py b/tests/resources/transaction/test_tx.py index b94028760..c1b981f95 100644 --- a/tests/resources/transaction/test_tx.py +++ b/tests/resources/transaction/test_tx.py @@ -62,6 +62,8 @@ def test_get_one(self): @inlineCallbacks def test_get_one_known_tx(self): + from hathor.transaction.transaction_metadata import ValidationState + # Tx tesnet 0033784bc8443ba851fd88d81c6f06774ae529f25c1fa8f026884ad0a0e98011 # We had a bug with this endpoint in this tx because the token_data from inputs # was being copied from the output @@ -84,6 +86,7 @@ def test_get_one_known_tx(self): '2dc703120a77192fc16eda9ed22e1b88ac40200000218def416095b08602003d3c40fb04737e1a2a848cfd2592490a71cd' '0248b9e7d6a626f45dec86975b00f4dd53f84f1f0091125250b044e49023fbbd0f74f6093cdd2226fdff3e09a1000002be') tx = Transaction.create_from_struct(bytes.fromhex(tx_hex), self.manager.tx_storage) + tx.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx) tx_parent1_hex = ('0001010102001c382847d8440d05da95420bee2ebeb32bc437f82a9ae47b0745c8a29a7b0d001c382847d844' @@ -95,6 +98,7 @@ def test_get_one_known_tx(self): '5250b044e49023fbbd0f74f6093cdd2226fdff3e09a1001f16fe62e3433bcc74b262c11a1fa94fcb38484f4d' '8fb080f53a0c9c57ddb000000120') tx_parent1 = Transaction.create_from_struct(bytes.fromhex(tx_parent1_hex), self.manager.tx_storage) + tx_parent1.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx_parent1) tx_parent2_hex = ('0001000103001f16fe62e3433bcc74b262c11a1fa94fcb38484f4d8fb080f53a0c9c57ddb001006946304402' @@ -106,6 +110,7 @@ def test_get_one_known_tx(self): '62e3433bcc74b262c11a1fa94fcb38484f4d8fb080f53a0c9c57ddb00065329457d13410ac711318bd941e16' 'd57709926b76e64763bf19c3f13eeac30000016d') tx_parent2 = Transaction.create_from_struct(bytes.fromhex(tx_parent2_hex), self.manager.tx_storage) + tx_parent2.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx_parent2) tx_input_hex = ('0001010203007231eee3cb6160d95172a409d634d0866eafc8775f5729fff6a61e7850aba500b3ab76c5337b55' @@ -120,6 +125,7 @@ def test_get_one_known_tx(self): '5e95ac369b31f46188ac40200000218def416082eba802000e4e54b2922c1fa34b5d427f1e96885612e28673ac' 'cfaf6e7ceb2ba91c9c84009c8174d4a46ebcc789d1989e3dec5b68cffeef239fd8cf86ef62728e2eacee000001b6') tx_input = Transaction.create_from_struct(bytes.fromhex(tx_input_hex), self.manager.tx_storage) + tx_input.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx_input) # XXX: this is completely dependant on MemoryTokensIndex implementation, hence use_memory_storage=True @@ -170,6 +176,8 @@ def test_get_one_known_tx(self): @inlineCallbacks def test_get_one_known_tx_with_authority(self): + from hathor.transaction.transaction_metadata import ValidationState + # Tx tesnet 00005f234469407614bf0abedec8f722bb5e534949ad37650f6077c899741ed7 # We had a bug with this endpoint in this tx because the token_data from inputs # was not considering authority mask @@ -186,6 +194,7 @@ def test_get_one_known_tx_with_authority(self): '7851af043c11e19f28675b010e8cf4d8da3278f126d2429490a804a7fb2c000023b318c91dcfd4b967b205dc938f9f5e2fd' '5114256caacfb8f6dd13db33000020393') tx = Transaction.create_from_struct(bytes.fromhex(tx_hex), self.manager.tx_storage) + tx.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx) tx_parent1_hex = ('0001010203000023b318c91dcfd4b967b205dc938f9f5e2fd5114256caacfb8f6dd13db330000023b318c91dcfd' @@ -200,6 +209,7 @@ def test_get_one_known_tx_with_authority(self): '08ef288ac40311513e4fef9d161087be202000023b318c91dcfd4b967b205dc938f9f5e2fd5114256caacfb8f6d' 'd13db3300038c3d3b69ce90bb88c0c4d6a87b9f0c349e5b10c9b7ce6714f996e512ac16400021261') tx_parent1 = Transaction.create_from_struct(bytes.fromhex(tx_parent1_hex), self.manager.tx_storage) + tx_parent1.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx_parent1) tx_parent2_hex = ('000201040000476810205cb3625d62897fcdad620e01d66649869329640f5504d77e960d01006a473045022100c' @@ -213,6 +223,7 @@ def test_get_one_known_tx_with_authority(self): '00d810') tx_parent2_bytes = bytes.fromhex(tx_parent2_hex) tx_parent2 = TokenCreationTransaction.create_from_struct(tx_parent2_bytes, self.manager.tx_storage) + tx_parent2.get_metadata().validation = ValidationState.FULL self.manager.tx_storage.save_transaction(tx_parent2) # Both inputs are the same as the last parent, so no need to manually add them diff --git a/tests/tx/test_cache_storage.py b/tests/tx/test_cache_storage.py index 6d92791a1..cb4461624 100644 --- a/tests/tx/test_cache_storage.py +++ b/tests/tx/test_cache_storage.py @@ -31,9 +31,11 @@ def tearDown(self): super().tearDown() def _get_new_tx(self, nonce): + from hathor.transaction.transaction_metadata import ValidationState tx = Transaction(nonce=nonce, storage=self.cache_storage) tx.update_hash() meta = TransactionMetadata(hash=tx.hash) + meta.validation = ValidationState.FULL tx._metadata = meta return tx diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 7d463a5d0..38a5ac514 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -226,6 +226,8 @@ def test_struct(self): def test_children_update(self): tx = self._gen_tx_spending_genesis_block() + tx.weight = 1.0 + tx.resolve() # get info before update children_len = [] @@ -234,6 +236,7 @@ def test_children_update(self): children_len.append(len(metadata.children)) # update metadata + tx.validate_full() tx.update_initial_metadata() # genesis transactions should have only this tx in their children set diff --git a/tests/wallet/test_wallet.py b/tests/wallet/test_wallet.py index fcaa87741..ed05b53a6 100644 --- a/tests/wallet/test_wallet.py +++ b/tests/wallet/test_wallet.py @@ -55,6 +55,8 @@ def test_wallet_keys_storage(self): self.assertEqual(key, key2) def test_wallet_create_transaction(self): + from hathor.transaction.transaction_metadata import ValidationState + genesis_private_key_bytes = get_private_key_bytes( self.genesis_private_key, encryption_algorithm=serialization.BestAvailableEncryption(PASSWORD) @@ -84,6 +86,7 @@ def test_wallet_create_transaction(self): tx1 = w.prepare_transaction_compute_inputs(Transaction, [out], self.storage) tx1.storage = self.storage tx1.update_hash() + tx1.get_metadata().validation = ValidationState.FULL self.storage.save_transaction(tx1) w.on_new_tx(tx1) self.assertEqual(len(w.spent_txs), 1) @@ -99,6 +102,7 @@ def test_wallet_create_transaction(self): outputs=[out], tx_storage=self.storage) tx2.storage = self.storage tx2.update_hash() + tx2.get_metadata().validation = ValidationState.FULL self.storage.save_transaction(tx2) w.on_new_tx(tx2) self.assertEqual(len(w.spent_txs), 2) diff --git a/tests/wallet/test_wallet_hd.py b/tests/wallet/test_wallet_hd.py index bec8154d1..c231053ba 100644 --- a/tests/wallet/test_wallet_hd.py +++ b/tests/wallet/test_wallet_hd.py @@ -25,6 +25,8 @@ def setUp(self): self.TOKENS = self.BLOCK_TOKENS def test_transaction_and_balance(self): + from hathor.transaction.transaction_metadata import ValidationState + # generate a new block and check if we increase balance new_address = self.wallet.get_unused_address() out = WalletOutputInfo(decode_address(new_address), self.TOKENS, timelock=None) @@ -42,6 +44,7 @@ def test_transaction_and_balance(self): tx1.update_hash() tx1.verify_script(tx1.inputs[0], block) tx1.storage = self.tx_storage + tx1.get_metadata().validation = ValidationState.FULL self.wallet.on_new_tx(tx1) self.tx_storage.save_transaction(tx1) self.assertEqual(len(self.wallet.spent_txs), 1) @@ -60,6 +63,7 @@ def test_transaction_and_balance(self): tx2.update_hash() tx2.storage = self.tx_storage tx2.verify_script(tx2.inputs[0], tx1) + tx2.get_metadata().validation = ValidationState.FULL self.tx_storage.save_transaction(tx2) self.wallet.on_new_tx(tx2) self.assertEqual(len(self.wallet.spent_txs), 2) From d6edf55cd72d1ab7a677900618f80720d4cd6463 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 27 Jan 2023 19:43:17 -0300 Subject: [PATCH 3/9] review changes 5: get_transaction allow_partially_valid protection --- hathor/indexes/deps_index.py | 2 +- hathor/indexes/memory_deps_index.py | 9 +++-- hathor/indexes/mempool_tips_index.py | 4 ++- hathor/indexes/rocksdb_deps_index.py | 8 +++-- hathor/manager.py | 18 ++++++---- hathor/p2p/sync_checkpoints.py | 2 +- hathor/transaction/base_transaction.py | 17 ++++++---- hathor/transaction/block.py | 4 +-- hathor/transaction/storage/cache_storage.py | 4 +-- hathor/transaction/storage/exceptions.py | 4 +++ hathor/transaction/storage/memory_storage.py | 7 ++-- hathor/transaction/storage/rocksdb_storage.py | 8 +++-- .../storage/transaction_storage.py | 30 ++++++++++------- hathor/transaction/transaction.py | 4 ++- tests/resources/transaction/test_tx.py | 33 +++++++++++++++++-- 15 files changed, 109 insertions(+), 45 deletions(-) diff --git a/hathor/indexes/deps_index.py b/hathor/indexes/deps_index.py index 0b7d07276..fbea6a67c 100644 --- a/hathor/indexes/deps_index.py +++ b/hathor/indexes/deps_index.py @@ -44,7 +44,7 @@ def get_requested_from_height(tx: BaseTransaction) -> int: # I'm defaulting the height to `inf` (practically), this should make it heightest priority when # choosing which transactions to fetch next return INF_HEIGHT - block = tx.storage.get_transaction(first_block) + block = tx.storage.get_transaction(first_block, allow_partially_valid=True) assert isinstance(block, Block) return block.get_metadata().height diff --git a/hathor/indexes/memory_deps_index.py b/hathor/indexes/memory_deps_index.py index afb0d0aba..7de1700c8 100644 --- a/hathor/indexes/memory_deps_index.py +++ b/hathor/indexes/memory_deps_index.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from functools import partial from typing import TYPE_CHECKING, Dict, FrozenSet, Iterator, List, Optional, Set, Tuple from structlog import get_logger @@ -75,7 +76,7 @@ def _update_new_deps_ready(self, tx: BaseTransaction) -> None: assert tx.hash is not None assert tx.storage is not None for candidate_hash in self._rev_dep_index.get(tx.hash, []): - candidate_tx = tx.storage.get_transaction(candidate_hash) + candidate_tx = tx.storage.get_transaction(candidate_hash, allow_partially_valid=True) if candidate_tx.is_ready_for_validation(): self._txs_with_deps_ready.add(candidate_hash) @@ -105,12 +106,13 @@ def remove_ready_for_validation(self, tx: bytes) -> None: self._txs_with_deps_ready.discard(tx) def next_ready_for_validation(self, tx_storage: 'TransactionStorage', *, dry_run: bool = False) -> Iterator[bytes]: + get_partially_validated = partial(tx_storage.get_transaction, allow_partially_valid=True) if dry_run: cur_ready = self._txs_with_deps_ready.copy() else: cur_ready, self._txs_with_deps_ready = self._txs_with_deps_ready, set() while cur_ready: - yield from sorted(cur_ready, key=lambda tx_hash: tx_storage.get_transaction(tx_hash).timestamp) + yield from sorted(cur_ready, key=lambda tx_hash: get_partially_validated(tx_hash).timestamp) if dry_run: cur_ready = self._txs_with_deps_ready - cur_ready else: @@ -129,7 +131,8 @@ def _get_rev_deps(self, tx: bytes) -> FrozenSet[bytes]: def known_children(self, tx: BaseTransaction) -> List[bytes]: assert tx.hash is not None assert tx.storage is not None - it_rev_deps = map(tx.storage.get_transaction, self._get_rev_deps(tx.hash)) + get_partially_validated = partial(tx.storage.get_transaction, allow_partially_valid=True) + it_rev_deps = map(get_partially_validated, self._get_rev_deps(tx.hash)) return [not_none(rev.hash) for rev in it_rev_deps if tx.hash in rev.parents] # needed-txs-index methods: diff --git a/hathor/indexes/mempool_tips_index.py b/hathor/indexes/mempool_tips_index.py index 9c322fcd8..6e7ea8485 100644 --- a/hathor/indexes/mempool_tips_index.py +++ b/hathor/indexes/mempool_tips_index.py @@ -14,6 +14,7 @@ from abc import abstractmethod from collections.abc import Collection +from functools import partial from typing import TYPE_CHECKING, Iterable, Iterator, Optional, Set, cast import structlog @@ -169,7 +170,8 @@ def update(self, tx: BaseTransaction, *, remove: Optional[bool] = None) -> None: self._add(tx.hash) def iter(self, tx_storage: 'TransactionStorage', max_timestamp: Optional[float] = None) -> Iterator[Transaction]: - it: Iterator[BaseTransaction] = map(tx_storage.get_transaction, self._index) + get_partially_validated = partial(tx_storage.get_transaction, allow_partially_valid=True) + it: Iterator[BaseTransaction] = map(get_partially_validated, self._index) if max_timestamp is not None: it = filter(lambda tx: tx.timestamp < not_none(max_timestamp), it) yield from cast(Iterator[Transaction], it) diff --git a/hathor/indexes/rocksdb_deps_index.py b/hathor/indexes/rocksdb_deps_index.py index d5e40b788..4967f0a8a 100644 --- a/hathor/indexes/rocksdb_deps_index.py +++ b/hathor/indexes/rocksdb_deps_index.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from enum import Enum +from functools import partial from typing import TYPE_CHECKING, FrozenSet, Iterator, List, Optional, Tuple from structlog import get_logger @@ -219,7 +220,7 @@ def _update_new_deps_ready(self, tx: BaseTransaction, batch: 'rocksdb.WriteBatch assert tx.hash is not None assert tx.storage is not None for candidate_hash in self._iter_rev_deps_of(tx.hash): - candidate_tx = tx.storage.get_transaction(candidate_hash) + candidate_tx = tx.storage.get_transaction(candidate_hash, allow_partially_valid=True) if candidate_tx.is_ready_for_validation(): self._add_ready(candidate_hash, batch) @@ -266,7 +267,7 @@ def next_ready_for_validation(self, tx_storage: 'TransactionStorage', *, dry_run def _drain_all_sorted_ready(self, tx_storage: 'TransactionStorage', batch: 'rocksdb.WriteBatch') -> List[bytes]: ready = list(self._drain_all_ready(tx_storage, batch)) - ready.sort(key=lambda tx_hash: tx_storage.get_transaction(tx_hash).timestamp) + ready.sort(key=lambda tx_hash: tx_storage.get_transaction(tx_hash, allow_partially_valid=True).timestamp) return ready def _drain_all_ready(self, tx_storage: 'TransactionStorage', batch: 'rocksdb.WriteBatch') -> Iterator[bytes]: @@ -315,7 +316,8 @@ def _iter_has_rev_deps(self) -> Iterator[bytes]: def known_children(self, tx: BaseTransaction) -> List[bytes]: assert tx.hash is not None assert tx.storage is not None - it_rev_deps = map(tx.storage.get_transaction, self._get_rev_deps(tx.hash)) + get_partially_validated = partial(tx.storage.get_transaction, allow_partially_valid=True) + it_rev_deps = map(get_partially_validated, self._get_rev_deps(tx.hash)) return [not_none(rev.hash) for rev in it_rev_deps if tx.hash in rev.parents] def _get_rev_deps(self, tx: bytes) -> FrozenSet[bytes]: diff --git a/hathor/manager.py b/hathor/manager.py index b7e8a95d4..29b96a979 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -433,9 +433,6 @@ def _initialize_components(self) -> None: self.log.debug('load blocks and transactions') for tx in self.tx_storage._topological_sort_dfs(): - if self._full_verification: - tx.update_initial_metadata() - assert tx.hash is not None tx_meta = tx.get_metadata() @@ -465,7 +462,11 @@ def _initialize_components(self) -> None: try: if self._full_verification: # TODO: deal with invalid tx + tx.calculate_height() + tx._update_parents_children_metadata() + if tx.can_validate_full(): + tx.calculate_min_height() if tx.is_genesis: assert tx.validate_checkpoint(self.checkpoints) assert tx.validate_full(skip_block_weight_verification=skip_block_weight_verification) @@ -544,7 +545,7 @@ def _initialize_components(self) -> None: for tx_hash in self.tx_storage.indexes.deps.iter(): if not self.tx_storage.transaction_exists(tx_hash): continue - tx = self.tx_storage.get_transaction(tx_hash) + tx = self.tx_storage.get_transaction(tx_hash, allow_partially_valid=True) if tx.get_metadata().validation.is_final(): depended_final_txs.append(tx) if self.tx_storage.indexes.deps is not None: @@ -697,7 +698,7 @@ def _sync_v2_resume_validations(self) -> None: for tx_hash in self.tx_storage.indexes.deps.iter(): if not self.tx_storage.transaction_exists(tx_hash): continue - tx = self.tx_storage.get_transaction(tx_hash) + tx = self.tx_storage.get_transaction(tx_hash, allow_partially_valid=True) if tx.get_metadata().validation.is_final(): depended_final_txs.append(tx) self.sync_v2_step_validations(depended_final_txs, quiet=True) @@ -983,7 +984,7 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non tx.storage = self.tx_storage try: - metadata = tx.get_metadata() + metadata = tx.get_metadata(allow_partial=partial) except TransactionDoesNotExist: if not fails_silently: raise InvalidNewTransaction('missing parent') @@ -1082,6 +1083,8 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non def sync_v2_step_validations(self, txs: Iterable[BaseTransaction], *, quiet: bool) -> None: """ Step all validations until none can be stepped anymore. """ + from functools import partial + assert self.tx_storage.indexes is not None assert self.tx_storage.indexes.deps is not None # cur_txs will be empty when there are no more new txs that reached full @@ -1090,7 +1093,8 @@ def sync_v2_step_validations(self, txs: Iterable[BaseTransaction], *, quiet: boo assert ready_tx.hash is not None self.tx_storage.indexes.deps.remove_ready_for_validation(ready_tx.hash) it_next_ready = self.tx_storage.indexes.deps.next_ready_for_validation(self.tx_storage) - for tx in map(self.tx_storage.get_transaction, it_next_ready): + get_partially_validated = partial(self.tx_storage.get_transaction, allow_partially_valid=True) + for tx in map(get_partially_validated, it_next_ready): assert tx.hash is not None tx.update_initial_metadata() try: diff --git a/hathor/p2p/sync_checkpoints.py b/hathor/p2p/sync_checkpoints.py index 14ca13914..25dc6e342 100644 --- a/hathor/p2p/sync_checkpoints.py +++ b/hathor/p2p/sync_checkpoints.py @@ -139,7 +139,7 @@ def _checkpoint_sync_interval(self, checkpoint: Checkpoint) -> Optional[_SyncInt # XXX: this could be optimized a lot, but it actually isn't that slow while start_height > end_height: try: - block = self.manager.tx_storage.get_transaction(start_hash) + block = self.manager.tx_storage.get_transaction(start_hash, allow_partially_valid=True) except TransactionDoesNotExist: break assert isinstance(block, Block) diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index b9266b234..f5f528d59 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -855,7 +855,8 @@ def start_mining(self, start: int = 0, end: int = MAX_NONCE, sleep_seconds: floa return None return None - def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True) -> TransactionMetadata: + def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True, allow_partial: bool = False + ) -> TransactionMetadata: """Return this tx's metadata. It first looks in our cache (tx._metadata) and then tries the tx storage. If it doesn't @@ -875,7 +876,7 @@ def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True) metadata = getattr(self, '_metadata', None) if not metadata and use_storage and self.storage: assert self.hash is not None - metadata = self.storage.get_metadata(self.hash) + metadata = self.storage.get_metadata(self.hash, allow_partially_valid=allow_partial) self._metadata = metadata if not metadata: # FIXME: there is code that set use_storage=False but relies on correct height being calculated @@ -899,11 +900,13 @@ def reset_metadata(self) -> None: assert self.storage is not None score = self.weight if self.is_genesis else 0 self._metadata = TransactionMetadata(hash=self.hash, score=score, - accumulated_weight=self.weight, - height=self.calculate_height(), - min_height=self.calculate_min_height()) - self._metadata.validation = ValidationState.INITIAL - self._metadata.voided_by = {settings.PARTIALLY_VALIDATED_ID} + accumulated_weight=self.weight) + if self.is_genesis: + self._metadata.validation = ValidationState.CHECKPOINT_FULL + self._metadata.voided_by = set() + else: + self._metadata.validation = ValidationState.INITIAL + self._metadata.voided_by = {settings.PARTIALLY_VALIDATED_ID} self._metadata._tx_ref = weakref.ref(self) self.storage.save_transaction(self, only_metadata=True) diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 3fcd8878b..5c0fdabd0 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -169,7 +169,7 @@ def get_block_parent(self) -> 'Block': """Return the parent block. """ assert self.storage is not None - block_parent = self.storage.get_transaction(self.get_block_parent_hash()) + block_parent = self.storage.get_transaction(self.get_block_parent_hash(), allow_partially_valid=True) assert isinstance(block_parent, Block) return block_parent @@ -271,7 +271,7 @@ def has_basic_block_parent(self) -> bool: parent_block_hash = self.parents[0] if not self.storage.transaction_exists(parent_block_hash): return False - metadata = self.storage.get_metadata(parent_block_hash) + metadata = self.storage.get_metadata(parent_block_hash, allow_partially_valid=True) assert metadata is not None return metadata.validation.is_at_least_basic() diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index f662b22f5..e908c4ca8 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -189,7 +189,7 @@ def transaction_exists(self, hash_bytes: bytes) -> bool: return True return self.store.transaction_exists(hash_bytes) - def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: + def _get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> BaseTransaction: tx: Optional[BaseTransaction] if hash_bytes in self.cache: tx = self._clone(self.cache[hash_bytes]) @@ -200,7 +200,7 @@ def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: if tx is not None: self.stats['hit'] += 1 else: - tx = self.store.get_transaction(hash_bytes) + tx = self.store.get_transaction(hash_bytes, allow_partially_valid=allow_partially_valid) tx.storage = self self.stats['miss'] += 1 self._update_cache(tx) diff --git a/hathor/transaction/storage/exceptions.py b/hathor/transaction/storage/exceptions.py index b587f426c..ee071a153 100644 --- a/hathor/transaction/storage/exceptions.py +++ b/hathor/transaction/storage/exceptions.py @@ -19,6 +19,10 @@ class TransactionDoesNotExist(HathorError): """You are trying to get a transaction that does not exist""" +class TransactionPartiallyValidatedError(TransactionDoesNotExist): + """You are trying to get a transaction that has not been fully validated yet, treated as non-existent""" + + class TransactionMetadataDoesNotExist(HathorError): """You are trying to get a metadata (of a transaction) that does not exist""" diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index c363ce188..21304bda7 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -14,7 +14,7 @@ from typing import Any, Dict, Iterator, Optional, TypeVar -from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionPartiallyValidatedError from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.transaction.transaction import BaseTransaction @@ -80,11 +80,14 @@ def _save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) def transaction_exists(self, hash_bytes: bytes) -> bool: return hash_bytes in self.transactions - def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: + def _get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> BaseTransaction: if hash_bytes in self.transactions: tx = self._clone(self.transactions[hash_bytes]) if hash_bytes in self.metadata: tx._metadata = self._clone(self.metadata[hash_bytes]) + assert tx._metadata is not None + if not allow_partially_valid and not tx._metadata.validation.is_fully_connected(): + raise TransactionPartiallyValidatedError(tx.hash_hex) return tx else: raise TransactionDoesNotExist(hash_bytes.hex()) diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index c6e48feae..a15197ada 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -18,7 +18,7 @@ from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager from hathor.storage import RocksDBStorage -from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionPartiallyValidatedError from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.util import json_dumpb, json_loadb @@ -113,7 +113,7 @@ def transaction_exists(self, hash_bytes: bytes) -> bool: tx_exists = self._db.get((self._cf_tx, hash_bytes)) is not None return tx_exists - def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': + def _get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> 'BaseTransaction': tx = self.get_transaction_from_weakref(hash_bytes) if tx is not None: return tx @@ -122,6 +122,10 @@ def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': if not tx: raise TransactionDoesNotExist(hash_bytes.hex()) + assert tx._metadata is not None + if not allow_partially_valid and not tx._metadata.validation.is_fully_connected(): + raise TransactionPartiallyValidatedError(tx.hash_hex) + assert tx.hash == hash_bytes self._save_to_weakref(tx) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 13c2a271c..5aabfdfda 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -27,7 +27,11 @@ from hathor.pubsub import PubSubManager from hathor.transaction.base_transaction import BaseTransaction from hathor.transaction.block import Block -from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionIsNotABlock +from hathor.transaction.storage.exceptions import ( + TransactionDoesNotExist, + TransactionIsNotABlock, + TransactionPartiallyValidatedError, +) from hathor.transaction.storage.migrations import BaseMigration, MigrationState, add_min_height_metadata from hathor.transaction.transaction import Transaction from hathor.transaction.transaction_metadata import TransactionMetadata @@ -409,7 +413,7 @@ def remove_transactions(self, txs: List[BaseTransaction]) -> None: self.save_transaction(spent_tx, only_metadata=True) assert not dangling_children, 'It is an error to try to remove transactions that would leave a gap in the DAG' for parent_hash, children_to_remove in parents_to_update.items(): - parent_tx = self.get_transaction(parent_hash) + parent_tx = self.get_transaction(parent_hash, allow_partially_valid=True) parent_meta = parent_tx.get_metadata() for child in children_to_remove: parent_meta.children.remove(child) @@ -429,7 +433,7 @@ def transaction_exists(self, hash_bytes: bytes) -> bool: def compare_bytes_with_local_tx(self, tx: BaseTransaction) -> bool: """Compare byte-per-byte `tx` with the local transaction.""" assert tx.hash is not None - local_tx = self.get_transaction(tx.hash) + local_tx = self.get_transaction(tx.hash, allow_partially_valid=True) local_tx_bytes = bytes(local_tx) tx_bytes = bytes(tx) if tx_bytes == local_tx_bytes: @@ -439,7 +443,7 @@ def compare_bytes_with_local_tx(self, tx: BaseTransaction) -> bool: return False @abstractmethod - def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: + def _get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> BaseTransaction: """Returns the transaction with hash `hash_bytes`. :param hash_bytes: Hash in bytes that will be checked. @@ -469,7 +473,7 @@ def _get_lock(self, hash_bytes: bytes) -> Optional[Lock]: self._weakref_lock_per_hash[hash_bytes] = lock return lock - def get_transaction(self, hash_bytes: bytes) -> BaseTransaction: + def get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> BaseTransaction: """Acquire the lock and get the transaction with hash `hash_bytes`. :param hash_bytes: Hash in bytes that will be checked. @@ -478,19 +482,23 @@ def get_transaction(self, hash_bytes: bytes) -> BaseTransaction: lock = self._get_lock(hash_bytes) assert lock is not None with lock: - tx = self._get_transaction(hash_bytes) + tx = self._get_transaction(hash_bytes, allow_partially_valid=allow_partially_valid) else: - tx = self._get_transaction(hash_bytes) + tx = self._get_transaction(hash_bytes, allow_partially_valid=allow_partially_valid) + if not allow_partially_valid: + tx_meta = tx.get_metadata() + if tx_meta.voided_by is not None and settings.PARTIALLY_VALIDATED_ID in tx_meta.voided_by: + raise TransactionPartiallyValidatedError(tx.hash_hex) return tx - def get_metadata(self, hash_bytes: bytes) -> Optional[TransactionMetadata]: + def get_metadata(self, hash_bytes: bytes, *, allow_partially_valid: bool = False) -> Optional[TransactionMetadata]: """Returns the transaction metadata with hash `hash_bytes`. :param hash_bytes: Hash in bytes that will be checked. :rtype :py:class:`hathor.transaction.TransactionMetadata` """ try: - tx = self.get_transaction(hash_bytes) + tx = self.get_transaction(hash_bytes, allow_partially_valid=allow_partially_valid) return tx.get_metadata(use_storage=False) except TransactionDoesNotExist: return None @@ -1212,7 +1220,7 @@ def _run_topological_sort_dfs(self, root: BaseTransaction, visited: Dict[bytes, for parent_hash in tx.parents[::-1]: if parent_hash not in visited: try: - parent = self.get_transaction(parent_hash) + parent = self.get_transaction(parent_hash, allow_partially_valid=True) except TransactionDoesNotExist: # XXX: it's possible transactions won't exist because of missing dependencies pass @@ -1222,7 +1230,7 @@ def _run_topological_sort_dfs(self, root: BaseTransaction, visited: Dict[bytes, for txin in tx.inputs: if txin.tx_id not in visited: try: - txinput = self.get_transaction(txin.tx_id) + txinput = self.get_transaction(txin.tx_id, allow_partially_valid=True) except TransactionDoesNotExist: # XXX: it's possible transactions won't exist because of missing dependencies pass diff --git a/hathor/transaction/transaction.py b/hathor/transaction/transaction.py index 6a218c9ab..8aacf1e38 100644 --- a/hathor/transaction/transaction.py +++ b/hathor/transaction/transaction.py @@ -13,6 +13,7 @@ # limitations under the License. import hashlib +from functools import partial from itertools import chain from struct import pack from typing import TYPE_CHECKING, Any, Dict, Iterator, List, NamedTuple, Optional, Set, Tuple @@ -277,8 +278,9 @@ def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: if self.is_genesis: return meta = self.get_metadata() + get_partially_validated = partial(self.storage.get_transaction, allow_partially_valid=True) # at least one child must be checkpoint validated - for child_tx in map(self.storage.get_transaction, meta.children): + for child_tx in map(get_partially_validated, meta.children): if child_tx.get_metadata().validation.is_checkpoint(): return raise InvalidNewTransaction(f'Invalid new transaction {self.hash_hex}: expected to reach a checkpoint but ' diff --git a/tests/resources/transaction/test_tx.py b/tests/resources/transaction/test_tx.py index c1b981f95..344a5b71c 100644 --- a/tests/resources/transaction/test_tx.py +++ b/tests/resources/transaction/test_tx.py @@ -67,7 +67,6 @@ def test_get_one_known_tx(self): # Tx tesnet 0033784bc8443ba851fd88d81c6f06774ae529f25c1fa8f026884ad0a0e98011 # We had a bug with this endpoint in this tx because the token_data from inputs # was being copied from the output - # First add needed data on storage tx_hex = ('0001020306001c382847d8440d05da95420bee2ebeb32bc437f82a9ae47b0745c8a29a7b0d007231eee3cb6160d95172' 'a409d634d0866eafc8775f5729fff6a61e7850aba500f4dd53f84f1f0091125250b044e49023fbbd0f74f6093cdd2226' @@ -181,7 +180,6 @@ def test_get_one_known_tx_with_authority(self): # Tx tesnet 00005f234469407614bf0abedec8f722bb5e534949ad37650f6077c899741ed7 # We had a bug with this endpoint in this tx because the token_data from inputs # was not considering authority mask - # First add needed data on storage tx_hex = ('0001010202000023b318c91dcfd4b967b205dc938f9f5e2fd5114256caacfb8f6dd13db330000023b318c91dcfd4b967b20' '5dc938f9f5e2fd5114256caacfb8f6dd13db33000006946304402200f7de9e866fbc2d600d6a46eb620fa2d72c9bf032250' @@ -495,6 +493,37 @@ def test_negative_timestamp(self): data = response.json_value() self.assertFalse(data['success']) + @inlineCallbacks + def test_partially_validated_not_found(self): + # Tx tesnet 0033784bc8443ba851fd88d81c6f06774ae529f25c1fa8f026884ad0a0e98011 + # We had a bug with this endpoint in this tx because the token_data from inputs + # was being copied from the output + + # First add needed data on storage + tx_hex = ('0001020306001c382847d8440d05da95420bee2ebeb32bc437f82a9ae47b0745c8a29a7b0d007231eee3cb6160d95172' + 'a409d634d0866eafc8775f5729fff6a61e7850aba500f4dd53f84f1f0091125250b044e49023fbbd0f74f6093cdd2226' + 'fdff3e09a101006946304402205dcbb7956d95b0e123954160d369e64bca7b176e1eb136e2dae5b95e46741509022072' + '6f99a363e8a4d79963492f4359c7589667eb0f45af7effe0dd4e51fbb5543d210288c10b8b1186b8c5f6bc05855590a6' + '522af35f269ddfdb8df39426a01ca9d2dd003d3c40fb04737e1a2a848cfd2592490a71cd0248b9e7d6a626f45dec8697' + '5b00006a4730450221008741dff52d97ce5f084518e1f4cac6bd98abdc88b98e6b18d6a8666fadac05f0022068951306' + '19eaf5433526e4803187c0aa08a8b1c46d9dc4ffaa89406fb2d4940c2102dd29eaadbb21a4de015d1812d5c0ec63cb8e' + 'e921e28580b6a9f8ff08db168c0e0096fb9b1a9e5fc34a9750bcccc746564c2b73f6defa381e130d9a4ea38cb1d80000' + '6a473045022100cb6b8abfb958d4029b0e6a89c828b65357456d20b8e6a8e42ad6d9a780fcddc4022035a8a46248b9c5' + '20b0205aa99ec5c390b40ae97a0b3ccc6e68e835ce5bde972a210306f7fdc08703152348484768fc7b85af900860a3d6' + 'fa85343524150d0370770b0000000100001976a914b9987a3866a7c26225c57a62b14e901377e2f9e288ac0000000200' + '001976a914b9987a3866a7c26225c57a62b14e901377e2f9e288ac0000000301001f0460b5a2b06f76a914b9987a3866' + 'a7c26225c57a62b14e901377e2f9e288ac0000006001001976a914b9987a3866a7c26225c57a62b14e901377e2f9e288' + 'ac0000000402001976a914b9987a3866a7c26225c57a62b14e901377e2f9e288ac000002b602001976a91479ae26cf2f' + '2dc703120a77192fc16eda9ed22e1b88ac40200000218def416095b08602003d3c40fb04737e1a2a848cfd2592490a71cd' + '0248b9e7d6a626f45dec86975b00f4dd53f84f1f0091125250b044e49023fbbd0f74f6093cdd2226fdff3e09a1000002be') + tx = Transaction.create_from_struct(bytes.fromhex(tx_hex), self.manager.tx_storage) + tx.mark_partially_validated() + self.manager.tx_storage.save_transaction(tx) + + response = yield self.web.get("transaction", {b'id': bytes(tx.hash_hex, 'utf-8')}) + data = response.json_value() + self.assertFalse(data['success']) + class SyncV1TransactionTest(unittest.SyncV1Params, BaseTransactionTest): __test__ = True From 5bd7e0949b65172c115cdaf469383f8482ecd4db Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 2 Feb 2023 13:42:55 -0300 Subject: [PATCH 4/9] review changes 6: simple stuff --- hathor/p2p/node_sync_v2.py | 5 ++++- hathor/transaction/transaction_metadata.py | 3 +++ tests/p2p/test_sync.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hathor/p2p/node_sync_v2.py b/hathor/p2p/node_sync_v2.py index d0f97f7ef..6f5dbce6a 100644 --- a/hathor/p2p/node_sync_v2.py +++ b/hathor/p2p/node_sync_v2.py @@ -157,8 +157,11 @@ def __init__(self, protocol: 'HathorProtocol', sync_checkpoints: SyncCheckpoint, # Whether we propagate transactions or not self._is_relaying = False - # Initial value + # This stores the known height of the block we're currently downloading, we know the height before we can + # calculate it because of checkpoints self._blk_height: Optional[int] = None + + # This stores the final height that we expect the last "get blocks" stream to end on self._blk_end_height: Optional[int] = None def get_status(self) -> Dict[str, Any]: diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 1ba8bc11e..aa82b679a 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -355,6 +355,9 @@ def clone(self) -> 'TransactionMetadata': def get_soft_height(self) -> int: """ Returns the soft-height, which is either the soft_height or height metadata. + + The "soft height" is a basically a height preview, this method will always return either the "preview" or the + "actual" height (which when set will erase the soft_height). """ if self.soft_height is not None: return self.soft_height diff --git a/tests/p2p/test_sync.py b/tests/p2p/test_sync.py index b9b04a5b4..7f72dc5a2 100644 --- a/tests/p2p/test_sync.py +++ b/tests/p2p/test_sync.py @@ -531,11 +531,13 @@ def test_tx_propagation_nat_peers(self): self.assertEqual(self.manager1.tx_storage.latest_timestamp, self.manager2.tx_storage.latest_timestamp) self.assertEqual(node_sync1.peer_height, node_sync1.synced_height) self.assertEqual(node_sync1.peer_height, self.manager1.tx_storage.get_height_best_block()) + self.assertConsensusEqual(self.manager1, self.manager2) node_sync2 = self.conn2.proto1.state.sync_manager self.assertEqual(self.manager2.tx_storage.latest_timestamp, self.manager3.tx_storage.latest_timestamp) self.assertEqual(node_sync2.peer_height, node_sync2.synced_height) self.assertEqual(node_sync2.peer_height, self.manager2.tx_storage.get_height_best_block()) + self.assertConsensusEqual(self.manager2, self.manager3) def test_block_sync_new_blocks_and_txs(self): self._add_new_blocks(25) From 9a2ea9934a33adec6f5a7d0a2d89db467f265a72 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 2 Feb 2023 13:43:44 -0300 Subject: [PATCH 5/9] XXX: this commit should be reverted (and uncommented) on the next branch --- hathor/transaction/block.py | 8 -------- tests/tx/test_tx.py | 13 ------------- 2 files changed, 21 deletions(-) diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 5c0fdabd0..aca67f655 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -288,14 +288,6 @@ def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: # XXX: it's fine to use `in` with NamedTuples if Checkpoint(meta.get_soft_height(), self.hash) in checkpoints: return - # otherwise at least one child must be checkpoint validated - # XXX: because we decided to remove "partial" children metadata, this won't work, we could add a new metadata - # field to track this - # for child_tx in map(self.storage.get_transaction, meta.children): - # if child_tx.get_metadata().validation.is_checkpoint(): - # return - # raise CheckpointError(f'Invalid new block {self.hash_hex}: expected to reach a checkpoint but none of ' - # 'its children is checkpoint-valid and its hash does not match any checkpoint') def verify_weight(self) -> None: """Validate minimum block difficulty.""" diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 38a5ac514..a7bfdbd7e 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -128,19 +128,6 @@ def test_checkpoint_validation(self): self.assertTrue(manager2.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False)) self.assertEqual(block2.get_metadata().validation, ValidationState.CHECKPOINT) - # XXX: this stopped failing because we removed the partial children metadata which simplified the validation - # and the expected expection is not raised anymore - # # otherwise it should fail if it's not in the checkpoints - # manager3 = self.create_peer('testnet', checkpoints=[]) - # del block2._metadata - # block2.storage = manager3.tx_storage - # block2.set_height(block2_height) - # # failing silently should not raise an exception, but should return False - # self.assertFalse(manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=True)) - # # otherwise it should raise the correct exception - # with self.assertRaises(InvalidNewTransaction): - # manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False) - def test_script(self): genesis_block = self.genesis_blocks[0] From 573ec48f8773d199c5022977f513bf5730eafeee Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Wed, 1 Mar 2023 13:03:10 -0300 Subject: [PATCH 6/9] Revert "XXX: this commit should be reverted (and uncommented) on the next branch" This reverts commit 1b73c52e0b99432ca5406c11d3f5bc21f312e79e. --- hathor/transaction/block.py | 8 ++++++++ tests/tx/test_tx.py | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index aca67f655..5c0fdabd0 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -288,6 +288,14 @@ def verify_checkpoint(self, checkpoints: List[Checkpoint]) -> None: # XXX: it's fine to use `in` with NamedTuples if Checkpoint(meta.get_soft_height(), self.hash) in checkpoints: return + # otherwise at least one child must be checkpoint validated + # XXX: because we decided to remove "partial" children metadata, this won't work, we could add a new metadata + # field to track this + # for child_tx in map(self.storage.get_transaction, meta.children): + # if child_tx.get_metadata().validation.is_checkpoint(): + # return + # raise CheckpointError(f'Invalid new block {self.hash_hex}: expected to reach a checkpoint but none of ' + # 'its children is checkpoint-valid and its hash does not match any checkpoint') def verify_weight(self) -> None: """Validate minimum block difficulty.""" diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index a7bfdbd7e..38a5ac514 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -128,6 +128,19 @@ def test_checkpoint_validation(self): self.assertTrue(manager2.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False)) self.assertEqual(block2.get_metadata().validation, ValidationState.CHECKPOINT) + # XXX: this stopped failing because we removed the partial children metadata which simplified the validation + # and the expected expection is not raised anymore + # # otherwise it should fail if it's not in the checkpoints + # manager3 = self.create_peer('testnet', checkpoints=[]) + # del block2._metadata + # block2.storage = manager3.tx_storage + # block2.set_height(block2_height) + # # failing silently should not raise an exception, but should return False + # self.assertFalse(manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=True)) + # # otherwise it should raise the correct exception + # with self.assertRaises(InvalidNewTransaction): + # manager3.on_new_tx(block2, sync_checkpoints=True, partial=True, fails_silently=False) + def test_script(self): genesis_block = self.genesis_blocks[0] From c52245abd94e196de43b15cd07c0b4977f172b9a Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 6 Jan 2023 13:32:29 -0300 Subject: [PATCH 7/9] WIP: remove STREAM_BECAME_VOIDED code There's a few tests failing occasionally though. --- hathor/p2p/node_sync_v2.py | 33 ++++++++++++++++-------------- tests/simulation/test_simulator.py | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/hathor/p2p/node_sync_v2.py b/hathor/p2p/node_sync_v2.py index 6f5dbce6a..7bd222186 100644 --- a/hathor/p2p/node_sync_v2.py +++ b/hathor/p2p/node_sync_v2.py @@ -56,7 +56,6 @@ class StreamEnd(IntFlag): END_HASH_REACHED = 0 NO_MORE_BLOCKS = 1 LIMIT_EXCEEDED = 2 - STREAM_BECAME_VOIDED = 3 # this will happen when the current chain becomes voided while it is being sent def __str__(self): if self is StreamEnd.END_HASH_REACHED: @@ -65,8 +64,6 @@ def __str__(self): return 'end of blocks, no more blocks to download from this peer' elif self is StreamEnd.LIMIT_EXCEEDED: return 'streaming limit exceeded' - elif self is StreamEnd.STREAM_BECAME_VOIDED: - return 'streamed block chain became voided' else: raise ValueError(f'invalid StreamEnd value: {self.value}') @@ -654,8 +651,22 @@ def handle_get_next_blocks(self, payload: str) -> None: def send_next_blocks(self, start_hash: bytes, end_hash: bytes) -> None: self.log.debug('start GET-NEXT-BLOCKS stream response') # XXX If I don't have this block it will raise TransactionDoesNotExist error. Should I handle this? - blk = self.tx_storage.get_transaction(start_hash) - assert isinstance(blk, Block) + try: + blk = self.tx_storage.get_transaction(start_hash) + assert isinstance(blk, Block) + except TransactionDoesNotExist: + # In case the tx does not exist we send a NOT-FOUND message + self.send_message(ProtocolMessages.NOT_FOUND, start_hash.hex()) + return + if blk.get_metadata().voided_by: + # XXX: using NOT_FOUND for when it is voided because externally it makes sense to behave as if voided + # blocks/transactions don't exist + self.log.debug('requested start block is voided', start_hash=start_hash.hex()) + self.send_message(ProtocolMessages.NOT_FOUND, start_hash.hex()) + return + if not self.tx_storage.transaction_exists(end_hash): + self.send_message(ProtocolMessages.NOT_FOUND, end_hash.hex()) + return self.blockchain_streaming = BlockchainStreaming(self, blk, end_hash) self.blockchain_streaming.start() @@ -1208,6 +1219,8 @@ def send_next(self) -> None: cur = self.current_block assert cur is not None assert cur.hash is not None + assert not (vby := cur.get_metadata().voided_by), f'failed to send {cur.hash_hex} because it is voided: ' \ + f'{", ".join(i.hex() for i in vby)}; reverse={self.reverse}' if cur.hash == self.end_hash: # only send the last when not reverse @@ -1227,11 +1240,6 @@ def send_next(self) -> None: self.node_sync.send_blocks_end(StreamEnd.LIMIT_EXCEEDED) return - if cur.get_metadata().voided_by: - self.stop() - self.node_sync.send_blocks_end(StreamEnd.STREAM_BECAME_VOIDED) - return - self.counter += 1 self.log.debug('send next block', blk_id=cur.hash.hex()) @@ -1301,11 +1309,6 @@ def send_next(self) -> None: assert cur.hash is not None cur_metadata = cur.get_metadata() - if cur_metadata.voided_by: - self.stop() - self.node_sync.send_blocks_end(StreamEnd.STREAM_BECAME_VOIDED) - return - assert cur_metadata.first_block is not None first_blk_meta = self.storage.get_metadata(cur_metadata.first_block) assert first_blk_meta is not None diff --git a/tests/simulation/test_simulator.py b/tests/simulation/test_simulator.py index fefca39cf..a03232d61 100644 --- a/tests/simulation/test_simulator.py +++ b/tests/simulation/test_simulator.py @@ -84,7 +84,7 @@ def test_many_miners_since_beginning(self): for miner in miners: miner.stop() - self.simulator.run(15) + self.simulator.run(300) for node in nodes[1:]: self.assertTipsEqual(nodes[0], node) From 5df47bd8008b9810ad441cebe0c9f3787afe18f2 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 12 Jan 2023 19:19:27 -0300 Subject: [PATCH 8/9] [skip ci] WIP --- hathor/conf/mainnet.py | 2 ++ hathor/conf/settings.py | 2 +- hathor/indexes/deps_index.py | 3 ++- hathor/indexes/manager.py | 2 +- hathor/p2p/node_sync_v2.py | 2 +- hathor/p2p/sync_checkpoints.py | 1 + 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hathor/conf/mainnet.py b/hathor/conf/mainnet.py index 67f418c0a..4722a35cb 100644 --- a/hathor/conf/mainnet.py +++ b/hathor/conf/mainnet.py @@ -62,6 +62,8 @@ cp(2_700_000, bytes.fromhex('00000000000000000cf3a35ab01a2281024ca4ca7871f5a6d67106eb36151038')), cp(2_800_000, bytes.fromhex('000000000000000004439733fd419a8a747e8afe2f89348a17c1fac24538a63c')), cp(2_900_000, bytes.fromhex('0000000000000000090cbd5a7958c82a2b969103001d92334f287dadcf3e01bc')), + cp(3_000_000, bytes.fromhex('000000000000000013c9086f4ce441f5db5de55a5e235f4f7f1ef223aedfe2db')), + cp(3_100_000, bytes.fromhex('00000000000000000d226a5998ffc65af89b1226126b1af1f8d0712a5301c775')), ], SOFT_VOIDED_TX_IDS=list(map(bytes.fromhex, [ '0000000012a922a6887497bed9c41e5ed7dc7213cae107db295602168266cd02', diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index d87db532e..2ce201b15 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -256,7 +256,7 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: PEER_MAX_CONNECTIONS: int = 125 # Maximum period without receiving any messages from ther peer (in seconds). - PEER_IDLE_TIMEOUT: int = 60 + PEER_IDLE_TIMEOUT: int = 600 # Filepath of ca certificate file to generate connection certificates CA_FILEPATH: str = os.path.join(os.path.dirname(__file__), '../p2p/ca.crt') diff --git a/hathor/indexes/deps_index.py b/hathor/indexes/deps_index.py index fbea6a67c..b47ade2e8 100644 --- a/hathor/indexes/deps_index.py +++ b/hathor/indexes/deps_index.py @@ -109,7 +109,8 @@ def init_loop_step(self, tx: BaseTransaction) -> None: tx_meta = tx.get_metadata() if tx_meta.voided_by: return - self.add_tx(tx, partial=False) + # self.add_tx(tx, partial=False) + self.add_tx(tx) def update(self, tx: BaseTransaction) -> None: assert tx.hash is not None diff --git a/hathor/indexes/manager.py b/hathor/indexes/manager.py index de0c2fd32..78f6f835c 100644 --- a/hathor/indexes/manager.py +++ b/hathor/indexes/manager.py @@ -186,7 +186,7 @@ def _manually_initialize(self, tx_storage: 'TransactionStorage') -> None: self.log.debug('indexes init') if indexes_to_init: - tx_iter = progress(tx_storage.topological_iterator(), log=self.log, total=tx_storage.get_vertices_count()) + tx_iter = progress(tx_storage._topological_sort_dfs(), log=self.log, total=tx_storage.get_vertices_count()) else: tx_iter = iter([]) for tx in tx_iter: diff --git a/hathor/p2p/node_sync_v2.py b/hathor/p2p/node_sync_v2.py index 7bd222186..50938fa76 100644 --- a/hathor/p2p/node_sync_v2.py +++ b/hathor/p2p/node_sync_v2.py @@ -1130,7 +1130,7 @@ def handle_data(self, payload: str) -> None: # --------------------------------- -DEAFAULT_STREAMING_LIMIT = 1_000 +DEAFAULT_STREAMING_LIMIT = 10_000 @implementer(IPushProducer) diff --git a/hathor/p2p/sync_checkpoints.py b/hathor/p2p/sync_checkpoints.py index 25dc6e342..3723edb6f 100644 --- a/hathor/p2p/sync_checkpoints.py +++ b/hathor/p2p/sync_checkpoints.py @@ -226,6 +226,7 @@ def _run_sync(self): sync_interval = self._get_next_sync_interval() if not sync_interval: + self.should_skip_sync_tx = False self.log.debug('no checkpoints to sync anymore') return From 3a63d6911c21fb3ff0608e27f7863e231c65cd71 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 2 Mar 2023 13:23:45 -0300 Subject: [PATCH 9/9] WIP2 --- hathor/indexes/address_index.py | 10 +++ hathor/indexes/base_index.py | 77 ++++++++++++++++++- hathor/indexes/deps_index.py | 12 ++- hathor/indexes/height_index.py | 11 ++- hathor/indexes/info_index.py | 13 +++- hathor/indexes/manager.py | 24 +++--- hathor/indexes/memory_timestamp_index.py | 3 +- hathor/indexes/memory_tips_index.py | 3 +- hathor/indexes/mempool_tips_index.py | 11 ++- hathor/indexes/partial_rocksdb_tips_index.py | 11 ++- hathor/indexes/rocksdb_timestamp_index.py | 10 ++- hathor/indexes/timestamp_index.py | 34 +++++++- hathor/indexes/tips_index.py | 34 +++++++- hathor/indexes/tokens_index.py | 11 ++- hathor/indexes/utxo_index.py | 11 ++- hathor/transaction/storage/memory_storage.py | 2 +- hathor/transaction/storage/rocksdb_storage.py | 8 +- .../storage/transaction_storage.py | 4 +- hathor/transaction/transaction_metadata.py | 20 ++++- tests/others/test_init_manager.py | 4 +- 20 files changed, 278 insertions(+), 35 deletions(-) diff --git a/hathor/indexes/address_index.py b/hathor/indexes/address_index.py index b8925b927..fca7ce4ca 100644 --- a/hathor/indexes/address_index.py +++ b/hathor/indexes/address_index.py @@ -17,6 +17,7 @@ from structlog import get_logger +from hathor.indexes.base_index import Scope from hathor.indexes.tx_group_index import TxGroupIndex from hathor.pubsub import HathorEvents from hathor.transaction import BaseTransaction @@ -26,12 +27,21 @@ logger = get_logger() +SCOPE = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, +) + class AddressIndex(TxGroupIndex[str]): """ Index of inputs/outputs by address """ pubsub: Optional['PubSubManager'] + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: self.add_tx(tx) diff --git a/hathor/indexes/base_index.py b/hathor/indexes/base_index.py index 9503c673f..1edd60dc1 100644 --- a/hathor/indexes/base_index.py +++ b/hathor/indexes/base_index.py @@ -13,12 +13,82 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional from hathor.transaction.base_transaction import BaseTransaction if TYPE_CHECKING: # pragma: no cover from hathor.indexes.manager import IndexesManager + from hathor.transaction.storage import TransactionStorage + + +class Scope(NamedTuple): + """ This class models the scope of transactions that an index is interested in. + + It is used for both selecting the optimal iterator for all the indexes that need to be initialized and for + filtering which transactions are fed to the index. + """ + include_blocks: bool + include_txs: bool + include_voided: bool + # XXX: these have a default value since it should be really rare to have it different + include_partial: bool = False + topological_order: bool = True # if False than ordering doesn't matter + + # XXX: this is used to join the scope of multiple indexes to get an overall scope that includes everything that + # each individual scope needs, the OR operator was chosen because it represents well the operation of keeping + # a property if either A or B needs it + def __or__(self, other): + # XXX: note that this doesn't necessarily have to be OR operations between properties, we want the operations + # that broaden the scope, and not narrow it. + # XXX: in the case of topological_order, we want to keep the "topological" ordering if any of them requires it, + # so it also is an OR operator + return Scope( + include_blocks=self.include_blocks | other.include_blocks, + include_txs=self.include_txs | other.include_txs, + include_voided=self.include_voided | other.include_voided, + include_partial=self.include_partial | other.include_partial, + topological_order=self.topological_order | other.topological_order, + ) + + def matches(self, tx: BaseTransaction) -> bool: + """ Check if a transaction matches this scope, True means the index is interested in this transaction. + """ + if tx.is_block and not self.include_blocks: + return False + if tx.is_transaction and not self.include_txs: + return False + tx_meta = tx.get_metadata() + if tx_meta.voided_by and not self.include_voided: + return False + if not tx_meta.validation.is_fully_connected() and not self.include_partial: + return False + # XXX: self.topologial_order doesn't affect self.match() + # passed all checks + return True + + def get_iterator(self, tx_storage: 'TransactionStorage') -> Iterator[BaseTransaction]: + iterator: Iterator[BaseTransaction] + # XXX: this is to mark if the chosen iterator will yield partial transactions + iterator_covers_partial: bool + if self.topological_order: + iterator = tx_storage.topological_iterator() + iterator_covers_partial = False + else: + iterator = tx_storage.get_all_transactions() + iterator_covers_partial = True + for tx in iterator: + if self.matches(tx): + yield tx + if self.include_partial and not iterator_covers_partial: + # if partial transactions are needed and were not already covered, we use get_all_transactions, which + # includes partial transactions, to yield them, skipping all that aren't partial + for tx in tx_storage.get_all_transactions(): + tx_meta = tx.get_metadata() + if tx_meta.validation.is_fully_connected(): + continue + if self.matches(tx): + yield tx class BaseIndex(ABC): @@ -35,6 +105,11 @@ def init_start(self, indexes_manager: 'IndexesManager') -> None: """ pass + @abstractmethod + def get_scope(self) -> Scope: + """ Returns the scope of interest of this index, whether the scope is configurable is up to the index.""" + raise NotImplementedError + @abstractmethod def get_db_name(self) -> Optional[str]: """ The returned string is used to generate the relevant attributes for storing an indexe's state in the db. diff --git a/hathor/indexes/deps_index.py b/hathor/indexes/deps_index.py index b47ade2e8..125d525b3 100644 --- a/hathor/indexes/deps_index.py +++ b/hathor/indexes/deps_index.py @@ -15,7 +15,7 @@ from abc import abstractmethod from typing import TYPE_CHECKING, Iterator, List -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction, Block if TYPE_CHECKING: # pragma: no cover @@ -25,6 +25,13 @@ # XXX: this arbitrary height limit must fit in a u32 (4-bytes unsigned), so it can be stored easily on rocksdb INF_HEIGHT: int = 2**32 - 1 +SCOPE = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, + include_partial=True +) + def get_requested_from_height(tx: BaseTransaction) -> int: """Return the height of the block that requested (directly or indirectly) the download of this transaction. @@ -105,6 +112,9 @@ class DepsIndex(BaseIndex): them. """ + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: tx_meta = tx.get_metadata() if tx_meta.voided_by: diff --git a/hathor/indexes/height_index.py b/hathor/indexes/height_index.py index 655fe7e70..fd591951e 100644 --- a/hathor/indexes/height_index.py +++ b/hathor/indexes/height_index.py @@ -15,11 +15,17 @@ from abc import abstractmethod from typing import List, NamedTuple, Optional, Tuple -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction, Block from hathor.transaction.genesis import BLOCK_GENESIS from hathor.util import not_none +SCOPE = Scope( + include_blocks=True, + include_txs=False, + include_voided=True, +) + class IndexEntry(NamedTuple): """Helper named tuple that implementations can use.""" @@ -40,6 +46,9 @@ class HeightIndex(BaseIndex): """Store the block hash for each given height """ + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: if not tx.is_block: return diff --git a/hathor/indexes/info_index.py b/hathor/indexes/info_index.py index 96c200eed..6da5345e9 100644 --- a/hathor/indexes/info_index.py +++ b/hathor/indexes/info_index.py @@ -16,11 +16,19 @@ from structlog import get_logger -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction logger = get_logger() +SCOPE = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, + # XXX: this index doesn't care about the ordering + topological_order=False, +) + class InfoIndex(BaseIndex): """ Index of general information about the storage @@ -30,6 +38,9 @@ def init_loop_step(self, tx: BaseTransaction) -> None: self.update_timestamps(tx) self.update_counts(tx) + def get_scope(self) -> Scope: + return SCOPE + @abstractmethod def update_timestamps(self, tx: BaseTransaction) -> None: raise NotImplementedError diff --git a/hathor/indexes/manager.py b/hathor/indexes/manager.py index 78f6f835c..69db037cb 100644 --- a/hathor/indexes/manager.py +++ b/hathor/indexes/manager.py @@ -307,13 +307,13 @@ def __init__(self) -> None: from hathor.indexes.memory_tips_index import MemoryTipsIndex self.info = MemoryInfoIndex() - self.all_tips = MemoryTipsIndex() - self.block_tips = MemoryTipsIndex() - self.tx_tips = MemoryTipsIndex() + self.all_tips = MemoryTipsIndex(all=True) + self.block_tips = MemoryTipsIndex(blocks=True) + self.tx_tips = MemoryTipsIndex(txs=True) - self.sorted_all = MemoryTimestampIndex() - self.sorted_blocks = MemoryTimestampIndex() - self.sorted_txs = MemoryTimestampIndex() + self.sorted_all = MemoryTimestampIndex(all=True) + self.sorted_blocks = MemoryTimestampIndex(blocks=True) + self.sorted_txs = MemoryTimestampIndex(txs=True) self.addresses = None self.tokens = None @@ -362,13 +362,13 @@ def __init__(self, db: 'rocksdb.DB') -> None: self.info = RocksDBInfoIndex(self._db) self.height = RocksDBHeightIndex(self._db) - self.all_tips = PartialRocksDBTipsIndex(self._db, 'all') - self.block_tips = PartialRocksDBTipsIndex(self._db, 'blocks') - self.tx_tips = PartialRocksDBTipsIndex(self._db, 'txs') + self.all_tips = PartialRocksDBTipsIndex(self._db, all=True) + self.block_tips = PartialRocksDBTipsIndex(self._db, blocks=True) + self.tx_tips = PartialRocksDBTipsIndex(self._db, txs=True) - self.sorted_all = RocksDBTimestampIndex(self._db, 'all') - self.sorted_blocks = RocksDBTimestampIndex(self._db, 'blocks') - self.sorted_txs = RocksDBTimestampIndex(self._db, 'txs') + self.sorted_all = RocksDBTimestampIndex(self._db, all=True) + self.sorted_blocks = RocksDBTimestampIndex(self._db, blocks=True) + self.sorted_txs = RocksDBTimestampIndex(self._db, txs=True) self.addresses = None self.tokens = None diff --git a/hathor/indexes/memory_timestamp_index.py b/hathor/indexes/memory_timestamp_index.py index d61e32677..d749a4924 100644 --- a/hathor/indexes/memory_timestamp_index.py +++ b/hathor/indexes/memory_timestamp_index.py @@ -35,7 +35,8 @@ class MemoryTimestampIndex(TimestampIndex): _index: 'SortedKeyList[TransactionIndexElement]' - def __init__(self) -> None: + def __init__(self, *, txs: bool = False, blocks: bool = False, all: bool = False): + super().__init__(txs=txs, blocks=blocks, all=all) self.log = logger.new() self.force_clear() diff --git a/hathor/indexes/memory_tips_index.py b/hathor/indexes/memory_tips_index.py index 46ff14f61..12134ba21 100644 --- a/hathor/indexes/memory_tips_index.py +++ b/hathor/indexes/memory_tips_index.py @@ -47,7 +47,8 @@ class MemoryTipsIndex(TipsIndex): # It is useful because the interval tree allows access only by the interval. tx_last_interval: Dict[bytes, Interval] - def __init__(self) -> None: + def __init__(self, *, txs: bool = False, blocks: bool = False, all: bool = False): + super().__init__(txs=txs, blocks=blocks, all=all) self.log = logger.new() self.tree = IntervalTree() self.tx_last_interval = {} diff --git a/hathor/indexes/mempool_tips_index.py b/hathor/indexes/mempool_tips_index.py index 6e7ea8485..59545cae7 100644 --- a/hathor/indexes/mempool_tips_index.py +++ b/hathor/indexes/mempool_tips_index.py @@ -19,17 +19,26 @@ import structlog -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction, Transaction from hathor.util import not_none if TYPE_CHECKING: # pragma: no cover from hathor.transaction.storage import TransactionStorage +SCOPE = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, +) + class MempoolTipsIndex(BaseIndex): """Index to access the tips of the mempool transactions, which haven't been confirmed by a block.""" + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: self.update(tx) diff --git a/hathor/indexes/partial_rocksdb_tips_index.py b/hathor/indexes/partial_rocksdb_tips_index.py index 8c59f3cb3..3bb8971fb 100644 --- a/hathor/indexes/partial_rocksdb_tips_index.py +++ b/hathor/indexes/partial_rocksdb_tips_index.py @@ -111,8 +111,15 @@ class PartialRocksDBTipsIndex(MemoryTipsIndex, RocksDBIndexUtils): # It is useful because the interval tree allows access only by the interval. tx_last_interval: Dict[bytes, Interval] - def __init__(self, db: 'rocksdb.DB', name: str) -> None: - MemoryTipsIndex.__init__(self) + def __init__(self, db: 'rocksdb.DB', *, txs: bool = False, blocks: bool = False, all: bool = False): + MemoryTipsIndex.__init__(self, txs=txs, blocks=blocks, all=all) + name: str + if txs: + name = 'txs' + elif blocks: + name = 'blocks' + elif all: + name = 'all' self.log = logger.new() # XXX: override MemoryTipsIndex logger so it shows the correct module RocksDBIndexUtils.__init__(self, db, f'tips-{name}'.encode()) self._name = name diff --git a/hathor/indexes/rocksdb_timestamp_index.py b/hathor/indexes/rocksdb_timestamp_index.py index a530b2453..d72ada0f4 100644 --- a/hathor/indexes/rocksdb_timestamp_index.py +++ b/hathor/indexes/rocksdb_timestamp_index.py @@ -38,7 +38,15 @@ class RocksDBTimestampIndex(TimestampIndex, RocksDBIndexUtils): It works nicely because rocksdb uses a tree sorted by key under the hood. """ - def __init__(self, db: 'rocksdb.DB', name: str) -> None: + def __init__(self, db: 'rocksdb.DB', *, txs: bool = False, blocks: bool = False, all: bool = False): + TimestampIndex.__init__(self, txs=txs, blocks=blocks, all=all) + name: str + if txs: + name = 'txs' + elif blocks: + name = 'blocks' + elif all: + name = 'all' self.log = logger.new() RocksDBIndexUtils.__init__(self, db, f'timestamp-sorted-{name}'.encode()) self._name = name diff --git a/hathor/indexes/timestamp_index.py b/hathor/indexes/timestamp_index.py index e2dea623e..bea994083 100644 --- a/hathor/indexes/timestamp_index.py +++ b/hathor/indexes/timestamp_index.py @@ -17,11 +17,29 @@ from structlog import get_logger -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction logger = get_logger() +SCOPE_ALL = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, +) + +SCOPE_TXS = Scope( + include_blocks=False, + include_txs=True, + include_voided=False, +) + +SCOPE_BLOCKS = Scope( + include_blocks=True, + include_txs=False, + include_voided=True, +) + class RangeIdx(NamedTuple): timestamp: int @@ -32,6 +50,20 @@ class TimestampIndex(BaseIndex): """ Index of transactions sorted by their timestamps. """ + def __init__(self, *, txs: bool = False, blocks: bool = False, all: bool = False): + if sum([txs, blocks, all]) != 1: + raise TypeError('Exactly one of these: txs, blocks, all, must be set to True') + self._scope: Scope + if txs: + self._scope = SCOPE_TXS + elif blocks: + self._scope = SCOPE_BLOCKS + elif all: + self._scope = SCOPE_ALL + + def get_scope(self) -> Scope: + return self._scope + def init_loop_step(self, tx: BaseTransaction) -> None: self.add_tx(tx) diff --git a/hathor/indexes/tips_index.py b/hathor/indexes/tips_index.py index 1b85cc523..b1cedf593 100644 --- a/hathor/indexes/tips_index.py +++ b/hathor/indexes/tips_index.py @@ -18,11 +18,29 @@ from intervaltree import Interval from structlog import get_logger -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction logger = get_logger() +SCOPE_ALL = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, +) + +SCOPE_TXS = Scope( + include_blocks=False, + include_txs=True, + include_voided=False, +) + +SCOPE_BLOCKS = Scope( + include_blocks=True, + include_txs=False, + include_voided=True, +) + class TipsIndex(BaseIndex): """ Use an interval tree to quick get the tips at a given timestamp. @@ -38,6 +56,20 @@ class TipsIndex(BaseIndex): TODO Use an interval tree stored in disk, possibly using a B-tree. """ + def __init__(self, *, txs: bool = False, blocks: bool = False, all: bool = False): + if sum([txs, blocks, all]) != 1: + raise TypeError('Exactly one of these: txs, blocks, all, must be set to True') + self._scope: Scope + if txs: + self._scope = SCOPE_TXS + elif blocks: + self._scope = SCOPE_BLOCKS + elif all: + self._scope = SCOPE_ALL + + def get_scope(self) -> Scope: + return self._scope + @abstractmethod def add_tx(self, tx: BaseTransaction) -> bool: """ Add a new transaction to the index diff --git a/hathor/indexes/tokens_index.py b/hathor/indexes/tokens_index.py index 27c38fa1d..5a5228927 100644 --- a/hathor/indexes/tokens_index.py +++ b/hathor/indexes/tokens_index.py @@ -15,9 +15,15 @@ from abc import ABC, abstractmethod from typing import Iterator, List, NamedTuple, Optional, Tuple -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction +SCOPE = Scope( + include_blocks=False, + include_txs=True, + include_voided=True, +) + class TokenUtxoInfo(NamedTuple): tx_hash: bytes @@ -62,6 +68,9 @@ class TokensIndex(BaseIndex): """ Index of tokens by token uid """ + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: tx_meta = tx.get_metadata() if tx_meta.voided_by: diff --git a/hathor/indexes/utxo_index.py b/hathor/indexes/utxo_index.py index 3c788bca9..b559cffe0 100644 --- a/hathor/indexes/utxo_index.py +++ b/hathor/indexes/utxo_index.py @@ -19,7 +19,7 @@ from structlog import get_logger from hathor.conf import HathorSettings -from hathor.indexes.base_index import BaseIndex +from hathor.indexes.base_index import BaseIndex, Scope from hathor.transaction import BaseTransaction, TxOutput from hathor.transaction.scripts import parse_address_script from hathor.util import sorted_merger @@ -27,6 +27,12 @@ logger = get_logger() settings = HathorSettings() +SCOPE = Scope( + include_blocks=True, + include_txs=True, + include_voided=True, +) + @dataclass(frozen=True) class UtxoIndexItem: @@ -104,6 +110,9 @@ def __init__(self): # interface methods provided by the base class + def get_scope(self) -> Scope: + return SCOPE + def init_loop_step(self, tx: BaseTransaction) -> None: self.update(tx) diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index 21304bda7..aef748ddb 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -92,7 +92,7 @@ def _get_transaction(self, hash_bytes: bytes, *, allow_partially_valid: bool = F else: raise TransactionDoesNotExist(hash_bytes.hex()) - def get_all_transactions(self) -> Iterator[BaseTransaction]: + def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: for tx in self.transactions.values(): tx = self._clone(tx) if tx.hash in self.metadata: diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index a15197ada..e9f8be389 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf import HathorSettings from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager from hathor.storage import RocksDBStorage from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionPartiallyValidatedError @@ -29,6 +30,7 @@ from hathor.transaction import BaseTransaction, TransactionMetadata logger = get_logger() +settings = HathorSettings() _DB_NAME = 'data_v2.db' _CF_NAME_TX = b'tx' @@ -150,7 +152,7 @@ def _get_tx(self, hash_bytes: bytes, tx_data: bytes) -> 'BaseTransaction': self._save_to_weakref(tx) return tx - def get_all_transactions(self) -> Iterator['BaseTransaction']: + def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: tx: Optional['BaseTransaction'] items = self._db.iteritems(self._cf_tx) @@ -167,6 +169,10 @@ def get_all_transactions(self) -> Iterator['BaseTransaction']: tx = self._get_tx(hash_bytes, tx_data) assert tx is not None + if not include_partial: + assert tx._metadata is not None + if tx._metadata.voided_by and settings.PARTIALLY_VALIDATED_ID in tx._metadata.voided_by: + continue yield tx def is_empty(self) -> bool: diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 5aabfdfda..ee96ec6ef 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -504,7 +504,7 @@ def get_metadata(self, hash_bytes: bytes, *, allow_partially_valid: bool = False return None @abstractmethod - def get_all_transactions(self) -> Iterator[BaseTransaction]: + def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: # TODO: verify the following claim: """Return all transactions that are not blocks. """ @@ -1147,7 +1147,7 @@ def _topological_sort_timestamp_index(self) -> Iterator[BaseTransaction]: yield from cur_blocks yield from cur_txs - def _topological_sort_metadata(self) -> Iterator[BaseTransaction]: + def _topological_sort_metadata(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: import heapq from dataclasses import dataclass, field diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index aa82b679a..9a15fc9c5 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -103,6 +103,7 @@ class TransactionMetadata: voided_by: Optional[Set[bytes]] received_by: List[int] children: List[bytes] + children_partial: List[bytes] twins: List[bytes] accumulated_weight: float score: float @@ -155,6 +156,10 @@ def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: # Store only the transactions' hash. self.children = [] + # List of partially validated transactions which have this transaction as parent. + # Store only the transactions' hash. + self.children_partial = [] + # Hash of the transactions that are twin to this transaction. # Twin transactions have the same inputs and outputs self.twins = [] @@ -241,7 +246,7 @@ def __eq__(self, other: Any) -> bool: if not isinstance(other, TransactionMetadata): return False for field in ['hash', 'conflict_with', 'voided_by', 'received_by', - 'children', 'accumulated_weight', 'twins', 'score', + 'children', 'children_partial', 'accumulated_weight', 'twins', 'score', 'first_block', 'validation', 'min_height', 'soft_height']: if (getattr(self, field) or None) != (getattr(other, field) or None): return False @@ -262,7 +267,11 @@ def __eq__(self, other: Any) -> bool: return True - def to_json(self) -> Dict[str, Any]: + def to_json(self, *, _extended: bool = False) -> Dict[str, Any]: + """ Serialize data to JSON + + `_extended` is an internal parameter, don't use it directly, use `to_json_extended` instead + """ data: Dict[str, Any] = {} data['hash'] = self.hash and self.hash.hex() data['spent_outputs'] = [] @@ -270,6 +279,8 @@ def to_json(self) -> Dict[str, Any]: data['spent_outputs'].append([idx, [h_bytes.hex() for h_bytes in hashes]]) data['received_by'] = list(self.received_by) data['children'] = [x.hex() for x in self.children] + if not _extended: + data['children_partial'] = [x.hex() for x in self.children_partial] data['conflict_with'] = [x.hex() for x in set(self.conflict_with)] if self.conflict_with else [] data['voided_by'] = [x.hex() for x in self.voided_by] if self.voided_by else [] data['twins'] = [x.hex() for x in self.twins] @@ -287,7 +298,9 @@ def to_json(self) -> Dict[str, Any]: return data def to_json_extended(self, tx_storage: 'TransactionStorage') -> Dict[str, Any]: - data = self.to_json() + """ Serialize data to JSON with attributes to be exposed to the APIs + """ + data = self.to_json(_extended=True) first_block_height: Optional[int] if self.first_block is not None: first_block = tx_storage.get_transaction(self.first_block) @@ -308,6 +321,7 @@ def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata': meta.spent_outputs[idx].append(bytes.fromhex(h_hex)) meta.received_by = list(data['received_by']) meta.children = [bytes.fromhex(h) for h in data['children']] + meta.children_partial = [bytes.fromhex(h) for h in data.get('children_partial', [])] if 'conflict_with' in data and data['conflict_with']: meta.conflict_with = [bytes.fromhex(h) for h in set(data['conflict_with'])] diff --git a/tests/others/test_init_manager.py b/tests/others/test_init_manager.py index 027ab3af2..375f74c32 100644 --- a/tests/others/test_init_manager.py +++ b/tests/others/test_init_manager.py @@ -25,12 +25,12 @@ def __init__(self, *args, **kwargs): def set_first_tx(self, tx: BaseTransaction) -> None: self._first_tx = tx - def get_all_transactions(self) -> Iterator[BaseTransaction]: + def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: skip_hash = None if self._first_tx: yield self._first_tx skip_hash = self._first_tx.hash - for tx in super().get_all_transactions(): + for tx in super().get_all_transactions(include_partial=include_partial): if tx.hash != skip_hash: yield tx