From 7b2dd3cea27e26cb9fc4af6b30844fff0b70fa56 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Thu, 20 Apr 2023 01:51:30 -0500 Subject: [PATCH] refactor(builder): Change HathorManager.__init__ and change tests to use TestBuilder --- hathor/builder/__init__.py | 22 ++ hathor/builder/builder.py | 346 ++++++++++++++++++ hathor/{builder.py => builder/cli_builder.py} | 4 + hathor/manager.py | 15 +- hathor/simulator/simulator.py | 1 + tests/others/test_builder.py | 175 +-------- tests/others/test_cli_builder.py | 175 +++++++++ tests/others/test_init_manager.py | 69 ++-- tests/tx/test_tx_storage.py | 48 +-- tests/unittest.py | 106 +++--- 10 files changed, 672 insertions(+), 289 deletions(-) create mode 100644 hathor/builder/__init__.py create mode 100644 hathor/builder/builder.py rename hathor/{builder.py => builder/cli_builder.py} (99%) create mode 100644 tests/others/test_cli_builder.py diff --git a/hathor/builder/__init__.py b/hathor/builder/__init__.py new file mode 100644 index 000000000..93271b01a --- /dev/null +++ b/hathor/builder/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.builder.builder import BuildArtifacts, Builder +from hathor.builder.cli_builder import CliBuilder + +__all__ = [ + 'BuildArtifacts', + 'Builder', + 'CliBuilder', +] diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py new file mode 100644 index 000000000..d82f503cd --- /dev/null +++ b/hathor/builder/builder.py @@ -0,0 +1,346 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Any, Dict, List, NamedTuple, Optional + +from structlog import get_logger + +from hathor.checkpoint import Checkpoint +from hathor.conf import HathorSettings +from hathor.conf.settings import HathorSettings as HathorSettingsType +from hathor.consensus import ConsensusAlgorithm +from hathor.event import EventManager +from hathor.indexes import IndexesManager +from hathor.manager import HathorManager +from hathor.p2p.peer_id import PeerId +from hathor.pubsub import PubSubManager +from hathor.storage import RocksDBStorage +from hathor.transaction.storage import TransactionMemoryStorage, TransactionRocksDBStorage, TransactionStorage +from hathor.util import Random, Reactor, get_environment_info +from hathor.wallet import BaseWallet, Wallet + +logger = get_logger() + + +class StorageType(Enum): + MEMORY = 'memory' + ROCKSDB = 'rocksdb' + + +class BuildArtifacts(NamedTuple): + """Artifacts created by a builder.""" + settings: HathorSettingsType + rng: Random + reactor: Reactor + manager: HathorManager + pubsub: PubSubManager + consensus: ConsensusAlgorithm + tx_storage: TransactionStorage + indexes: Optional[IndexesManager] + wallet: Optional[BaseWallet] + rocksdb_storage: Optional[RocksDBStorage] + + +class Builder: + """Builder builds the core objects to run a full node. + + Example: + + builder = Builder() + builder.use_memory() + artifacts = builder.build() + """ + def __init__(self) -> None: + self.log = logger.new() + self.artifacts: Optional[BuildArtifacts] = None + + self._settings: HathorSettingsType = HathorSettings() + self._rng: Random = Random() + self._checkpoints: Optional[List[Checkpoint]] = None + self._capabilities: Optional[List[str]] = None + + self._peer_id: Optional[PeerId] = None + self._network: Optional[str] = None + self._cmdline: str = '' + + self._storage_type: Optional[StorageType] = None + self._force_memory_index: bool = False + + self._event_manager: Optional[EventManager] = None + + self._rocksdb_path: Optional[str] = None + self._rocksdb_storage: Optional[RocksDBStorage] = None + + self._tx_storage: Optional[TransactionStorage] = None + + self._reactor: Optional[Reactor] = None + self._pubsub: Optional[PubSubManager] = None + + self._wallet: Optional[BaseWallet] = None + self._wallet_directory: Optional[str] = None + self._wallet_unlock: Optional[bytes] = None + + self._enable_address_index: bool = False + self._enable_tokens_index: bool = False + self._enable_utxo_index: bool = False + + self._enable_sync_v1: Optional[bool] = None + self._enable_sync_v2: Optional[bool] = None + + self._stratum_port: Optional[int] = None + + def build(self) -> BuildArtifacts: + if self.artifacts is not None: + raise ValueError('cannot call build twice') + + settings = self._get_settings() + reactor = self._get_reactor() + pubsub = self._get_or_create_pubsub() + + peer_id = self._get_peer_id() + + soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS) + consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub) + + wallet = self._get_or_create_wallet() + tx_storage = self._get_or_create_tx_storage() + indexes = tx_storage.indexes + assert indexes is not None + + if self._enable_address_index: + indexes.enable_address_index(pubsub) + + if self._enable_tokens_index: + indexes.enable_tokens_index() + + if self._enable_utxo_index: + indexes.enable_utxo_index() + + kwargs: Dict[str, Any] = {} + + if self._enable_sync_v1 is not None: + kwargs['enable_sync_v1'] = self._enable_sync_v1 + + if self._enable_sync_v2 is not None: + kwargs['enable_sync_v2'] = self._enable_sync_v2 + + if self._stratum_port is not None: + kwargs['stratum_port'] = self._stratum_port + + if self._network is None: + raise TypeError('you must set a network') + + if self._event_manager is not None: + kwargs['event_manager'] = self._event_manager + + manager = HathorManager( + reactor, + pubsub=pubsub, + consensus_algorithm=consensus_algorithm, + peer_id=peer_id, + tx_storage=tx_storage, + network=self._network, + wallet=wallet, + rng=self._rng, + checkpoints=self._checkpoints, + capabilities=self._capabilities, + environment_info=get_environment_info(self._cmdline, peer_id.id), + **kwargs + ) + + self.artifacts = BuildArtifacts( + settings=settings, + rng=self._rng, + reactor=reactor, + manager=manager, + pubsub=pubsub, + consensus=consensus_algorithm, + tx_storage=tx_storage, + indexes=indexes, + wallet=wallet, + rocksdb_storage=self._rocksdb_storage, + ) + + return self.artifacts + + def check_if_can_modify(self) -> None: + if self.artifacts is not None: + raise ValueError('cannot modify after build() is called') + + def set_event_manager(self, event_manager: EventManager) -> 'Builder': + self.check_if_can_modify() + self._event_manager = event_manager + return self + + def set_rng(self, rng: Random) -> 'Builder': + self.check_if_can_modify() + self._rng = rng + return self + + def set_checkpoints(self, checkpoints: List[Checkpoint]) -> 'Builder': + self.check_if_can_modify() + self._checkpoints = checkpoints + return self + + def set_capabilities(self, capabilities: List[str]) -> 'Builder': + self.check_if_can_modify() + self._capabilities = capabilities + return self + + def set_peer_id(self, peer_id: PeerId) -> 'Builder': + self.check_if_can_modify() + self._peer_id = peer_id + return self + + def _get_settings(self) -> HathorSettingsType: + return self._settings + + def _get_reactor(self) -> Reactor: + if self._reactor is not None: + return self._reactor + raise ValueError('reactor not set') + + def _get_peer_id(self) -> PeerId: + if self._peer_id is not None: + return self._peer_id + raise ValueError('peer_id not set') + + def _get_or_create_pubsub(self) -> PubSubManager: + if self._pubsub is None: + self._pubsub = PubSubManager(self._get_reactor()) + return self._pubsub + + def _get_or_create_rocksdb_storage(self) -> RocksDBStorage: + assert self._rocksdb_path is not None + if self._rocksdb_storage is None: + self._rocksdb_storage = RocksDBStorage(path=self._rocksdb_path) + return self._rocksdb_storage + + def _get_or_create_tx_storage(self) -> TransactionStorage: + if self._tx_storage is not None: + return self._tx_storage + if self._storage_type == StorageType.MEMORY: + return TransactionMemoryStorage() + elif self._storage_type == StorageType.ROCKSDB: + rocksdb_storage = self._get_or_create_rocksdb_storage() + use_memory_index = self._force_memory_index + return TransactionRocksDBStorage(rocksdb_storage, use_memory_indexes=use_memory_index) + raise NotImplementedError + + def use_memory(self) -> 'Builder': + self.check_if_can_modify() + self._storage_type = StorageType.MEMORY + return self + + def use_rocksdb(self, path: str) -> 'Builder': + self.check_if_can_modify() + self._storage_type = StorageType.ROCKSDB + self._rocksdb_path = path + return self + + def force_memory_index(self) -> 'Builder': + self.check_if_can_modify() + self._force_memory_index = True + return self + + def _get_or_create_wallet(self) -> Optional[BaseWallet]: + if self._wallet is not None: + assert self._wallet_directory is None + assert self._wallet_unlock is None + return self._wallet + + if self._wallet_directory is None: + return None + wallet = Wallet(directory=self._wallet_directory) + if self._wallet_unlock is not None: + wallet.unlock(self._wallet_unlock) + return wallet + + def set_wallet(self, wallet: BaseWallet) -> 'Builder': + self.check_if_can_modify() + self._wallet = wallet + return self + + def enable_keypair_wallet(self, directory: str, *, unlock: Optional[bytes] = None) -> 'Builder': + self.check_if_can_modify() + self._wallet_directory = directory + self._wallet_unlock = unlock + return self + + def enable_stratum_server(self, port: int) -> 'Builder': + self.check_if_can_modify() + self._stratum_port = port + return self + + def enable_address_index(self) -> 'Builder': + self.check_if_can_modify() + self._enable_address_index = True + return self + + def enable_tokens_index(self) -> 'Builder': + self.check_if_can_modify() + self._enable_tokens_index = True + return self + + def enable_utxo_index(self) -> 'Builder': + self.check_if_can_modify() + self._enable_utxo_index = True + return self + + def enable_wallet_index(self) -> 'Builder': + self.check_if_can_modify() + self.enable_address_index() + self.enable_tokens_index() + return self + + def set_tx_storage(self, tx_storage: TransactionStorage) -> 'Builder': + self.check_if_can_modify() + self._tx_storage = tx_storage + return self + + def set_reactor(self, reactor: Reactor) -> 'Builder': + self.check_if_can_modify() + self._reactor = reactor + return self + + def set_pubsub(self, pubsub: PubSubManager) -> 'Builder': + self.check_if_can_modify() + self._pubsub = pubsub + return self + + def set_network(self, network: str) -> 'Builder': + self.check_if_can_modify() + self._network = network + return self + + def enable_sync_v1(self) -> 'Builder': + self.check_if_can_modify() + self._enable_sync_v1 = True + return self + + def disable_sync_v1(self) -> 'Builder': + self.check_if_can_modify() + self._enable_sync_v1 = False + return self + + def enable_sync_v2(self) -> 'Builder': + self.check_if_can_modify() + self._enable_sync_v2 = True + return self + + def disable_sync_v2(self) -> 'Builder': + self.check_if_can_modify() + self._enable_sync_v2 = False + return self diff --git a/hathor/builder.py b/hathor/builder/cli_builder.py similarity index 99% rename from hathor/builder.py rename to hathor/builder/cli_builder.py index 6da0adfe2..5686e4cd2 100644 --- a/hathor/builder.py +++ b/hathor/builder/cli_builder.py @@ -42,6 +42,10 @@ class CliBuilder: + """CliBuilder builds the core objects from args. + + TODO Refactor to use Builder. It could even be ported to a Builder.from_args classmethod. + """ def __init__(self) -> None: self.log = logger.new() diff --git a/hathor/manager.py b/hathor/manager.py index 24e8a3dc1..ff9990193 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -85,11 +85,11 @@ def __init__(self, *, pubsub: PubSubManager, consensus_algorithm: ConsensusAlgorithm, - peer_id: Optional[PeerId] = None, - network: Optional[str] = None, + peer_id: PeerId, + tx_storage: TransactionStorage, + network: str, hostname: Optional[str] = None, wallet: Optional[BaseWallet] = None, - tx_storage: Optional[TransactionStorage] = None, event_manager: Optional[EventManager] = None, stratum_port: Optional[int] = None, ssl: bool = True, @@ -101,7 +101,7 @@ def __init__(self, environment_info: Optional[EnvironmentInfo] = None): """ :param reactor: Twisted reactor which handles the mainloop and the events. - :param peer_id: Id of this node. If not given, a new one is created. + :param peer_id: Id of this node. :param network: Name of the network this node participates. Usually it is either testnet or mainnet. :type network: string @@ -121,9 +121,6 @@ def __init__(self, if not (enable_sync_v1 or enable_sync_v2): raise TypeError(f'{type(self).__name__}() at least one sync version is required') - if tx_storage is None: - raise TypeError(f'{type(self).__name__}() missing 1 required positional argument: \'tx_storage\'') - self._enable_sync_v1 = enable_sync_v1 self._enable_sync_v2 = enable_sync_v2 @@ -147,8 +144,8 @@ def __init__(self, # Remote address, which can be different from local address. self.remote_address = None - self.my_peer = peer_id or PeerId() - self.network = network or 'testnet' + self.my_peer = peer_id + self.network = network self.is_started: bool = False diff --git a/hathor/simulator/simulator.py b/hathor/simulator/simulator.py index 329af090e..89bb3d369 100644 --- a/hathor/simulator/simulator.py +++ b/hathor/simulator/simulator.py @@ -130,6 +130,7 @@ def stop(self) -> None: def create_peer(self, network: Optional[str] = None, peer_id: Optional[PeerId] = None, enable_sync_v1: bool = True, enable_sync_v2: bool = True, soft_voided_tx_ids: Optional[Set[bytes]] = None) -> HathorManager: + # TODO Refactor to use Builder. assert self._started if network is None: network = self._network diff --git a/tests/others/test_builder.py b/tests/others/test_builder.py index ee0fd74b5..b17b7c8c4 100644 --- a/tests/others/test_builder.py +++ b/tests/others/test_builder.py @@ -1,175 +1,22 @@ -from typing import List - -import pytest - -from hathor.builder import CliBuilder -from hathor.event import EventManager -from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage -from hathor.event.websocket import EventWebsocketFactory -from hathor.exception import BuilderError -from hathor.indexes import MemoryIndexesManager, RocksDBIndexesManager -from hathor.manager import HathorManager -from hathor.p2p.sync_version import SyncVersion -from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage, TransactionRocksDBStorage -from hathor.wallet import HDWallet, Wallet from tests import unittest -from tests.utils import HAS_ROCKSDB +from tests.unittest import TestBuilder class BuilderTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.reactor = self.clock + self.builder = TestBuilder() + self.builder.use_memory() - from hathor.cli.run_node import RunNode - self.parser = RunNode.create_parser() - self.builder = CliBuilder() - - def _build_with_error(self, args: List[str], err_msg: str) -> None: - args = self.parser.parse_args(args) - with self.assertRaises(BuilderError) as cm: - self.builder.create_manager(self.reactor, args) - self.builder.register_resources(args, dry_run=True) - self.assertEqual(err_msg, str(cm.exception)) - - def _build(self, args: List[str]) -> HathorManager: - args = self.parser.parse_args(args) - manager = self.builder.create_manager(self.reactor, args) - self.assertIsNotNone(manager) - self.builder.register_resources(args, dry_run=True) - return manager - - def test_empty(self): - self._build_with_error([], '--data is expected') - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_all_default(self): - data_dir = self.mkdtemp() - manager = self._build(['--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - self.assertIsNone(manager.wallet) - self.assertEqual('unittests', manager.network) - self.assertIn(SyncVersion.V1, manager.connections._sync_factories) - self.assertNotIn(SyncVersion.V2, manager.connections._sync_factories) - self.assertFalse(self.builder._build_prometheus) - self.assertFalse(self.builder._build_status) - self.assertIsNone(manager._event_manager) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_cache_storage(self): - data_dir = self.mkdtemp() - manager = self._build(['--cache', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionCacheStorage) - self.assertIsInstance(manager.tx_storage.store, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - self.assertIsNone(manager.tx_storage.store.indexes) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_default_storage_memory_indexes(self): - data_dir = self.mkdtemp() - manager = self._build(['--memory-indexes', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_default_storage_with_rocksdb_indexes(self): - data_dir = self.mkdtemp() - manager = self._build(['--x-rocksdb-indexes', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_rocksdb_storage(self): - data_dir = self.mkdtemp() - manager = self._build(['--rocksdb-storage', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - - def test_memory_storage(self): - manager = self._build(['--memory-storage']) - self.assertIsInstance(manager.tx_storage, TransactionMemoryStorage) - self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) - - def test_memory_storage_with_rocksdb_indexes(self): - self._build_with_error(['--memory-storage', '--x-rocksdb-indexes'], 'RocksDB indexes require RocksDB data') - - def test_sync_bridge(self): - manager = self._build(['--memory-storage', '--x-sync-bridge']) - self.assertIn(SyncVersion.V1, manager.connections._sync_factories) - self.assertIn(SyncVersion.V2, manager.connections._sync_factories) - - def test_sync_v2_only(self): - manager = self._build(['--memory-storage', '--x-sync-v2-only']) - self.assertNotIn(SyncVersion.V1, manager.connections._sync_factories) - self.assertIn(SyncVersion.V2, manager.connections._sync_factories) - - def test_keypair_wallet(self): - manager = self._build(['--memory-storage', '--wallet', 'keypair']) - self.assertIsInstance(manager.wallet, Wallet) - - def test_hd_wallet(self): - manager = self._build(['--memory-storage', '--wallet', 'hd']) - self.assertIsInstance(manager.wallet, HDWallet) - - def test_invalid_wallet(self): - self._build_with_error(['--memory-storage', '--wallet', 'invalid-wallet'], 'Invalid type of wallet') - - def test_status(self): - self._build([ - '--memory-storage', - '--status', '8080', - '--utxo-index', - '--enable-debug-api', - '--enable-crash-api' - ]) - self.assertTrue(self.builder._build_status) - self.clean_pending(required_to_quiesce=False) - - def test_prometheus_no_data(self): - args = ['--memory-storage', '--prometheus'] - self._build_with_error(args, 'To run prometheus exporter you must have a data path') - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_prometheus(self): - data_dir = self.mkdtemp() - self._build(['--prometheus', '--data', data_dir]) - self.assertTrue(self.builder._build_prometheus) - self.clean_pending(required_to_quiesce=False) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_memory_and_rocksdb_indexes(self): - data_dir = self.mkdtemp() - args = ['--memory-indexes', '--x-rocksdb-indexes', '--data', data_dir] - self._build_with_error(args, 'You cannot use --memory-indexes and --x-rocksdb-indexes.') - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') - def test_event_queue_with_rocksdb_storage(self): - data_dir = self.mkdtemp() - manager = self._build(['--x-enable-event-queue', '--rocksdb-storage', '--data', data_dir]) - - self.assertIsInstance(manager._event_manager, EventManager) - self.assertIsInstance(manager._event_manager._event_storage, EventRocksDBStorage) - self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) - self.assertFalse(manager._event_manager.emit_load_events) - - def test_event_queue_with_memory_storage(self): - manager = self._build(['--x-enable-event-queue', '--memory-storage']) - - self.assertIsInstance(manager._event_manager, EventManager) - self.assertIsInstance(manager._event_manager._event_storage, EventMemoryStorage) - self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) - self.assertFalse(manager._event_manager.emit_load_events) + def test_multiple_calls_to_build(self): + self.builder.build() - def test_event_queue_with_full_verification(self): - args = ['--x-enable-event-queue', '--memory-storage', '--x-full-verification'] - self._build_with_error(args, '--x-full-verification cannot be used with --x-enable-event-queue') + with self.assertRaises(ValueError): + self.builder.build() - def test_event_queue_with_emit_load_events(self): - manager = self._build(['--x-enable-event-queue', '--memory-storage', '--x-emit-load-events']) + def test_check_if_can_modify(self): + self.builder.build() - self.assertIsInstance(manager._event_manager, EventManager) - self.assertIsInstance(manager._event_manager._event_storage, EventMemoryStorage) - self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) - self.assertTrue(manager._event_manager.emit_load_events) + with self.assertRaises(ValueError): + self.builder.set_reactor(self.reactor) diff --git a/tests/others/test_cli_builder.py b/tests/others/test_cli_builder.py new file mode 100644 index 000000000..ee0fd74b5 --- /dev/null +++ b/tests/others/test_cli_builder.py @@ -0,0 +1,175 @@ +from typing import List + +import pytest + +from hathor.builder import CliBuilder +from hathor.event import EventManager +from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage +from hathor.event.websocket import EventWebsocketFactory +from hathor.exception import BuilderError +from hathor.indexes import MemoryIndexesManager, RocksDBIndexesManager +from hathor.manager import HathorManager +from hathor.p2p.sync_version import SyncVersion +from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage, TransactionRocksDBStorage +from hathor.wallet import HDWallet, Wallet +from tests import unittest +from tests.utils import HAS_ROCKSDB + + +class BuilderTestCase(unittest.TestCase): + def setUp(self): + super().setUp() + + self.reactor = self.clock + + from hathor.cli.run_node import RunNode + self.parser = RunNode.create_parser() + self.builder = CliBuilder() + + def _build_with_error(self, args: List[str], err_msg: str) -> None: + args = self.parser.parse_args(args) + with self.assertRaises(BuilderError) as cm: + self.builder.create_manager(self.reactor, args) + self.builder.register_resources(args, dry_run=True) + self.assertEqual(err_msg, str(cm.exception)) + + def _build(self, args: List[str]) -> HathorManager: + args = self.parser.parse_args(args) + manager = self.builder.create_manager(self.reactor, args) + self.assertIsNotNone(manager) + self.builder.register_resources(args, dry_run=True) + return manager + + def test_empty(self): + self._build_with_error([], '--data is expected') + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_all_default(self): + data_dir = self.mkdtemp() + manager = self._build(['--data', data_dir]) + self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) + self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) + self.assertIsNone(manager.wallet) + self.assertEqual('unittests', manager.network) + self.assertIn(SyncVersion.V1, manager.connections._sync_factories) + self.assertNotIn(SyncVersion.V2, manager.connections._sync_factories) + self.assertFalse(self.builder._build_prometheus) + self.assertFalse(self.builder._build_status) + self.assertIsNone(manager._event_manager) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_cache_storage(self): + data_dir = self.mkdtemp() + manager = self._build(['--cache', '--data', data_dir]) + self.assertIsInstance(manager.tx_storage, TransactionCacheStorage) + self.assertIsInstance(manager.tx_storage.store, TransactionRocksDBStorage) + self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) + self.assertIsNone(manager.tx_storage.store.indexes) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_default_storage_memory_indexes(self): + data_dir = self.mkdtemp() + manager = self._build(['--memory-indexes', '--data', data_dir]) + self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) + self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_default_storage_with_rocksdb_indexes(self): + data_dir = self.mkdtemp() + manager = self._build(['--x-rocksdb-indexes', '--data', data_dir]) + self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) + self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_rocksdb_storage(self): + data_dir = self.mkdtemp() + manager = self._build(['--rocksdb-storage', '--data', data_dir]) + self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) + self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) + + def test_memory_storage(self): + manager = self._build(['--memory-storage']) + self.assertIsInstance(manager.tx_storage, TransactionMemoryStorage) + self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) + + def test_memory_storage_with_rocksdb_indexes(self): + self._build_with_error(['--memory-storage', '--x-rocksdb-indexes'], 'RocksDB indexes require RocksDB data') + + def test_sync_bridge(self): + manager = self._build(['--memory-storage', '--x-sync-bridge']) + self.assertIn(SyncVersion.V1, manager.connections._sync_factories) + self.assertIn(SyncVersion.V2, manager.connections._sync_factories) + + def test_sync_v2_only(self): + manager = self._build(['--memory-storage', '--x-sync-v2-only']) + self.assertNotIn(SyncVersion.V1, manager.connections._sync_factories) + self.assertIn(SyncVersion.V2, manager.connections._sync_factories) + + def test_keypair_wallet(self): + manager = self._build(['--memory-storage', '--wallet', 'keypair']) + self.assertIsInstance(manager.wallet, Wallet) + + def test_hd_wallet(self): + manager = self._build(['--memory-storage', '--wallet', 'hd']) + self.assertIsInstance(manager.wallet, HDWallet) + + def test_invalid_wallet(self): + self._build_with_error(['--memory-storage', '--wallet', 'invalid-wallet'], 'Invalid type of wallet') + + def test_status(self): + self._build([ + '--memory-storage', + '--status', '8080', + '--utxo-index', + '--enable-debug-api', + '--enable-crash-api' + ]) + self.assertTrue(self.builder._build_status) + self.clean_pending(required_to_quiesce=False) + + def test_prometheus_no_data(self): + args = ['--memory-storage', '--prometheus'] + self._build_with_error(args, 'To run prometheus exporter you must have a data path') + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_prometheus(self): + data_dir = self.mkdtemp() + self._build(['--prometheus', '--data', data_dir]) + self.assertTrue(self.builder._build_prometheus) + self.clean_pending(required_to_quiesce=False) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_memory_and_rocksdb_indexes(self): + data_dir = self.mkdtemp() + args = ['--memory-indexes', '--x-rocksdb-indexes', '--data', data_dir] + self._build_with_error(args, 'You cannot use --memory-indexes and --x-rocksdb-indexes.') + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_event_queue_with_rocksdb_storage(self): + data_dir = self.mkdtemp() + manager = self._build(['--x-enable-event-queue', '--rocksdb-storage', '--data', data_dir]) + + self.assertIsInstance(manager._event_manager, EventManager) + self.assertIsInstance(manager._event_manager._event_storage, EventRocksDBStorage) + self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) + self.assertFalse(manager._event_manager.emit_load_events) + + def test_event_queue_with_memory_storage(self): + manager = self._build(['--x-enable-event-queue', '--memory-storage']) + + self.assertIsInstance(manager._event_manager, EventManager) + self.assertIsInstance(manager._event_manager._event_storage, EventMemoryStorage) + self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) + self.assertFalse(manager._event_manager.emit_load_events) + + def test_event_queue_with_full_verification(self): + args = ['--x-enable-event-queue', '--memory-storage', '--x-full-verification'] + self._build_with_error(args, '--x-full-verification cannot be used with --x-enable-event-queue') + + def test_event_queue_with_emit_load_events(self): + manager = self._build(['--x-enable-event-queue', '--memory-storage', '--x-emit-load-events']) + + self.assertIsInstance(manager._event_manager, EventManager) + self.assertIsInstance(manager._event_manager._event_storage, EventMemoryStorage) + self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) + self.assertTrue(manager._event_manager.emit_load_events) diff --git a/tests/others/test_init_manager.py b/tests/others/test_init_manager.py index ee3e7bce8..bdeb5f4e0 100644 --- a/tests/others/test_init_manager.py +++ b/tests/others/test_init_manager.py @@ -1,12 +1,11 @@ from typing import Iterator from hathor.conf import HathorSettings -from hathor.consensus import ConsensusAlgorithm -from hathor.manager import HathorManager from hathor.pubsub import PubSubManager from hathor.transaction import BaseTransaction from hathor.transaction.storage import TransactionMemoryStorage from tests import unittest +from tests.unittest import TestBuilder from tests.utils import ( add_blocks_unlock_reward, add_new_block, @@ -42,69 +41,47 @@ def setUp(self): self.tx_storage = ModifiedTransactionMemoryStorage() self.pubsub = PubSubManager(self.clock) - soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS) - self.consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub=self.pubsub) - def test_invalid_arguments(self): # this is a base case, it shouldn't raise any error # (otherwise we might not be testing the correct thing below) - manager = HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - tx_storage=self.tx_storage - ) + builder = TestBuilder() + builder.set_tx_storage(self.tx_storage) + artifacts = builder.build() + manager = artifacts.manager del manager # disabling both sync versions should be invalid with self.assertRaises(TypeError): - HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - tx_storage=self.tx_storage, - enable_sync_v1=False, - enable_sync_v2=False - ) - - # not passing a storage should be invalid - with self.assertRaises(TypeError): - HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - ) + builder = TestBuilder() + builder.set_tx_storage(self.tx_storage) + builder.disable_sync_v1() + builder.disable_sync_v2() + builder.build() def tests_init_with_stratum(self): - manager = HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - tx_storage=self.tx_storage, - stratum_port=50505 - ) + builder = TestBuilder() + builder.set_tx_storage(self.tx_storage) + builder.enable_stratum_server(50505) + artifacts = builder.build() + manager = artifacts.manager manager.start() manager.stop() del manager def test_double_start(self): - manager = HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - tx_storage=self.tx_storage - ) + builder = TestBuilder() + builder.set_tx_storage(self.tx_storage) + artifacts = builder.build() + manager = artifacts.manager manager.start() with self.assertRaises(Exception): manager.start() def test_wrong_stop(self): - manager = HathorManager( - self.clock, - pubsub=self.pubsub, - consensus_algorithm=self.consensus_algorithm, - tx_storage=self.tx_storage - ) + builder = TestBuilder() + builder.set_tx_storage(self.tx_storage) + artifacts = builder.build() + manager = artifacts.manager with self.assertRaises(Exception): manager.stop() manager.start() diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index f5f8ec17b..bd77f2028 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -9,9 +9,7 @@ from twisted.trial import unittest from hathor.conf import HathorSettings -from hathor.consensus import ConsensusAlgorithm from hathor.daa import TestMode, _set_test_mode -from hathor.pubsub import PubSubManager from hathor.simulator.clock import MemoryReactorHeapClock from hathor.storage.rocksdb_storage import RocksDBStorage from hathor.transaction import Block, Transaction, TxInput, TxOutput @@ -19,7 +17,7 @@ from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage, TransactionRocksDBStorage from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.transaction_metadata import ValidationState -from hathor.wallet import Wallet +from tests.unittest import TestBuilder from tests.utils import ( BURN_ADDRESS, HAS_ROCKSDB, @@ -38,41 +36,33 @@ class BaseTransactionStorageTest(unittest.TestCase): __test__ = False def setUp(self, tx_storage, reactor=None): - from hathor.manager import HathorManager + self.tmpdir = tempfile.mkdtemp() + + builder = TestBuilder() + builder.set_tx_storage(tx_storage) + builder.enable_keypair_wallet(self.tmpdir, unlock=b'teste') + builder.enable_address_index() + builder.enable_tokens_index() + if reactor is not None: + builder.set_reactor(reactor) + + artifacts = builder.build() + self.reactor = artifacts.reactor + self.pubsub = artifacts.pubsub + self.manager = artifacts.manager + self.tx_storage = artifacts.tx_storage + + assert artifacts.wallet is not None - if not reactor: - self.reactor = MemoryReactorHeapClock() - else: - self.reactor = reactor self.reactor.advance(time.time()) - self.tx_storage = tx_storage - self.pubsub = PubSubManager(self.reactor) - tx_storage._manually_initialize() + self.tx_storage._manually_initialize() assert tx_storage.first_timestamp > 0 self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] - self.tmpdir = tempfile.mkdtemp() - wallet = Wallet(directory=self.tmpdir) - wallet.unlock(b'teste') - - soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS) - consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub=self.pubsub) - - self.manager = HathorManager( - self.reactor, - pubsub=self.pubsub, - consensus_algorithm=consensus_algorithm, - tx_storage=self.tx_storage, - wallet=wallet - ) - - self.tx_storage.indexes.enable_address_index(self.manager.pubsub) - self.tx_storage.indexes.enable_tokens_index() - block_parents = [tx.hash for tx in chain(self.genesis_blocks, self.genesis_txs)] output = TxOutput(200, P2PKH.create_output_script(BURN_ADDRESS)) self.block = Block(timestamp=MIN_TIMESTAMP, weight=12, outputs=[output], parents=block_parents, diff --git a/tests/unittest.py b/tests/unittest.py index 723cd918b..c4c47438c 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -9,17 +9,14 @@ from twisted.internet.task import Clock from twisted.trial import unittest -from hathor.builder import CliBuilder +from hathor.builder import Builder from hathor.conf import HathorSettings -from hathor.consensus import ConsensusAlgorithm from hathor.daa import TestMode, _set_test_mode -from hathor.manager import HathorManager from hathor.p2p.peer_id import PeerId from hathor.p2p.sync_version import SyncVersion -from hathor.pubsub import PubSubManager -from hathor.storage.rocksdb_storage import RocksDBStorage +from hathor.simulator.clock import MemoryReactorHeapClock from hathor.transaction import BaseTransaction -from hathor.util import Random, get_environment_info, reactor +from hathor.util import Random, Reactor, reactor from hathor.wallet import HDWallet, Wallet logger = get_logger() @@ -73,6 +70,22 @@ class SyncBridgeParams: _enable_sync_v2 = True +class TestBuilder(Builder): + def __init__(self) -> None: + super().__init__() + self.set_network('testnet') + + def _get_peer_id(self) -> PeerId: + if self._peer_id is not None: + return self._peer_id + return PeerId() + + def _get_reactor(self) -> Reactor: + if self._reactor: + return self._reactor + return MemoryReactorHeapClock() + + class TestCase(unittest.TestCase): _enable_sync_v1: bool _enable_sync_v2: bool @@ -143,52 +156,63 @@ def create_peer(self, network, peer_id=None, wallet=None, tx_storage=None, unloc enable_sync_v2 = self._enable_sync_v2 assert enable_sync_v1 or enable_sync_v2, 'enable at least one sync version' + builder = TestBuilder() + builder.set_rng(self.rng) + builder.set_reactor(self.clock) + builder.set_network(network) + + if checkpoints is not None: + builder.set_checkpoints(checkpoints) + + if pubsub: + builder.set_pubsub(pubsub) + if peer_id is None: peer_id = PeerId() + builder.set_peer_id(peer_id) + if not wallet: wallet = self._create_test_wallet() if unlock_wallet: wallet.unlock(b'MYPASS') - if tx_storage is None: - if self.use_memory_storage: - from hathor.transaction.storage.memory_storage import TransactionMemoryStorage - tx_storage = TransactionMemoryStorage() - else: - from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage - directory = tempfile.mkdtemp() - self.tmpdirs.append(directory) - rocksdb_storage = RocksDBStorage(path=directory) - self._pending_cleanups.append(rocksdb_storage.close) - tx_storage = TransactionRocksDBStorage(rocksdb_storage, use_memory_indexes=use_memory_index) + builder.set_wallet(wallet) + + if event_manager: + builder.set_event_manager(event_manager) + + if tx_storage is not None: + builder.set_tx_storage(tx_storage) + elif self.use_memory_storage: + builder.use_memory() + else: + directory = tempfile.mkdtemp() + self.tmpdirs.append(directory) + builder.use_rocksdb(directory) - pubsub = pubsub or PubSubManager(self.clock) + if use_memory_index is True: + builder.force_memory_index() + + if enable_sync_v1 is True: + builder.enable_sync_v1() + elif enable_sync_v1 is False: + builder.disable_sync_v1() + + if enable_sync_v2 is True: + builder.enable_sync_v2() + elif enable_sync_v2 is False: + builder.disable_sync_v2() - builder = CliBuilder() if wallet_index: - builder.enable_wallet_index(tx_storage.indexes, pubsub) + builder.enable_wallet_index() if utxo_index: - tx_storage.indexes.enable_utxo_index() - - soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS) - consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub=pubsub) - - manager = HathorManager( - self.clock, - pubsub=pubsub, - consensus_algorithm=consensus_algorithm, - peer_id=peer_id, - network=network, - wallet=wallet, - tx_storage=tx_storage, - event_manager=event_manager, - capabilities=capabilities, - rng=self.rng, - enable_sync_v1=enable_sync_v1, - enable_sync_v2=enable_sync_v2, - checkpoints=checkpoints, - environment_info=get_environment_info("", peer_id.id) - ) + builder.enable_utxo_index() + + artifacts = builder.build() + manager = artifacts.manager + + if artifacts.rocksdb_storage: + self._pending_cleanups.append(artifacts.rocksdb_storage.close) # XXX: just making sure that tests set this up correctly if enable_sync_v2: