diff --git a/docs/event-queue-feature.md b/docs/event-queue-feature.md index 81025e740..0f564391f 100644 --- a/docs/event-queue-feature.md +++ b/docs/event-queue-feature.md @@ -20,21 +20,19 @@ To enable the Event Queue feature, you must add this CLI option when running the For example: ```bash -poetry run hathor-cli run_node --memory-storage --status 8080 --testnet --enable-event-queue +poetry run hathor-cli run_node --temp-data --status 8080 --testnet --enable-event-queue ``` ### First run -If this is the first time your full node is running with the event queue enabled, there are 3 possibilities: +If this is the first time your full node is running with the event queue enabled, there are 2 possibilities: -1. You're running the full node using memory storage, like in the example above; -2. You're running the full node using RocksDB storage (the default option), and - 1. You're performing a sync from scratch, that is, you don't have an existing database, or - 2. You're running from an existing database. +1. You're performing a sync from scratch or you're using a temporary database (like in the example above), that is, you don't have an existing database, or +2. You're running from an existing database. -For cases 1 and 2.1, the full node will start normally, events will be generated in real time while vertices are synced, and they'll be sent to the WebSocket connection accordingly, as explained below. +For case 1, the full node will start normally, events will be generated in real time while vertices are synced and they'll be sent to the WebSocket connection accordingly, as explained below. -For case 2.2, an extra loading step will be performed during full node initialization, generating events for all existing vertices in your database. This step is slower than normal full node initialization and can take several minutes. Note that this will only be necessary once — after initialization, the events generated for your database are persisted and will be used in subsequent runs. +For case 2, an extra loading step will be performed during full node initialization, generating events for all existing vertices in your database. This step is slower than normal full node initialization and can take several minutes. Note that this will only be necessary once — after initialization, the events generated for your database are persisted and will be used in subsequent runs. ### Subsequent runs when using RocksDB diff --git a/extras/custom_tests/side_dag/utils.py b/extras/custom_tests/side_dag/utils.py index 7793c30ac..4359b1d36 100644 --- a/extras/custom_tests/side_dag/utils.py +++ b/extras/custom_tests/side_dag/utils.py @@ -29,11 +29,11 @@ python -m hathor run_node_with_side_dag --disable-logs --testnet - --memory-storage + --temp-data --x-localhost-only --procname-prefix {HATHOR_PROCESS_PREFIX} --side-dag-testnet - --side-dag-memory-storage + --side-dag-temp-data --side-dag-x-localhost-only --side-dag-procname-prefix {SIDE_DAG_PROCESS_PREFIX} """ diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index 3c5cccf5f..753337568 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from enum import Enum, IntEnum +import tempfile +from enum import IntEnum from typing import Any, Callable, NamedTuple, Optional, TypeAlias from structlog import get_logger -from typing_extensions import assert_never from hathor.checkpoint import Checkpoint from hathor.conf.settings import HathorSettings as HathorSettingsType @@ -24,14 +24,14 @@ from hathor.consensus.poa import PoaBlockProducer, PoaSigner from hathor.daa import DifficultyAdjustmentAlgorithm from hathor.event import EventManager -from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage +from hathor.event.storage import EventRocksDBStorage, EventStorage from hathor.event.websocket import EventWebsocketFactory from hathor.execution_manager import ExecutionManager from hathor.feature_activation.bit_signaling_service import BitSignalingService from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import FeatureService from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage -from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager +from hathor.indexes import IndexesManager, RocksDBIndexesManager from hathor.manager import HathorManager from hathor.mining.cpu_mining_service import CpuMiningService from hathor.p2p.manager import ConnectionsManager @@ -40,12 +40,7 @@ from hathor.reactor import ReactorProtocol as Reactor from hathor.storage import RocksDBStorage from hathor.stratum import StratumFactory -from hathor.transaction.storage import ( - TransactionCacheStorage, - TransactionMemoryStorage, - TransactionRocksDBStorage, - TransactionStorage, -) +from hathor.transaction.storage import TransactionCacheStorage, TransactionRocksDBStorage, TransactionStorage from hathor.transaction.vertex_parser import VertexParser from hathor.util import Random, get_environment_info from hathor.verification.verification_service import VerificationService @@ -87,11 +82,6 @@ def add_factories( p2p_manager.enable_sync_version(SyncVersion.V2) -class StorageType(Enum): - MEMORY = 'memory' - ROCKSDB = 'rocksdb' - - class BuildArtifacts(NamedTuple): """Artifacts created by a builder.""" peer: PrivatePeer @@ -107,7 +97,7 @@ class BuildArtifacts(NamedTuple): bit_signaling_service: BitSignalingService indexes: Optional[IndexesManager] wallet: Optional[BaseWallet] - rocksdb_storage: Optional[RocksDBStorage] + rocksdb_storage: RocksDBStorage stratum_factory: Optional[StratumFactory] @@ -123,7 +113,7 @@ class Builder: Example: builder = Builder() - builder.use_memory() + builder.enable_event_queue() artifacts = builder.build() """ def __init__(self) -> None: @@ -138,9 +128,6 @@ def __init__(self) -> None: self._peer: Optional[PrivatePeer] = None self._cmdline: str = '' - self._storage_type: StorageType = StorageType.MEMORY - self._force_memory_index: bool = False - self._event_manager: Optional[EventManager] = None self._enable_event_queue: Optional[bool] = None @@ -156,7 +143,7 @@ def __init__(self) -> None: self._vertex_verifiers_builder: _VertexVerifiersBuilder | None = None self._verification_service: Optional[VerificationService] = None - self._rocksdb_path: Optional[str] = None + self._rocksdb_path: str | tempfile.TemporaryDirectory | None = None self._rocksdb_storage: Optional[RocksDBStorage] = None self._rocksdb_cache_capacity: Optional[int] = None @@ -217,6 +204,7 @@ def build(self) -> BuildArtifacts: event_manager = self._get_or_create_event_manager() indexes = self._get_or_create_indexes_manager() tx_storage = self._get_or_create_tx_storage() + rocksdb_storage = self._get_or_create_rocksdb_storage() feature_service = self._get_or_create_feature_service() bit_signaling_service = self._get_or_create_bit_signaling_service() verification_service = self._get_or_create_verification_service() @@ -285,7 +273,7 @@ def build(self) -> BuildArtifacts: tx_storage=tx_storage, indexes=indexes, wallet=wallet, - rocksdb_storage=self._rocksdb_storage, + rocksdb_storage=rocksdb_storage, stratum_factory=stratum_factory, feature_service=feature_service, bit_signaling_service=bit_signaling_service @@ -383,20 +371,11 @@ def _create_stratum_server(self, manager: HathorManager) -> StratumFactory: return stratum_factory def _get_or_create_rocksdb_storage(self) -> RocksDBStorage: - assert self._rocksdb_path is not None - - if self._rocksdb_storage is not None: - return self._rocksdb_storage - - kwargs = {} - if self._rocksdb_cache_capacity is not None: - kwargs = dict(cache_capacity=self._rocksdb_cache_capacity) - - self._rocksdb_storage = RocksDBStorage( - path=self._rocksdb_path, - **kwargs - ) - + if self._rocksdb_storage is None: + self._rocksdb_storage = RocksDBStorage( + path=self._rocksdb_path, + cache_capacity=self._rocksdb_cache_capacity, + ) if self._rocksdb_path else RocksDBStorage.create_temp(self._rocksdb_cache_capacity) return self._rocksdb_storage def _get_or_create_p2p_manager(self) -> ConnectionsManager: @@ -428,19 +407,12 @@ def _get_or_create_p2p_manager(self) -> ConnectionsManager: return self._p2p_manager def _get_or_create_indexes_manager(self) -> IndexesManager: - if self._indexes_manager is not None: - return self._indexes_manager - - if self._force_memory_index or self._storage_type == StorageType.MEMORY: - self._indexes_manager = MemoryIndexesManager(settings=self._get_or_create_settings()) - - elif self._storage_type == StorageType.ROCKSDB: + if self._indexes_manager is None: rocksdb_storage = self._get_or_create_rocksdb_storage() - self._indexes_manager = RocksDBIndexesManager(rocksdb_storage) - - else: - raise NotImplementedError - + self._indexes_manager = RocksDBIndexesManager( + rocksdb_storage, + settings=self._get_or_create_settings(), + ) return self._indexes_manager def _get_or_create_tx_storage(self) -> TransactionStorage: @@ -456,21 +428,14 @@ def _get_or_create_tx_storage(self) -> TransactionStorage: if self._tx_storage_cache: store_indexes = None - if self._storage_type == StorageType.MEMORY: - self._tx_storage = TransactionMemoryStorage(indexes=store_indexes, settings=settings) - - elif self._storage_type == StorageType.ROCKSDB: - rocksdb_storage = self._get_or_create_rocksdb_storage() - vertex_parser = self._get_or_create_vertex_parser() - self._tx_storage = TransactionRocksDBStorage( - rocksdb_storage, - indexes=store_indexes, - settings=settings, - vertex_parser=vertex_parser, - ) - - else: - raise NotImplementedError + rocksdb_storage = self._get_or_create_rocksdb_storage() + vertex_parser = self._get_or_create_vertex_parser() + self._tx_storage = TransactionRocksDBStorage( + rocksdb_storage, + indexes=store_indexes, + settings=settings, + vertex_parser=vertex_parser, + ) if self._tx_storage_cache: reactor = self._get_reactor() @@ -484,16 +449,9 @@ def _get_or_create_tx_storage(self) -> TransactionStorage: return self._tx_storage def _get_or_create_event_storage(self) -> EventStorage: - if self._event_storage is not None: - pass - elif self._storage_type == StorageType.MEMORY: - self._event_storage = EventMemoryStorage() - elif self._storage_type == StorageType.ROCKSDB: + if self._event_storage is None: rocksdb_storage = self._get_or_create_rocksdb_storage() self._event_storage = EventRocksDBStorage(rocksdb_storage) - else: - raise NotImplementedError - return self._event_storage def _get_or_create_event_manager(self) -> EventManager: @@ -557,14 +515,11 @@ def _get_or_create_verification_service(self) -> VerificationService: return self._verification_service - def _get_or_create_feature_storage(self) -> FeatureActivationStorage | None: - match self._storage_type: - case StorageType.MEMORY: return None - case StorageType.ROCKSDB: return FeatureActivationStorage( - settings=self._get_or_create_settings(), - rocksdb_storage=self._get_or_create_rocksdb_storage() - ) - case _: assert_never(self._storage_type) + def _get_or_create_feature_storage(self) -> FeatureActivationStorage: + return FeatureActivationStorage( + settings=self._get_or_create_settings(), + rocksdb_storage=self._get_or_create_rocksdb_storage() + ) def _get_or_create_vertex_verifiers(self) -> VertexVerifiers: if self._vertex_verifiers is None: @@ -633,33 +588,28 @@ def _get_or_create_poa_block_producer(self) -> PoaBlockProducer | None: return self._poa_block_producer - def use_memory(self) -> 'Builder': + def set_rocksdb_path(self, path: str | tempfile.TemporaryDirectory) -> 'Builder': + if self._tx_storage: + raise ValueError('cannot set rocksdb path after tx storage is set') self.check_if_can_modify() - self._storage_type = StorageType.MEMORY + self._rocksdb_path = path return self - def use_rocksdb( - self, - path: str, - cache_capacity: Optional[int] = None - ) -> 'Builder': + def set_rocksdb_cache_capacity(self, cache_capacity: int) -> 'Builder': + if self._tx_storage: + raise ValueError('cannot set rocksdb cache capacity after tx storage is set') self.check_if_can_modify() - self._storage_type = StorageType.ROCKSDB - self._rocksdb_path = path self._rocksdb_cache_capacity = cache_capacity return self def use_tx_storage_cache(self, capacity: Optional[int] = None) -> 'Builder': + if self._tx_storage: + raise ValueError('cannot set tx storage cache capacity after tx storage is set') self.check_if_can_modify() self._tx_storage_cache = True self._tx_storage_cache_capacity = capacity return self - def force_memory_index(self) -> 'Builder': - self.check_if_can_modify() - self._force_memory_index = True - return self - def _get_or_create_wallet(self) -> Optional[BaseWallet]: if self._wallet is not None: return self._wallet @@ -688,21 +638,29 @@ def enable_stratum_server(self) -> 'Builder': return self def enable_address_index(self) -> 'Builder': + if self._tx_storage or self._indexes_manager: + raise ValueError('cannot enable index after tx storage or indexes manager is set') self.check_if_can_modify() self._enable_address_index = True return self def enable_tokens_index(self) -> 'Builder': + if self._tx_storage or self._indexes_manager: + raise ValueError('cannot enable index after tx storage or indexes manager is set') self.check_if_can_modify() self._enable_tokens_index = True return self def enable_utxo_index(self) -> 'Builder': + if self._tx_storage or self._indexes_manager: + raise ValueError('cannot enable index after tx storage or indexes manager is set') self.check_if_can_modify() self._enable_utxo_index = True return self def enable_wallet_index(self) -> 'Builder': + if self._tx_storage or self._indexes_manager: + raise ValueError('cannot enable index after tx storage or indexes manager is set') self.check_if_can_modify() self.enable_address_index() self.enable_tokens_index() @@ -716,6 +674,9 @@ def enable_event_queue(self) -> 'Builder': def set_tx_storage(self, tx_storage: TransactionStorage) -> 'Builder': self.check_if_can_modify() self._tx_storage = tx_storage + internal = tx_storage.store if isinstance(tx_storage, TransactionCacheStorage) else tx_storage + assert isinstance(internal, TransactionRocksDBStorage) + self._rocksdb_storage = internal._rocksdb_storage return self def set_event_storage(self, event_storage: EventStorage) -> 'Builder': diff --git a/hathor/builder/cli_builder.py b/hathor/builder/cli_builder.py index 29d87f38b..e946023b2 100644 --- a/hathor/builder/cli_builder.py +++ b/hathor/builder/cli_builder.py @@ -30,7 +30,7 @@ from hathor.feature_activation.bit_signaling_service import BitSignalingService from hathor.feature_activation.feature_service import FeatureService from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage -from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager +from hathor.indexes import IndexesManager, RocksDBIndexesManager from hathor.manager import HathorManager from hathor.mining.cpu_mining_service import CpuMiningService from hathor.p2p.manager import ConnectionsManager @@ -76,17 +76,12 @@ def create_manager(self, reactor: Reactor) -> HathorManager: from hathor.builder import SyncSupportLevel from hathor.conf.get_settings import get_global_settings, get_settings_source from hathor.daa import TestMode - from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage + from hathor.event.storage import EventRocksDBStorage, EventStorage from hathor.event.websocket.factory import EventWebsocketFactory from hathor.p2p.netfilter.utils import add_peer_id_blacklist from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery from hathor.storage import RocksDBStorage - from hathor.transaction.storage import ( - TransactionCacheStorage, - TransactionMemoryStorage, - TransactionRocksDBStorage, - TransactionStorage, - ) + from hathor.transaction.storage import TransactionCacheStorage, TransactionRocksDBStorage, TransactionStorage from hathor.util import get_environment_info settings = get_global_settings() @@ -126,50 +121,32 @@ def create_manager(self, reactor: Reactor) -> HathorManager: tx_storage: TransactionStorage event_storage: EventStorage indexes: IndexesManager - feature_storage: FeatureActivationStorage | None = None - self.rocksdb_storage: Optional[RocksDBStorage] = None self.event_ws_factory: Optional[EventWebsocketFactory] = None - if self._args.memory_storage: - self.log.warn('--memory-storage is deprecated, use --temp-data instead') - self.check_or_raise(not self._args.data, '--data should not be used with --memory-storage') - self.check_or_raise(not self._args.temp_data, '--temp-data should not be used with --memory-storage') - # if using MemoryStorage, no need to have cache - indexes = MemoryIndexesManager() - tx_storage = TransactionMemoryStorage(indexes, settings=settings) - event_storage = EventMemoryStorage() - self.check_or_raise(not self._args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data') - self.log.info('with storage', storage_class=type(tx_storage).__name__) - else: - self.check_or_raise( - bool(self._args.data) or self._args.temp_data, - 'either --data or --temp-data is expected' - ) - if self._args.rocksdb_storage: - self.log.warn('--rocksdb-storage is now implied, no need to specify it') - cache_capacity = self._args.rocksdb_cache - self.rocksdb_storage = ( - RocksDBStorage(path=self._args.data, cache_capacity=cache_capacity) - if self._args.data else RocksDBStorage.create_temp(cache_capacity) - ) + memory_msg = 'is deprecated. use --temp-data instead' + self.check_or_raise(not self._args.memory_storage, f'--memory-storage {memory_msg}') + self.check_or_raise(not self._args.memory_indexes, f'--memory-indexes {memory_msg}') - # Initialize indexes manager. - if self._args.memory_indexes: - self.log.warn('--memory-indexes is deprecated') - indexes = MemoryIndexesManager() - else: - indexes = RocksDBIndexesManager(self.rocksdb_storage) - - kwargs: dict[str, Any] = {} - if self._args.disable_cache: - # We should only pass indexes if cache is disabled. Otherwise, - # only TransactionCacheStorage should have indexes. - kwargs['indexes'] = indexes - tx_storage = TransactionRocksDBStorage( - self.rocksdb_storage, settings=settings, vertex_parser=vertex_parser, **kwargs - ) - event_storage = EventRocksDBStorage(self.rocksdb_storage) - feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage) + self.check_or_raise(bool(self._args.data) or self._args.temp_data, 'either --data or --temp-data is expected') + cache_capacity = self._args.rocksdb_cache + self.rocksdb_storage = ( + RocksDBStorage(path=self._args.data, cache_capacity=cache_capacity) + if self._args.data else RocksDBStorage.create_temp(cache_capacity) + ) + + # Initialize indexes manager. + indexes = RocksDBIndexesManager(self.rocksdb_storage, settings=settings) + + kwargs: dict[str, Any] = {} + if self._args.disable_cache: + # We should only pass indexes if cache is disabled. Otherwise, + # only TransactionCacheStorage should have indexes. + kwargs['indexes'] = indexes + tx_storage = TransactionRocksDBStorage( + self.rocksdb_storage, settings=settings, vertex_parser=vertex_parser, **kwargs + ) + event_storage = EventRocksDBStorage(self.rocksdb_storage) + feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage) self.log.info('with storage', storage_class=type(tx_storage).__name__, path=self._args.data) @@ -180,13 +157,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: self.check_or_raise(self._args.cache_size is None, 'cannot use --disable-cache with --cache-size') self.check_or_raise(self._args.cache_interval is None, 'cannot use --disable-cache with --cache-interval') - if self._args.memory_storage: - if self._args.cache_size: - self.log.warn('using --cache-size with --memory-storage has no effect') - if self._args.cache_interval: - self.log.warn('using --cache-interval with --memory-storage has no effect') - - if not self._args.disable_cache and not self._args.memory_storage: + if not self._args.disable_cache: tx_storage = TransactionCacheStorage(tx_storage, reactor, indexes=indexes, settings=settings) tx_storage.capacity = self._args.cache_size if self._args.cache_size is not None else DEFAULT_CACHE_SIZE if self._args.cache_interval: @@ -384,14 +355,6 @@ def create_manager(self, reactor: Reactor) -> HathorManager: entrypoints = [PeerEndpoint.parse(desc) for desc in self._args.bootstrap] p2p_manager.add_peer_discovery(BootstrapPeerDiscovery(entrypoints)) - if self._args.x_rocksdb_indexes: - self.log.warn('--x-rocksdb-indexes is now the default, no need to specify it') - if self._args.memory_indexes: - raise BuilderError('You cannot use --memory-indexes and --x-rocksdb-indexes.') - - if self._args.memory_indexes and self._args.memory_storage: - self.log.warn('--memory-indexes is implied for memory storage or JSON storage') - for description in self._args.listen: p2p_manager.add_listen_address_description(description) diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 21429d1fa..c745b346c 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -99,9 +99,7 @@ def create_parser(cls) -> ArgumentParser: data_group.add_argument('--data', help='Data directory') data_group.add_argument('--temp-data', action='store_true', help='Automatically create storage in a temporary directory') - storage = parser.add_mutually_exclusive_group() - storage.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend (default)') - storage.add_argument('--memory-storage', action='store_true', help=SUPPRESS) # deprecated + parser.add_argument('--memory-storage', action='store_true', help=SUPPRESS) # deprecated parser.add_argument('--memory-indexes', action='store_true', help=SUPPRESS) # deprecated parser.add_argument('--rocksdb-cache', type=int, help='RocksDB block-table cache size (bytes)', default=None) parser.add_argument('--wallet', help='Set wallet type. Options are hd (Hierarchical Deterministic) or keypair', @@ -142,7 +140,6 @@ def create_parser(cls) -> ArgumentParser: parser.add_argument('--x-sync-v2-only', action='store_true', help=SUPPRESS) # deprecated parser.add_argument('--x-sync-bridge', action='store_true', help=SUPPRESS) # deprecated parser.add_argument('--x-localhost-only', action='store_true', help='Only connect to peers on localhost') - parser.add_argument('--x-rocksdb-indexes', action='store_true', help=SUPPRESS) parser.add_argument('--x-enable-event-queue', action='store_true', help='Deprecated: use --enable-event-queue instead.') parser.add_argument('--enable-event-queue', action='store_true', help='Enable event queue mechanism') @@ -253,7 +250,7 @@ def prepare(self, *, register_resources: bool = True) -> None: tx_storage=self.manager.tx_storage, indexes=self.manager.tx_storage.indexes, wallet=self.manager.wallet, - rocksdb_storage=getattr(builder, 'rocksdb_storage', None), + rocksdb_storage=builder.rocksdb_storage, stratum_factory=self.manager.stratum_factory, feature_service=self.manager.vertex_handler._feature_service, bit_signaling_service=self.manager._bit_signaling_service, diff --git a/hathor/cli/run_node_args.py b/hathor/cli/run_node_args.py index 7c05f879d..e46059b51 100644 --- a/hathor/cli/run_node_args.py +++ b/hathor/cli/run_node_args.py @@ -40,7 +40,6 @@ class RunNodeArgs(BaseModel, extra=Extra.allow): stratum: Optional[int] x_stratum_ipv6_interface: Optional[str] data: Optional[str] - rocksdb_storage: bool memory_storage: bool memory_indexes: bool temp_data: bool @@ -74,7 +73,6 @@ class RunNodeArgs(BaseModel, extra=Extra.allow): sync_v1_only: bool sync_v2_only: bool x_localhost_only: bool - x_rocksdb_indexes: bool x_enable_event_queue: bool enable_event_queue: bool peer_id_blacklist: list[str] diff --git a/hathor/cli/side_dag.py b/hathor/cli/side_dag.py index 20132f2d5..6658b8f08 100644 --- a/hathor/cli/side_dag.py +++ b/hathor/cli/side_dag.py @@ -82,10 +82,10 @@ def main(capture_stdout: bool) -> None: $ python -m hathor run_node_with_side_dag --testnet --procname-prefix testnet- - --memory-storage + --temp-data --side-dag-config-yaml ./my-side-dag.yml --side-dag-procname-prefix my-side-dag- - --side-dag-memory-storage + --side-dag-temp-data --json-logs both ``` diff --git a/hathor/event/storage/__init__.py b/hathor/event/storage/__init__.py index 57017aa4c..aaaeb30d7 100644 --- a/hathor/event/storage/__init__.py +++ b/hathor/event/storage/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. from hathor.event.storage.event_storage import EventStorage -from hathor.event.storage.memory_storage import EventMemoryStorage from hathor.event.storage.rocksdb_storage import EventRocksDBStorage -__all__ = ['EventStorage', 'EventMemoryStorage', 'EventRocksDBStorage'] +__all__ = ['EventStorage', 'EventRocksDBStorage'] diff --git a/hathor/event/storage/memory_storage.py b/hathor/event/storage/memory_storage.py deleted file mode 100644 index 6de5c6df5..000000000 --- a/hathor/event/storage/memory_storage.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2022 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, Iterator, Optional - -from hathor.event.model.base_event import BaseEvent -from hathor.event.model.node_state import NodeState -from hathor.event.storage.event_storage import EventStorage - - -class EventMemoryStorage(EventStorage): - def __init__(self) -> None: - self._events: list[BaseEvent] = [] - self._last_event: Optional[BaseEvent] = None - self._last_group_id: Optional[int] = None - self._stream_id: Optional[str] = None - self._node_state: Optional[NodeState] = None - self._event_queue_enabled: bool = False - - def save_event(self, event: BaseEvent) -> None: - if event.id != len(self._events): - raise ValueError('invalid event.id, ids must be sequential and leave no gaps') - self._last_event = event - if event.group_id is not None: - self._last_group_id = event.group_id - self._events.append(event) - - def save_events(self, events: Iterable[BaseEvent]) -> None: - for event in events: - self.save_event(event) - - def get_event(self, key: int) -> Optional[BaseEvent]: - if key < 0: - raise ValueError(f'event.id \'{key}\' must be non-negative') - if key >= len(self._events): - return None - event = self._events[key] - assert event.id == key - return event - - def get_last_event(self) -> Optional[BaseEvent]: - return self._last_event - - def get_last_group_id(self) -> Optional[int]: - return self._last_group_id - - def iter_from_event(self, key: int) -> Iterator[BaseEvent]: - if key < 0: - raise ValueError(f'event.id \'{key}\' must be non-negative') - - while key < len(self._events): - yield self._events[key] - key += 1 - - def reset_events(self) -> None: - self._events = [] - self._last_event = None - self._last_group_id = None - self._stream_id = None - - def reset_all(self) -> None: - self.reset_events() - self._node_state = None - self._event_queue_enabled = False - - def save_node_state(self, state: NodeState) -> None: - self._node_state = state - - def get_node_state(self) -> Optional[NodeState]: - return self._node_state - - def save_event_queue_state(self, enabled: bool) -> None: - self._event_queue_enabled = enabled - - def get_event_queue_state(self) -> bool: - return self._event_queue_enabled - - def save_stream_id(self, stream_id: str) -> None: - self._stream_id = stream_id - - def get_stream_id(self) -> Optional[str]: - return self._stream_id diff --git a/hathor/indexes/__init__.py b/hathor/indexes/__init__.py index 7bbabca88..d11fcbfec 100644 --- a/hathor/indexes/__init__.py +++ b/hathor/indexes/__init__.py @@ -13,12 +13,11 @@ # limitations under the License. from hathor.indexes.address_index import AddressIndex -from hathor.indexes.manager import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager +from hathor.indexes.manager import IndexesManager, RocksDBIndexesManager from hathor.indexes.timestamp_index import TimestampIndex __all__ = [ 'IndexesManager', - 'MemoryIndexesManager', 'RocksDBIndexesManager', 'AddressIndex', 'TimestampIndex', diff --git a/hathor/indexes/base_index.py b/hathor/indexes/base_index.py index 98b1c0721..5d1cb87b2 100644 --- a/hathor/indexes/base_index.py +++ b/hathor/indexes/base_index.py @@ -19,7 +19,6 @@ from structlog import get_logger -from hathor.conf.get_settings import get_global_settings from hathor.indexes.scope import Scope from hathor.transaction.base_transaction import BaseTransaction @@ -36,8 +35,8 @@ class BaseIndex(ABC): This class exists so we can interact with indexes without knowing anything specific to its implemented. It was created to generalize how we initialize indexes and keep track of which ones are up-to-date. """ - def __init__(self, *, settings: HathorSettings | None = None) -> None: - self._settings = settings or get_global_settings() + def __init__(self, *, settings: HathorSettings) -> None: + self._settings = settings self.log = logger.new() def init_start(self, indexes_manager: 'IndexesManager') -> None: @@ -57,7 +56,7 @@ def get_db_name(self) -> Optional[str]: """ The returned string is used to generate the relevant attributes for storing an indexe's state in the db. If None is returned, the database will not store the index initialization state and they will always be - initialized. This is the expected mode that memory-only indexes will use. + initialized. """ raise NotImplementedError diff --git a/hathor/indexes/manager.py b/hathor/indexes/manager.py index 051850656..af648479a 100644 --- a/hathor/indexes/manager.py +++ b/hathor/indexes/manager.py @@ -261,70 +261,25 @@ def del_tx(self, tx: BaseTransaction, *, remove_all: bool = False, relax_assert: self.tokens.del_tx(tx, remove_all=remove_all) -class MemoryIndexesManager(IndexesManager): - def __init__(self, *, settings: HathorSettings | None = None) -> None: - from hathor.indexes.memory_height_index import MemoryHeightIndex - from hathor.indexes.memory_info_index import MemoryInfoIndex - from hathor.indexes.memory_timestamp_index import MemoryTimestampIndex - from hathor.indexes.memory_tips_index import MemoryTipsIndex - - self.info = MemoryInfoIndex() - self.all_tips = MemoryTipsIndex(scope_type=TipsScopeType.ALL) - self.block_tips = MemoryTipsIndex(scope_type=TipsScopeType.BLOCKS) - self.tx_tips = MemoryTipsIndex(scope_type=TipsScopeType.TXS) - - self.sorted_all = MemoryTimestampIndex(scope_type=TimestampScopeType.ALL) - self.sorted_blocks = MemoryTimestampIndex(scope_type=TimestampScopeType.BLOCKS) - self.sorted_txs = MemoryTimestampIndex(scope_type=TimestampScopeType.TXS) - - self.addresses = None - self.tokens = None - self.utxo = None - self.height = MemoryHeightIndex(settings=settings) - self.mempool_tips = None - - # XXX: this has to be at the end of __init__, after everything has been initialized - self.__init_checks__() - - def enable_address_index(self, pubsub: 'PubSubManager') -> None: - from hathor.indexes.memory_address_index import MemoryAddressIndex - if self.addresses is None: - self.addresses = MemoryAddressIndex(pubsub) - - def enable_tokens_index(self) -> None: - from hathor.indexes.memory_tokens_index import MemoryTokensIndex - if self.tokens is None: - self.tokens = MemoryTokensIndex() - - def enable_utxo_index(self) -> None: - from hathor.indexes.memory_utxo_index import MemoryUtxoIndex - if self.utxo is None: - self.utxo = MemoryUtxoIndex() - - def enable_mempool_index(self) -> None: - from hathor.indexes.memory_mempool_tips_index import MemoryMempoolTipsIndex - if self.mempool_tips is None: - self.mempool_tips = MemoryMempoolTipsIndex() - - class RocksDBIndexesManager(IndexesManager): - def __init__(self, rocksdb_storage: 'RocksDBStorage') -> None: + def __init__(self, rocksdb_storage: 'RocksDBStorage', *, settings: HathorSettings) -> None: from hathor.indexes.partial_rocksdb_tips_index import PartialRocksDBTipsIndex from hathor.indexes.rocksdb_height_index import RocksDBHeightIndex from hathor.indexes.rocksdb_info_index import RocksDBInfoIndex from hathor.indexes.rocksdb_timestamp_index import RocksDBTimestampIndex + self.settings = settings self._db = rocksdb_storage.get_db() - self.info = RocksDBInfoIndex(self._db) - self.height = RocksDBHeightIndex(self._db) - self.all_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.ALL) - self.block_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.BLOCKS) - self.tx_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.TXS) + self.info = RocksDBInfoIndex(self._db, settings=settings) + self.height = RocksDBHeightIndex(self._db, settings=settings) + self.all_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.ALL, settings=settings) + self.block_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.BLOCKS, settings=settings) + self.tx_tips = PartialRocksDBTipsIndex(self._db, scope_type=TipsScopeType.TXS, settings=settings) - self.sorted_all = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.ALL) - self.sorted_blocks = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.BLOCKS) - self.sorted_txs = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.TXS) + self.sorted_all = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.ALL, settings=settings) + self.sorted_blocks = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.BLOCKS, settings=settings) + self.sorted_txs = RocksDBTimestampIndex(self._db, scope_type=TimestampScopeType.TXS, settings=settings) self.addresses = None self.tokens = None @@ -337,20 +292,20 @@ def __init__(self, rocksdb_storage: 'RocksDBStorage') -> None: def enable_address_index(self, pubsub: 'PubSubManager') -> None: from hathor.indexes.rocksdb_address_index import RocksDBAddressIndex if self.addresses is None: - self.addresses = RocksDBAddressIndex(self._db, pubsub=pubsub) + self.addresses = RocksDBAddressIndex(self._db, pubsub=pubsub, settings=self.settings) def enable_tokens_index(self) -> None: from hathor.indexes.rocksdb_tokens_index import RocksDBTokensIndex if self.tokens is None: - self.tokens = RocksDBTokensIndex(self._db) + self.tokens = RocksDBTokensIndex(self._db, settings=self.settings) def enable_utxo_index(self) -> None: from hathor.indexes.rocksdb_utxo_index import RocksDBUtxoIndex if self.utxo is None: - self.utxo = RocksDBUtxoIndex(self._db) + self.utxo = RocksDBUtxoIndex(self._db, settings=self.settings) def enable_mempool_index(self) -> None: from hathor.indexes.memory_mempool_tips_index import MemoryMempoolTipsIndex if self.mempool_tips is None: # XXX: use of RocksDBMempoolTipsIndex is very slow and was suspended - self.mempool_tips = MemoryMempoolTipsIndex() + self.mempool_tips = MemoryMempoolTipsIndex(settings=self.settings) diff --git a/hathor/indexes/memory_address_index.py b/hathor/indexes/memory_address_index.py deleted file mode 100644 index 4360bda21..000000000 --- a/hathor/indexes/memory_address_index.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TYPE_CHECKING, Iterable, Optional - -from structlog import get_logger - -from hathor.indexes.address_index import AddressIndex -from hathor.indexes.memory_tx_group_index import MemoryTxGroupIndex -from hathor.transaction import BaseTransaction - -if TYPE_CHECKING: # pragma: no cover - from hathor.pubsub import PubSubManager - -logger = get_logger() - - -class MemoryAddressIndex(MemoryTxGroupIndex[str], AddressIndex): - """ Index of inputs/outputs by address - """ - - def __init__(self, pubsub: Optional['PubSubManager'] = None) -> None: - super().__init__() - self.pubsub = pubsub - if self.pubsub: - self._subscribe_pubsub_events() - - def get_db_name(self) -> Optional[str]: - return None - - def _extract_keys(self, tx: BaseTransaction) -> Iterable[str]: - return tx.get_related_addresses() - - def add_tx(self, tx: BaseTransaction) -> None: - super().add_tx(tx) - self._publish_tx(tx) - - def get_from_address(self, address: str) -> list[bytes]: - return list(self._get_from_key(address)) - - def get_sorted_from_address(self, address: str, tx_start: Optional[BaseTransaction] = None) -> Iterable[bytes]: - return self._get_sorted_from_key(address, tx_start) - - def is_address_empty(self, address: str) -> bool: - return self._is_key_empty(address) diff --git a/hathor/indexes/memory_height_index.py b/hathor/indexes/memory_height_index.py deleted file mode 100644 index 18a0546ae..000000000 --- a/hathor/indexes/memory_height_index.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -from hathor.conf.settings import HathorSettings -from hathor.indexes.height_index import HeightIndex, HeightInfo, IndexEntry - - -class MemoryHeightIndex(HeightIndex): - """Store the block hash for each given height - """ - - _index: list[IndexEntry] - - def __init__(self, *, settings: HathorSettings | None = None) -> None: - super().__init__(settings=settings) - self.force_clear() - - def get_db_name(self) -> Optional[str]: - return None - - def force_clear(self) -> None: - self._index = [self.get_genesis_block_entry()] - - def _add(self, height: int, block_hash: bytes, timestamp: int, *, can_reorg: bool) -> None: - if len(self._index) < height: - raise ValueError(f'parent hash required (current height: {len(self._index)}, new height: {height})') - elif len(self._index) == height: - self._index.append(IndexEntry(block_hash, timestamp)) - elif self._index[height].hash != block_hash: - if can_reorg: - del self._index[height:] - self._index.append(IndexEntry(block_hash, timestamp)) - else: - self.log.error( - 'adding would cause a re-org', - height=height, - current_block=self._index[height].hash.hex(), - new_block=block_hash.hex() - ) - raise ValueError('adding would cause a re-org, use can_reorg=True to accept re-orgs') - else: - # nothing to do (there are more blocks, but the block at height currently matches the added block) - pass - - def add_new(self, height: int, block_hash: bytes, timestamp: int) -> None: - self._add(height, block_hash, timestamp, can_reorg=False) - - def add_reorg(self, height: int, block_hash: bytes, timestamp: int) -> None: - self._add(height, block_hash, timestamp, can_reorg=True) - - def get(self, height: int) -> Optional[bytes]: - if len(self._index) <= height: - return None - return self._index[height].hash - - def get_tip(self) -> bytes: - return self._index[-1].hash - - def get_height_tip(self) -> HeightInfo: - height = len(self._index) - 1 - return HeightInfo(height, self._index[height].hash) - - def get_n_height_tips(self, n_blocks: int) -> list[HeightInfo]: - if n_blocks < 1: - raise ValueError('n_blocks must be a positive, non-zero, integer') - # highest height that is included, will be the first element - h_high = len(self._index) - 1 - # lowest height that is not included, -1 if it reaches the genesis - h_low = max(h_high - n_blocks, -1) - return [HeightInfo(h, self._index[h].hash) for h in range(h_high, h_low, -1)] diff --git a/hathor/indexes/memory_info_index.py b/hathor/indexes/memory_info_index.py index 656cc7972..d86d93589 100644 --- a/hathor/indexes/memory_info_index.py +++ b/hathor/indexes/memory_info_index.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Optional +from hathor.conf.settings import HathorSettings from hathor.indexes.info_index import InfoIndex from hathor.transaction import BaseTransaction @@ -22,8 +23,8 @@ class MemoryInfoIndex(InfoIndex): - def __init__(self): - super().__init__() + def __init__(self, *, settings: HathorSettings) -> None: + super().__init__(settings=settings) self._block_count = 0 self._tx_count = 0 self._first_timestamp = 0 diff --git a/hathor/indexes/memory_mempool_tips_index.py b/hathor/indexes/memory_mempool_tips_index.py index 564ad3bf6..3373c59fa 100644 --- a/hathor/indexes/memory_mempool_tips_index.py +++ b/hathor/indexes/memory_mempool_tips_index.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.mempool_tips_index import ByteCollectionMempoolTipsIndex logger = get_logger() @@ -24,7 +25,8 @@ class MemoryMempoolTipsIndex(ByteCollectionMempoolTipsIndex): _index: set[bytes] - def __init__(self): + def __init__(self, *, settings: HathorSettings) -> None: + super().__init__(settings=settings) self.log = logger.new() self.force_clear() diff --git a/hathor/indexes/memory_timestamp_index.py b/hathor/indexes/memory_timestamp_index.py deleted file mode 100644 index a6c1c06a0..000000000 --- a/hathor/indexes/memory_timestamp_index.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterator, Optional - -from sortedcontainers import SortedKeyList -from structlog import get_logger - -from hathor.indexes.timestamp_index import RangeIdx, ScopeType, TimestampIndex -from hathor.indexes.utils import ( - TransactionIndexElement, - get_newer_sorted_key_list, - get_newest_sorted_key_list, - get_older_sorted_key_list, -) -from hathor.transaction import BaseTransaction - -logger = get_logger() - - -class MemoryTimestampIndex(TimestampIndex): - """ Index of transactions sorted by their timestamps. - """ - - _index: 'SortedKeyList[TransactionIndexElement]' - - def __init__(self, *, scope_type: ScopeType): - super().__init__(scope_type=scope_type) - self.log = logger.new() - self.force_clear() - - def get_db_name(self) -> Optional[str]: - return None - - def force_clear(self) -> None: - self._index = SortedKeyList(key=lambda x: (x.timestamp, x.hash)) - - def add_tx(self, tx: BaseTransaction) -> bool: - # It is safe to use the in operator because it is O(log(n)). - # http://www.grantjenks.com/docs/sortedcontainers/sortedlist.html#sortedcontainers.SortedList.__contains__ - element = TransactionIndexElement(tx.timestamp, tx.hash) - if element in self._index: - return False - self._index.add(element) - return True - - def del_tx(self, tx: BaseTransaction) -> None: - idx = self._index.bisect_key_left((tx.timestamp, tx.hash)) - if idx < len(self._index) and self._index[idx].hash == tx.hash: - self._index.pop(idx) - - def get_newest(self, count: int) -> tuple[list[bytes], bool]: - return get_newest_sorted_key_list(self._index, count) - - def get_older(self, timestamp: int, hash_bytes: bytes, count: int) -> tuple[list[bytes], bool]: - return get_older_sorted_key_list(self._index, timestamp, hash_bytes, count) - - def get_newer(self, timestamp: int, hash_bytes: bytes, count: int) -> tuple[list[bytes], bool]: - return get_newer_sorted_key_list(self._index, timestamp, hash_bytes, count) - - def get_hashes_and_next_idx(self, from_idx: RangeIdx, count: int) -> tuple[list[bytes], Optional[RangeIdx]]: - timestamp, offset = from_idx - idx = self._index.bisect_key_left((timestamp, b'')) - txs = SortedKeyList(key=lambda x: (x.timestamp, x.hash)) - txs.update(self._index[idx:idx+offset+count]) - ret_txs = txs[offset:offset+count] - hashes = [tx.hash for tx in ret_txs] - if len(ret_txs) < count: - return hashes, None - else: - next_offset = offset + count - next_timestamp = ret_txs[-1].timestamp - if next_timestamp != timestamp: - next_idx = txs.bisect_key_left((next_timestamp, b'')) - next_offset -= next_idx - return hashes, RangeIdx(next_timestamp, next_offset) - - def iter(self) -> Iterator[bytes]: - for element in self._index: - yield element.hash diff --git a/hathor/indexes/memory_tips_index.py b/hathor/indexes/memory_tips_index.py index 58c9f447a..b1b419bfa 100644 --- a/hathor/indexes/memory_tips_index.py +++ b/hathor/indexes/memory_tips_index.py @@ -18,6 +18,7 @@ from intervaltree import Interval, IntervalTree from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.tips_index import ScopeType, TipsIndex from hathor.transaction import BaseTransaction @@ -47,8 +48,8 @@ class MemoryTipsIndex(TipsIndex): # It is useful because the interval tree allows access only by the interval. tx_last_interval: dict[bytes, Interval] - def __init__(self, *, scope_type: ScopeType): - super().__init__(scope_type=scope_type) + def __init__(self, *, scope_type: ScopeType, settings: HathorSettings) -> None: + super().__init__(scope_type=scope_type, settings=settings) self.log = logger.new() self.tree = IntervalTree() self.tx_last_interval = {} diff --git a/hathor/indexes/memory_tokens_index.py b/hathor/indexes/memory_tokens_index.py deleted file mode 100644 index 74e5160af..000000000 --- a/hathor/indexes/memory_tokens_index.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict -from typing import Iterator, Optional, cast - -from sortedcontainers import SortedKeyList -from structlog import get_logger - -from hathor.indexes.tokens_index import TokenIndexInfo, TokensIndex, TokenUtxoInfo -from hathor.indexes.utils import ( - TransactionIndexElement, - get_newer_sorted_key_list, - get_newest_sorted_key_list, - get_older_sorted_key_list, -) -from hathor.transaction import BaseTransaction, Transaction -from hathor.transaction.base_transaction import TxVersion -from hathor.util import is_token_uid_valid - -logger = get_logger() - - -class MemoryTokenIndexInfo(TokenIndexInfo): - _name: Optional[str] - _symbol: Optional[str] - _total: int - _mint: set[TokenUtxoInfo] - _melt: set[TokenUtxoInfo] - _transactions: 'SortedKeyList[TransactionIndexElement]' - - def __init__(self, name: Optional[str] = None, symbol: Optional[str] = None, total: int = 0, - mint: Optional[set[TokenUtxoInfo]] = None, melt: Optional[set[TokenUtxoInfo]] = None) -> None: - self._name = name - self._symbol = symbol - self._total = total - self._mint = mint or set() - self._melt = melt or set() - # Saves the (timestamp, hash) of the transactions that include this token - self._transactions = SortedKeyList(key=lambda x: (x.timestamp, x.hash)) - - def get_name(self) -> Optional[str]: - return self._name - - def get_symbol(self) -> Optional[str]: - return self._symbol - - def get_total(self) -> int: - return self._total - - def iter_mint_utxos(self) -> Iterator[TokenUtxoInfo]: - yield from self._mint - - def iter_melt_utxos(self) -> Iterator[TokenUtxoInfo]: - yield from self._melt - - -class MemoryTokensIndex(TokensIndex): - def __init__(self) -> None: - self.log = logger.new() - self.force_clear() - - def get_db_name(self) -> Optional[str]: - return None - - def force_clear(self) -> None: - self._tokens: dict[bytes, MemoryTokenIndexInfo] = defaultdict(MemoryTokenIndexInfo) - - def _add_to_index(self, tx: BaseTransaction, index: int) -> None: - """ Add tx to mint/melt indexes and total amount - """ - - tx_output = tx.outputs[index] - token_uid = tx.get_token_uid(tx_output.get_token_index()) - - if tx_output.is_token_authority(): - if tx_output.can_mint_token(): - # add to mint index - self._tokens[token_uid]._mint.add(TokenUtxoInfo(tx.hash, index)) - if tx_output.can_melt_token(): - # add to melt index - self._tokens[token_uid]._melt.add(TokenUtxoInfo(tx.hash, index)) - else: - self._tokens[token_uid]._total += tx_output.value - - def _remove_from_index(self, tx: BaseTransaction, index: int) -> None: - """ Remove tx from mint/melt indexes and total amount - """ - - tx_output = tx.outputs[index] - token_uid = tx.get_token_uid(tx_output.get_token_index()) - - if tx_output.is_token_authority(): - if tx_output.can_mint_token(): - # remove from mint index - self._tokens[token_uid]._mint.discard(TokenUtxoInfo(tx.hash, index)) - if tx_output.can_melt_token(): - # remove from melt index - self._tokens[token_uid]._melt.discard(TokenUtxoInfo(tx.hash, index)) - else: - self._tokens[token_uid]._total -= tx_output.value - - def add_tx(self, tx: BaseTransaction) -> None: - for tx_input in tx.inputs: - spent_tx = tx.get_spent_tx(tx_input) - self._remove_from_index(spent_tx, tx_input.index) - - for index in range(len(tx.outputs)): - self._add_to_index(tx, index) - - # if it's a TokenCreationTransaction, update name and symbol - if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: - from hathor.transaction.token_creation_tx import TokenCreationTransaction - tx = cast(TokenCreationTransaction, tx) - status = self._tokens[tx.hash] - status._name = tx.token_name - status._symbol = tx.token_symbol - - if tx.is_transaction: - # Adding this tx to the transactions key list - assert isinstance(tx, Transaction) - for token_uid in tx.tokens: - transactions = self._tokens[token_uid]._transactions - # It is safe to use the in operator because it is O(log(n)). - # http://www.grantjenks.com/docs/sortedcontainers/sortedlist.html#sortedcontainers.SortedList.__contains__ - element = TransactionIndexElement(tx.timestamp, tx.hash) - if element in transactions: - return - transactions.add(element) - - def remove_tx(self, tx: BaseTransaction) -> None: - for tx_input in tx.inputs: - spent_tx = tx.get_spent_tx(tx_input) - self._add_to_index(spent_tx, tx_input.index) - - for index in range(len(tx.outputs)): - self._remove_from_index(tx, index) - - if tx.is_transaction: - # Removing this tx from the transactions key list - assert isinstance(tx, Transaction) - for token_uid in tx.tokens: - transactions = self._tokens[token_uid]._transactions - idx = transactions.bisect_key_left((tx.timestamp, tx.hash)) - if idx < len(transactions) and transactions[idx].hash == tx.hash: - transactions.pop(idx) - - # if it's a TokenCreationTransaction, remove it from index - if tx.version == TxVersion.TOKEN_CREATION_TRANSACTION: - del self._tokens[tx.hash] - - def iter_all_tokens(self) -> Iterator[tuple[bytes, TokenIndexInfo]]: - yield from self._tokens.items() - - def get_token_info(self, token_uid: bytes) -> TokenIndexInfo: - assert is_token_uid_valid(token_uid) - if token_uid not in self._tokens: - raise KeyError('unknown token') - info = self._tokens[token_uid] - return info - - def get_transactions_count(self, token_uid: bytes) -> int: - assert is_token_uid_valid(token_uid) - if token_uid not in self._tokens: - return 0 - info = self._tokens[token_uid] - return len(info._transactions) - - def get_newest_transactions(self, token_uid: bytes, count: int) -> tuple[list[bytes], bool]: - assert is_token_uid_valid(token_uid) - if token_uid not in self._tokens: - return [], False - transactions = self._tokens[token_uid]._transactions - return get_newest_sorted_key_list(transactions, count) - - def get_older_transactions(self, token_uid: bytes, timestamp: int, hash_bytes: bytes, count: int - ) -> tuple[list[bytes], bool]: - assert is_token_uid_valid(token_uid) - if token_uid not in self._tokens: - return [], False - transactions = self._tokens[token_uid]._transactions - return get_older_sorted_key_list(transactions, timestamp, hash_bytes, count) - - def get_newer_transactions(self, token_uid: bytes, timestamp: int, hash_bytes: bytes, count: int - ) -> tuple[list[bytes], bool]: - assert is_token_uid_valid(token_uid) - if token_uid not in self._tokens: - return [], False - transactions = self._tokens[token_uid]._transactions - return get_newer_sorted_key_list(transactions, timestamp, hash_bytes, count) diff --git a/hathor/indexes/memory_tx_group_index.py b/hathor/indexes/memory_tx_group_index.py deleted file mode 100644 index 99a679f21..000000000 --- a/hathor/indexes/memory_tx_group_index.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import abstractmethod -from collections import defaultdict -from typing import Iterable, Optional, Sized, TypeVar - -from structlog import get_logger - -from hathor.indexes.tx_group_index import TxGroupIndex -from hathor.transaction import BaseTransaction - -logger = get_logger() - -KT = TypeVar('KT', bound=Sized) - - -class MemoryTxGroupIndex(TxGroupIndex[KT]): - """Memory implementation of the TxGroupIndex. This class is abstract and cannot be used directly. - """ - - index: defaultdict[KT, set[tuple[int, bytes]]] - - def __init__(self) -> None: - self.force_clear() - - def force_clear(self) -> None: - self.index = defaultdict(set) - - def _add_tx(self, key: KT, tx: BaseTransaction) -> None: - self.index[key].add((tx.timestamp, tx.hash)) - - @abstractmethod - def _extract_keys(self, tx: BaseTransaction) -> Iterable[KT]: - """Extract the keys related to a given tx. The transaction will be added to all extracted keys.""" - raise NotImplementedError - - def add_tx(self, tx: BaseTransaction) -> None: - - for key in self._extract_keys(tx): - self._add_tx(key, tx) - - def remove_tx(self, tx: BaseTransaction) -> None: - - for key in self._extract_keys(tx): - self.index[key].discard((tx.timestamp, tx.hash)) - - def _get_from_key(self, key: KT) -> Iterable[bytes]: - for _, h in self.index[key]: - yield h - - def _get_sorted_from_key(self, key: KT, tx_start: Optional[BaseTransaction] = None) -> Iterable[bytes]: - sorted_elements = sorted(self.index[key]) - found = False - for _, h in sorted_elements: - if tx_start and h == tx_start.hash: - found = True - - if found or not tx_start: - yield h - - def _is_key_empty(self, key: KT) -> bool: - return not bool(self.index[key]) diff --git a/hathor/indexes/memory_utxo_index.py b/hathor/indexes/memory_utxo_index.py deleted file mode 100644 index ff1872800..000000000 --- a/hathor/indexes/memory_utxo_index.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict -from dataclasses import dataclass, field -from typing import Iterator, NamedTuple, Optional, Union - -from sortedcontainers import SortedSet -from structlog import get_logger - -from hathor.indexes.utxo_index import UtxoIndex, UtxoIndexItem - -logger = get_logger() - - -class _IndexKey(NamedTuple): - token_uid: bytes - address: str - - -class _NoLockItem(NamedTuple): - amount: int - tx_id: bytes - # XXX: using idx instead of index because `def index` exists in parent class - idx: int - - -class _TimeLockItem(NamedTuple): - timelock: int - amount: int - tx_id: bytes - # XXX: using idx instead of index because `def index` exists in parent class - idx: int - - -class _HeightLockItem(NamedTuple): - heightlock: int - amount: int - tx_id: bytes - # XXX: using idx instead of index because `def index` exists in parent class - idx: int - - -@dataclass(frozen=True) -class _IndexItem: - nolock: 'SortedSet[_NoLockItem]' = field(default_factory=SortedSet) - timelock: 'SortedSet[_TimeLockItem]' = field(default_factory=SortedSet) - heightlock: 'SortedSet[_HeightLockItem]' = field(default_factory=SortedSet) - - -class MemoryUtxoIndex(UtxoIndex): - _index: defaultdict[_IndexKey, _IndexItem] - - def __init__(self): - super().__init__() - self._index = defaultdict(_IndexItem) - - def get_db_name(self) -> Optional[str]: - return None - - def force_clear(self) -> None: - self._index.clear() - - def _add_utxo(self, item: UtxoIndexItem) -> None: - self.log.debug('add utxo', item=item) - subindex = self._index[_IndexKey(item.token_uid, item.address)] - if item.timelock is not None: - subindex.timelock.add(_TimeLockItem(item.timelock, item.amount, item.tx_id, item.index)) - elif item.heightlock is not None: - subindex.heightlock.add(_HeightLockItem(item.heightlock, item.amount, item.tx_id, item.index)) - else: - subindex.nolock.add(_NoLockItem(item.amount, item.tx_id, item.index)) - - def _remove_utxo(self, item: UtxoIndexItem) -> None: - self.log.debug('del utxo', item=item) - subindex = self._index[_IndexKey(item.token_uid, item.address)] - if item.timelock is not None: - subindex.timelock.discard(_TimeLockItem(item.timelock, item.amount, item.tx_id, item.index)) - elif item.heightlock is not None: - subindex.heightlock.discard(_HeightLockItem(item.heightlock, item.amount, item.tx_id, item.index)) - else: - subindex.nolock.discard(_NoLockItem(item.amount, item.tx_id, item.index)) - - def _iter_utxos_nolock(self, *, token_uid: bytes, address: str, target_amount: int) -> Iterator[UtxoIndexItem]: - subindex = self._index[_IndexKey(token_uid, address)].nolock - # this will point to the next value that is equal or higher than target_amount - idx_next_amount = subindex.bisect((target_amount,)) + 1 - for i in subindex.islice(stop=idx_next_amount, reverse=True): - yield UtxoIndexItem(token_uid, i.tx_id, i.idx, address, i.amount, None, None) - - def _iter_utxos_timelock(self, *, token_uid: bytes, address: str, target_amount: int, - target_timestamp: Optional[int] = None) -> Iterator[UtxoIndexItem]: - import math - seek_timestamp: Union[int, float] - if target_timestamp is None: - seek_timestamp = math.inf - else: - seek_timestamp = target_timestamp - subindex = self._index[_IndexKey(token_uid, address)].timelock - # this will point to the next value that is equal or higher than target_amount - idx_next_amount = subindex.bisect((seek_timestamp, target_amount)) + 1 - for i in subindex.islice(stop=idx_next_amount, reverse=True): - # it might happen that the first one is out of the timestamp range - if i.timelock > seek_timestamp: - continue - yield UtxoIndexItem(token_uid, i.tx_id, i.idx, address, i.amount, i.timelock, None) - - def _iter_utxos_heightlock(self, *, token_uid: bytes, address: str, target_amount: int, - target_height: Optional[int] = None) -> Iterator[UtxoIndexItem]: - import math - seek_height: Union[int, float] - if target_height is None: - seek_height = math.inf - else: - seek_height = target_height - subindex = self._index[_IndexKey(token_uid, address)].heightlock - # this will point to the next value that is equal or higher than target_amount - idx_next_amount = subindex.bisect((seek_height, target_amount)) + 1 - for i in subindex.islice(stop=idx_next_amount, reverse=True): - # it might happen that the first one is out of the heightlock range - if i.heightlock > seek_height: - continue - yield UtxoIndexItem(token_uid, i.tx_id, i.idx, address, i.amount, None, i.heightlock) diff --git a/hathor/indexes/partial_rocksdb_tips_index.py b/hathor/indexes/partial_rocksdb_tips_index.py index 4a0d83c6d..7eca9a7bd 100644 --- a/hathor/indexes/partial_rocksdb_tips_index.py +++ b/hathor/indexes/partial_rocksdb_tips_index.py @@ -18,6 +18,7 @@ from intervaltree import Interval, IntervalTree from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.memory_tips_index import MemoryTipsIndex from hathor.indexes.rocksdb_utils import RocksDBIndexUtils from hathor.indexes.tips_index import ScopeType @@ -61,8 +62,8 @@ class PartialRocksDBTipsIndex(MemoryTipsIndex, RocksDBIndexUtils): # It is useful because the interval tree allows access only by the interval. tx_last_interval: dict[bytes, Interval] - def __init__(self, db: 'rocksdb.DB', *, scope_type: ScopeType): - MemoryTipsIndex.__init__(self, scope_type=scope_type) + def __init__(self, db: 'rocksdb.DB', *, scope_type: ScopeType, settings: HathorSettings) -> None: + MemoryTipsIndex.__init__(self, scope_type=scope_type, settings=settings) self._name = scope_type.get_name() self.log = logger.new() # XXX: override MemoryTipsIndex logger so it shows the correct module RocksDBIndexUtils.__init__(self, db, f'tips-{self._name}'.encode()) diff --git a/hathor/indexes/rocksdb_address_index.py b/hathor/indexes/rocksdb_address_index.py index cd7f78096..0cc829abc 100644 --- a/hathor/indexes/rocksdb_address_index.py +++ b/hathor/indexes/rocksdb_address_index.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.address_index import AddressIndex from hathor.indexes.rocksdb_tx_group_index import RocksDBTxGroupIndex from hathor.indexes.rocksdb_utils import RocksDBIndexUtils @@ -38,9 +39,10 @@ class RocksDBAddressIndex(RocksDBTxGroupIndex[str], AddressIndex, RocksDBIndexUt _KEY_SIZE = 34 - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None, + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None, pubsub: Optional['PubSubManager'] = None) -> None: RocksDBTxGroupIndex.__init__(self, db, cf_name or _CF_NAME_ADDRESS_INDEX) + AddressIndex.__init__(self, settings=settings) self.pubsub = pubsub if self.pubsub: diff --git a/hathor/indexes/rocksdb_height_index.py b/hathor/indexes/rocksdb_height_index.py index 562bbf43c..eee0470b3 100644 --- a/hathor/indexes/rocksdb_height_index.py +++ b/hathor/indexes/rocksdb_height_index.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.height_index import HeightIndex, HeightInfo, IndexEntry from hathor.indexes.rocksdb_utils import RocksDBIndexUtils @@ -42,9 +43,9 @@ class RocksDBHeightIndex(HeightIndex, RocksDBIndexUtils): It works nicely because rocksdb uses a tree sorted by key under the hood. """ - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None) -> None: + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None) -> None: self.log = logger.new() - HeightIndex.__init__(self) + HeightIndex.__init__(self, settings=settings) RocksDBIndexUtils.__init__(self, db, cf_name or _CF_NAME_HEIGHT_INDEX) def get_db_name(self) -> Optional[str]: diff --git a/hathor/indexes/rocksdb_info_index.py b/hathor/indexes/rocksdb_info_index.py index 6b6025146..093f13c83 100644 --- a/hathor/indexes/rocksdb_info_index.py +++ b/hathor/indexes/rocksdb_info_index.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.memory_info_index import MemoryInfoIndex from hathor.indexes.rocksdb_utils import RocksDBIndexUtils from hathor.transaction import BaseTransaction @@ -37,10 +38,10 @@ class RocksDBInfoIndex(MemoryInfoIndex, RocksDBIndexUtils): - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None) -> None: + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None) -> None: self.log = logger.new() RocksDBIndexUtils.__init__(self, db, cf_name or _CF_NAME_ADDRESS_INDEX) - MemoryInfoIndex.__init__(self) + MemoryInfoIndex.__init__(self, settings=settings) def init_start(self, indexes_manager: 'IndexesManager') -> None: self._load_all_values() diff --git a/hathor/indexes/rocksdb_mempool_tips_index.py b/hathor/indexes/rocksdb_mempool_tips_index.py index a2c6c7ffe..8ff96447c 100644 --- a/hathor/indexes/rocksdb_mempool_tips_index.py +++ b/hathor/indexes/rocksdb_mempool_tips_index.py @@ -16,6 +16,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.mempool_tips_index import ByteCollectionMempoolTipsIndex from hathor.indexes.rocksdb_utils import RocksDBSimpleSet @@ -31,7 +32,8 @@ class RocksDBMempoolTipsIndex(ByteCollectionMempoolTipsIndex): _index: RocksDBSimpleSet - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None) -> None: + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None) -> None: + super().__init__(settings=settings) self.log = logger.new() _cf_name = cf_name or _CF_NAME_MEMPOOL_TIPS_INDEX self._index = RocksDBSimpleSet(db, self.log, cf_name=_cf_name) diff --git a/hathor/indexes/rocksdb_timestamp_index.py b/hathor/indexes/rocksdb_timestamp_index.py index c505820a3..a519ba11b 100644 --- a/hathor/indexes/rocksdb_timestamp_index.py +++ b/hathor/indexes/rocksdb_timestamp_index.py @@ -16,10 +16,11 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.rocksdb_utils import RocksDBIndexUtils, incr_key -from hathor.indexes.timestamp_index import RangeIdx, ScopeType, TimestampIndex +from hathor.indexes.timestamp_index import ScopeType, TimestampIndex from hathor.transaction import BaseTransaction -from hathor.util import collect_n, skip_n +from hathor.util import collect_n if TYPE_CHECKING: # pragma: no cover import rocksdb @@ -38,8 +39,8 @@ class RocksDBTimestampIndex(TimestampIndex, RocksDBIndexUtils): It works nicely because rocksdb uses a tree sorted by key under the hood. """ - def __init__(self, db: 'rocksdb.DB', *, scope_type: ScopeType): - TimestampIndex.__init__(self, scope_type=scope_type) + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, scope_type: ScopeType) -> None: + TimestampIndex.__init__(self, scope_type=scope_type, settings=settings) self._name = scope_type.get_name() self.log = logger.new() RocksDBIndexUtils.__init__(self, db, f'timestamp-sorted-{self._name}'.encode()) @@ -135,30 +136,6 @@ def get_newer(self, timestamp: int, hash_bytes: bytes, count: int) -> tuple[list it = (x for _, x in self._iter(timestamp, hash_bytes)) return collect_n(it, count) - def get_hashes_and_next_idx(self, from_idx: RangeIdx, count: int) -> tuple[list[bytes], Optional[RangeIdx]]: - if count <= 0: - raise ValueError(f'count must be positive, got {count}') - timestamp, offset = from_idx - it = skip_n(self._iter(timestamp), offset) - hashes: list[bytes] = [] - n = count - next_timestamp = timestamp - next_offset = offset - while n > 0: - try: - timestamp, tx_hash = next(it) - except StopIteration: - return hashes, None - hashes.append(tx_hash) - if next_timestamp != timestamp: - # XXX: this is to match how the memory index works, it basically resets to 1, not 0 - next_offset = 1 - next_timestamp = timestamp - else: - next_offset += 1 - n -= 1 - return hashes, RangeIdx(next_timestamp, next_offset) - def iter(self) -> Iterator[bytes]: it = self._db.iterkeys(self._cf) it.seek_to_first() diff --git a/hathor/indexes/rocksdb_tokens_index.py b/hathor/indexes/rocksdb_tokens_index.py index 198e26463..72e85ef91 100644 --- a/hathor/indexes/rocksdb_tokens_index.py +++ b/hathor/indexes/rocksdb_tokens_index.py @@ -18,7 +18,7 @@ from structlog import get_logger -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.indexes.rocksdb_utils import ( InternalUid, RocksDBIndexUtils, @@ -84,9 +84,9 @@ class RocksDBTokensIndex(TokensIndex, RocksDBIndexUtils): It works nicely because rocksdb uses a tree sorted by key under the hood. """ - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None) -> None: - self._settings = get_global_settings() + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None) -> None: self.log = logger.new() + TokensIndex.__init__(self, settings=settings) RocksDBIndexUtils.__init__(self, db, cf_name or _CF_NAME_TOKENS_INDEX) def get_db_name(self) -> Optional[str]: diff --git a/hathor/indexes/rocksdb_utils.py b/hathor/indexes/rocksdb_utils.py index 431bfc2f6..c6f372136 100644 --- a/hathor/indexes/rocksdb_utils.py +++ b/hathor/indexes/rocksdb_utils.py @@ -120,6 +120,16 @@ def _clone_into_dict(self) -> dict[bytes, bytes]: it.seek_to_first() return {k: v for (_, k), v in it} + def get_all_internal(self) -> Iterable[bytes]: + """ + Return all internal content of this index, sorted — that is, its rocksdb keys. + Mostly useful for comparing different index instances in tests. + """ + it = self._db.iterkeys(self._cf) + it.seek_to_first() + for _cf, rocksdb_key in it: + yield rocksdb_key + class RocksDBSimpleSet(Collection[bytes], RocksDBIndexUtils): def __init__(self, db: 'rocksdb.DB', log: 'structlog.stdlib.BoundLogger', *, cf_name: bytes) -> None: diff --git a/hathor/indexes/rocksdb_utxo_index.py b/hathor/indexes/rocksdb_utxo_index.py index 923530ffb..5aa44c93b 100644 --- a/hathor/indexes/rocksdb_utxo_index.py +++ b/hathor/indexes/rocksdb_utxo_index.py @@ -19,6 +19,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.crypto.util import decode_address, get_address_b58_from_bytes from hathor.indexes.rocksdb_utils import InternalUid, RocksDBIndexUtils, from_internal_token_uid, to_internal_token_uid from hathor.indexes.utxo_index import UtxoIndex, UtxoIndexItem @@ -311,8 +312,8 @@ class RocksDBUtxoIndex(UtxoIndex, RocksDBIndexUtils): It works nicely because rocksdb uses a tree sorted by key under the hood. """ - def __init__(self, db: 'rocksdb.DB', *, cf_name: Optional[bytes] = None) -> None: - super().__init__() + def __init__(self, db: 'rocksdb.DB', *, settings: HathorSettings, cf_name: Optional[bytes] = None) -> None: + super().__init__(settings=settings) self.log = logger.new() RocksDBIndexUtils.__init__(self, db, cf_name or _CF_NAME_UTXO_INDEX) diff --git a/hathor/indexes/timestamp_index.py b/hathor/indexes/timestamp_index.py index 76d15a1d7..765238cca 100644 --- a/hathor/indexes/timestamp_index.py +++ b/hathor/indexes/timestamp_index.py @@ -14,10 +14,11 @@ from abc import abstractmethod from enum import Enum -from typing import Iterator, NamedTuple, Optional +from typing import Iterator, NamedTuple from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.base_index import BaseIndex from hathor.indexes.scope import Scope from hathor.transaction import BaseTransaction @@ -55,7 +56,8 @@ class TimestampIndex(BaseIndex): """ Index of transactions sorted by their timestamps. """ - def __init__(self, *, scope_type: ScopeType): + def __init__(self, *, scope_type: ScopeType, settings: HathorSettings) -> None: + super().__init__(settings=settings) self._scope_type = scope_type def get_scope(self) -> Scope: @@ -112,12 +114,6 @@ def get_newer(self, timestamp: int, hash_bytes: bytes, count: int) -> tuple[list """ raise NotImplementedError - @abstractmethod - def get_hashes_and_next_idx(self, from_idx: RangeIdx, count: int) -> tuple[list[bytes], Optional[RangeIdx]]: - """ Get up to count hashes if available and the next range-index, this is used by sync-v1. - """ - raise NotImplementedError - @abstractmethod def iter(self) -> Iterator[bytes]: """ Iterate over the transactions in the index order, that is, sorted by timestamp. diff --git a/hathor/indexes/tips_index.py b/hathor/indexes/tips_index.py index 992745b52..6472d6301 100644 --- a/hathor/indexes/tips_index.py +++ b/hathor/indexes/tips_index.py @@ -18,6 +18,7 @@ from intervaltree import Interval from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.base_index import BaseIndex from hathor.indexes.scope import Scope from hathor.transaction import BaseTransaction @@ -60,7 +61,8 @@ class TipsIndex(BaseIndex): TODO Use an interval tree stored in disk, possibly using a B-tree. """ - def __init__(self, *, scope_type: ScopeType): + def __init__(self, *, scope_type: ScopeType, settings: HathorSettings) -> None: + super().__init__(settings=settings) self._scope_type = scope_type def get_scope(self) -> Scope: diff --git a/hathor/indexes/utils.py b/hathor/indexes/utils.py deleted file mode 100644 index 949e59c0e..000000000 --- a/hathor/indexes/utils.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import NamedTuple - -from sortedcontainers import SortedKeyList -from structlog import get_logger - -logger = get_logger() - - -class TransactionIndexElement(NamedTuple): - timestamp: int - hash: bytes - - -def get_newest_sorted_key_list(key_list: 'SortedKeyList[TransactionIndexElement]', count: int - ) -> tuple[list[bytes], bool]: - """ Get newest data from a sorted key list - Return the elements (quantity is the 'count' parameter) and a boolean indicating if has more - """ - if count < 0: - raise ValueError(f'count must be non-negative, got {count}') - # XXX: count=0 is supported, this if prevents doing key_list[-0:] which would return all transactions - if count == 0: - return [], False - newest = key_list[-count:] - newest.reverse() - if count >= len(key_list): - has_more = False - else: - has_more = True - return [tx_index.hash for tx_index in newest], has_more - - -def get_older_sorted_key_list(key_list: 'SortedKeyList[TransactionIndexElement]', timestamp: int, - hash_bytes: bytes, count: int) -> tuple[list[bytes], bool]: - """ Get sorted key list data from the timestamp/hash_bytes reference to the oldest - Return the elements (quantity is the 'count' parameter) and a boolean indicating if has more - """ - if count < 0: - raise ValueError(f'count must be non-negative, got {count}') - # Get idx of element - idx = key_list.bisect_key_left((timestamp, hash_bytes)) - first_idx = max(0, idx - count) - txs = key_list[first_idx:idx] - # Reverse because we want the newest first - txs.reverse() - return [tx_index.hash for tx_index in txs], first_idx > 0 - - -def get_newer_sorted_key_list(key_list: 'SortedKeyList[TransactionIndexElement]', timestamp: int, - hash_bytes: bytes, count: int) -> tuple[list[bytes], bool]: - """ Get sorted key list data from the timestamp/hash_bytes reference to the newest - Return the elements (quantity is the 'count' parameter) and a boolean indicating if has more - """ - if count < 0: - raise ValueError(f'count must be non-negative, got {count}') - # Get idx of element - idx = key_list.bisect_key_left((timestamp, hash_bytes)) - last_idx = min(len(key_list), idx + 1 + count) - txs = key_list[idx + 1:last_idx] - # Reverse because we want the newest first - txs.reverse() - return [tx_index.hash for tx_index in txs], last_idx < len(key_list) diff --git a/hathor/indexes/utxo_index.py b/hathor/indexes/utxo_index.py index bfdc0df78..8b5dcde93 100644 --- a/hathor/indexes/utxo_index.py +++ b/hathor/indexes/utxo_index.py @@ -19,6 +19,7 @@ from structlog import get_logger from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.indexes.base_index import BaseIndex from hathor.indexes.scope import Scope from hathor.transaction import BaseTransaction, Block, TxOutput @@ -107,7 +108,8 @@ class UtxoIndex(BaseIndex): address can be extracted from. """ - def __init__(self): + def __init__(self, *, settings: HathorSettings) -> None: + super().__init__(settings=settings) self.log = logger.new() # interface methods provided by the base class diff --git a/hathor/simulator/simulator.py b/hathor/simulator/simulator.py index ce5730520..2eee437f7 100644 --- a/hathor/simulator/simulator.py +++ b/hathor/simulator/simulator.py @@ -82,7 +82,6 @@ def get_default_builder(self) -> Builder: .set_peer(PrivatePeer.auto_generated()) \ .set_soft_voided_tx_ids(set()) \ .enable_sync_v2() \ - .use_memory() \ .set_settings(self.settings) def create_peer(self, builder: Optional[Builder] = None) -> HathorManager: diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 9f5f5a06d..83bddb78a 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -336,18 +336,16 @@ def set_feature_state(self, *, feature: Feature, state: FeatureState, save: bool """ previous_state = self.get_feature_state(feature=feature) - if state == previous_state: - return - - assert previous_state is None - assert self.storage is not None - - metadata = self.get_metadata() - feature_states = metadata.feature_states or {} - feature_states[feature] = state - metadata.feature_states = feature_states + if state != previous_state: + # we are settings the state for the first time in this block + assert previous_state is None + metadata = self.get_metadata() + feature_states = metadata.feature_states or {} + feature_states[feature] = state + metadata.feature_states = feature_states if save: + assert self.storage is not None self.storage.save_transaction(self, only_metadata=True) def get_feature_activation_bit_value(self, bit: int) -> int: diff --git a/hathor/transaction/static_metadata.py b/hathor/transaction/static_metadata.py index 03855479a..2e61ea17d 100644 --- a/hathor/transaction/static_metadata.py +++ b/hathor/transaction/static_metadata.py @@ -101,7 +101,7 @@ def create( height=height, min_height=min_height, feature_activation_bit_counts=feature_activation_bit_counts, - feature_states={}, # This will be populated in a future PR + feature_states={}, # This will be populated in a future PR, it's currently still in normal metadata ) @staticmethod diff --git a/hathor/transaction/storage/__init__.py b/hathor/transaction/storage/__init__.py index 0238d8e9a..c0a060722 100644 --- a/hathor/transaction/storage/__init__.py +++ b/hathor/transaction/storage/__init__.py @@ -13,14 +13,12 @@ # limitations under the License. from hathor.transaction.storage.cache_storage import TransactionCacheStorage -from hathor.transaction.storage.memory_storage import TransactionMemoryStorage from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage from hathor.transaction.storage.transaction_storage import TransactionStorage from hathor.transaction.storage.vertex_storage_protocol import VertexStorageProtocol __all__ = [ 'TransactionStorage', - 'TransactionMemoryStorage', 'TransactionCacheStorage', 'TransactionRocksDBStorage', 'VertexStorageProtocol' diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index 322dd91d0..e69de29bb 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -1,129 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TYPE_CHECKING, Any, Iterator, Optional, TypeVar - -from typing_extensions import override - -from hathor.indexes import IndexesManager -from hathor.transaction import BaseTransaction -from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.migrations import MigrationState -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage -from hathor.transaction.transaction_metadata import TransactionMetadata - -if TYPE_CHECKING: - from hathor.conf.settings import HathorSettings - -_Clonable = TypeVar('_Clonable', BaseTransaction, TransactionMetadata) - - -class TransactionMemoryStorage(BaseTransactionStorage): - def __init__( - self, - indexes: Optional[IndexesManager] = None, - *, - settings: 'HathorSettings', - _clone_if_needed: bool = False, - ) -> None: - """ - :param _clone_if_needed: *private parameter*, defaults to True, controls whether to clone - transaction/blocks/metadata when returning those objects. - :type _clone_if_needed: bool - """ - self.transactions: dict[bytes, BaseTransaction] = {} - self.metadata: dict[bytes, TransactionMetadata] = {} - # Store custom key/value attributes - self.attributes: dict[str, Any] = {} - self._clone_if_needed = _clone_if_needed - super().__init__(indexes=indexes, settings=settings) - - def _check_and_set_network(self) -> None: - # XXX: does not apply to memory storage, can safely be ignored - pass - - def _check_and_apply_migrations(self): - # XXX: does not apply to memory storage, can safely be ignored - pass - - def _clone(self, x: _Clonable) -> _Clonable: - if self._clone_if_needed: - return x.clone() - else: - return x - - def get_migration_state(self, migration_name: str) -> MigrationState: - # XXX: it will always return COMPLETED, migrations don't apply to memory storage - return MigrationState.COMPLETED - - def set_migration_state(self, migration_name: str, state: MigrationState) -> None: - # XXX: do nothing, migrations have no effect on memory storage - pass - - def remove_transaction(self, tx: BaseTransaction) -> None: - super().remove_transaction(tx) - self.transactions.pop(tx.hash, None) - self.metadata.pop(tx.hash, None) - - def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - super().save_transaction(tx, only_metadata=only_metadata) - self._save_transaction(tx, only_metadata=only_metadata) - - def _save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: - if not only_metadata: - self.transactions[tx.hash] = self._clone(tx) - meta = getattr(tx, '_metadata', None) - if meta: - self.metadata[tx.hash] = self._clone(meta) - - @override - def _save_static_metadata(self, tx: BaseTransaction) -> None: - # We do not need to explicitly save the static metadata as the tx object already holds it in memory - pass - - def transaction_exists(self, hash_bytes: bytes) -> bool: - return hash_bytes in self.transactions - - def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: - if hash_bytes in self.transactions: - tx = self._clone(self.transactions[hash_bytes]) - if hash_bytes in self.metadata: - tx._metadata = self._clone(self.metadata[hash_bytes]) - assert tx._metadata is not None - assert tx._static_metadata is not None - return tx - else: - raise TransactionDoesNotExist(hash_bytes.hex()) - - def _get_all_transactions(self) -> Iterator[BaseTransaction]: - for tx in self.transactions.values(): - tx = self._clone(tx) - if tx.hash in self.metadata: - tx._metadata = self._clone(self.metadata[tx.hash]) - yield tx - - def _get_local_vertices_count(self) -> int: - return len(self.transactions) - - def is_empty(self) -> bool: - return self._get_local_vertices_count() <= 3 - - def add_value(self, key: str, value: str) -> None: - self.attributes[key] = value - - def remove_value(self, key: str) -> None: - self.attributes.pop(key, None) - - def get_value(self, key: str) -> Optional[str]: - return self.attributes.get(key) diff --git a/tests/cli/test_db_export.py b/tests/cli/test_db_export.py index c4bfbb027..89c767005 100644 --- a/tests/cli/test_db_export.py +++ b/tests/cli/test_db_export.py @@ -8,5 +8,5 @@ class TestDbExport(unittest.TestCase): def test_db_export(self): tmp_dir = self.mkdtemp() tmp_file = os.path.join(tmp_dir, 'test_file') - db_export = DbExport(argv=['--memory-storage', '--export-file', tmp_file]) + db_export = DbExport(argv=['--temp-data', '--export-file', tmp_file]) assert db_export is not None diff --git a/tests/cli/test_db_import.py b/tests/cli/test_db_import.py index 8be9cfcaf..a7e4ece39 100644 --- a/tests/cli/test_db_import.py +++ b/tests/cli/test_db_import.py @@ -7,5 +7,5 @@ class TestDbImport(unittest.TestCase): def test_db_import(self): _, tmp_file = tempfile.mkstemp() - db_import = DbImport(argv=['--memory-storage', '--import-file', tmp_file]) + db_import = DbImport(argv=['--temp-data', '--import-file', tmp_file]) assert db_import is not None diff --git a/tests/cli/test_quick_test.py b/tests/cli/test_quick_test.py index 9257d6f7b..1d3d866c4 100644 --- a/tests/cli/test_quick_test.py +++ b/tests/cli/test_quick_test.py @@ -11,7 +11,7 @@ def start_manager(self) -> None: def register_signal_handlers(self) -> None: pass - quick_test = CustomQuickTest(argv=['--memory-storage', '--no-wait']) + quick_test = CustomQuickTest(argv=['--temp-data', '--no-wait']) assert quick_test is not None self.clean_pending(required_to_quiesce=False) diff --git a/tests/cli/test_run_node.py b/tests/cli/test_run_node.py index 84d73d2ef..57a71e649 100644 --- a/tests/cli/test_run_node.py +++ b/tests/cli/test_run_node.py @@ -7,7 +7,7 @@ class RunNodeTest(unittest.TestCase): # In this case we just want to go through the code to see if it's okay - def test_memory_storage(self): + def test_temp_data(self): class CustomRunNode(RunNode): def start_manager(self) -> None: pass @@ -15,7 +15,7 @@ def start_manager(self) -> None: def register_signal_handlers(self) -> None: pass - run_node = CustomRunNode(argv=['--memory-storage']) + run_node = CustomRunNode(argv=['--temp-data']) self.assertTrue(run_node is not None) @patch('twisted.internet.reactor.listenTCP') @@ -28,7 +28,7 @@ def start_manager(self) -> None: def register_signal_handlers(self) -> None: pass - run_node = CustomRunNode(argv=['--memory-storage', '--status', '1234']) + run_node = CustomRunNode(argv=['--temp-data', '--status', '1234']) self.assertTrue(run_node is not None) mock_listenTCP.assert_called_with(1234, ANY) @@ -43,7 +43,7 @@ def start_manager(self) -> None: def register_signal_handlers(self) -> None: pass - run_node = CustomRunNode(argv=['--memory-storage', '--x-enable-ipv6', '--status', '1234']) + run_node = CustomRunNode(argv=['--temp-data', '--x-enable-ipv6', '--status', '1234']) self.assertTrue(run_node is not None) mock_listenTCP.assert_called_with(1234, ANY, interface='::0') @@ -59,4 +59,4 @@ def register_signal_handlers(self) -> None: # Should call system exit with self.assertRaises(SystemExit): - CustomRunNode(argv=['--memory-storage', '--x-disable-ipv4', '--status', '1234']) + CustomRunNode(argv=['--temp-data', '--x-disable-ipv4', '--status', '1234']) diff --git a/tests/cli/test_shell.py b/tests/cli/test_shell.py index d85d4cfa5..b446bcf00 100644 --- a/tests/cli/test_shell.py +++ b/tests/cli/test_shell.py @@ -7,8 +7,8 @@ class ShellTest(unittest.TestCase): # In this case we just want to go through the code to see if it's okay - def test_shell_execution_memory_storage(self): - shell = Shell(argv=['--memory-storage', '--', '--extra-arg']) + def test_shell_execution_temp_data(self): + shell = Shell(argv=['--temp-data', '--', '--extra-arg']) self.assertTrue(shell is not None) def test_shell_execution_default_storage(self): diff --git a/tests/cli/test_sysctl_init.py b/tests/cli/test_sysctl_init.py index 2063d7f76..d7a3e5a03 100644 --- a/tests/cli/test_sysctl_init.py +++ b/tests/cli/test_sysctl_init.py @@ -132,7 +132,7 @@ def register_signal_handlers(self) -> None: run_node = CustomRunNode(argv=[ '--sysctl', 'tcp:8181', '--sysctl-init-file', sysctl_init_file_path, # relative to src/hathor - '--memory-storage', + '--temp-data', ]) self.assertTrue(run_node is not None) conn = run_node.manager.connections @@ -192,7 +192,7 @@ def register_signal_handlers(self) -> None: run_node = CustomRunNode(argv=[ '--sysctl', 'tcp:8181', '--sysctl-init-file', sysctl_init_file_path, # relative to src/hathor - '--memory-storage', + '--temp-data', ]) self.assertTrue(run_node is not None) conn = run_node.manager.connections diff --git a/tests/consensus/test_consensus.py b/tests/consensus/test_consensus.py index 13ada2786..fcda1bdd2 100644 --- a/tests/consensus/test_consensus.py +++ b/tests/consensus/test_consensus.py @@ -2,7 +2,6 @@ from hathor.execution_manager import ExecutionManager from hathor.simulator.utils import add_new_block, add_new_blocks, gen_new_tx -from hathor.transaction.storage import TransactionMemoryStorage from hathor.util import not_none from tests import unittest from tests.utils import add_blocks_unlock_reward, add_new_double_spending, add_new_transactions @@ -11,7 +10,7 @@ class ConsensusTestCase(unittest.TestCase): def setUp(self) -> None: super().setUp() - self.tx_storage = TransactionMemoryStorage(settings=self._settings) + self.tx_storage = self.create_tx_storage() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/event/event_simulation_tester.py b/tests/event/event_simulation_tester.py index 4df16abeb..f3ffced2a 100644 --- a/tests/event/event_simulation_tester.py +++ b/tests/event/event_simulation_tester.py @@ -18,7 +18,6 @@ from twisted.internet.testing import StringTransport -from hathor.builder import Builder from hathor.event.websocket import EventWebsocketProtocol from hathor.event.websocket.request import Request from hathor.event.websocket.response import EventResponse, InvalidRequestResponse @@ -29,11 +28,11 @@ class BaseEventSimulationTester(SimulatorTestCase): - builder: Builder - - def _create_artifacts(self) -> None: + def setUp(self) -> None: + super().setUp() peer = PrivatePeer.auto_generated() - builder = self.builder.set_peer(peer) \ + builder = self.simulator.get_default_builder() \ + .set_peer(peer) \ .enable_event_queue() artifacts = self.simulator.create_artifacts(builder) @@ -89,22 +88,3 @@ def _decode_values(values: bytes) -> Iterable[dict[str, Any]]: yield json_loadb(value) buf = new_buf - - -class MemoryEventSimulationTester(BaseEventSimulationTester): - def setUp(self) -> None: - super().setUp() - self.builder = self.simulator.get_default_builder() - self._create_artifacts() - - -class RocksDBEventSimulationTester(BaseEventSimulationTester): - def setUp(self) -> None: - super().setUp() - import tempfile - - directory = tempfile.mkdtemp() - self.tmpdirs.append(directory) - - self.builder = self.simulator.get_default_builder().use_rocksdb(path=directory) - self._create_artifacts() diff --git a/tests/event/test_event_manager.py b/tests/event/test_event_manager.py index 6c79ac44f..b77c8cf62 100644 --- a/tests/event/test_event_manager.py +++ b/tests/event/test_event_manager.py @@ -1,5 +1,5 @@ from hathor.event.model.event_type import EventType -from hathor.event.storage.memory_storage import EventMemoryStorage +from hathor.event.storage import EventRocksDBStorage from hathor.pubsub import HathorEvents from hathor.util import not_none from tests import unittest @@ -9,7 +9,9 @@ class EventManagerTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.network = 'testnet' - self.event_storage = EventMemoryStorage() + self.event_storage = EventRocksDBStorage( + rocksdb_storage=self.create_rocksdb_storage(), + ) self.manager = self.create_peer( self.network, enable_event_queue=True, diff --git a/tests/event/test_event_reorg.py b/tests/event/test_event_reorg.py index 249291be5..b105b3bc9 100644 --- a/tests/event/test_event_reorg.py +++ b/tests/event/test_event_reorg.py @@ -1,5 +1,5 @@ from hathor.event.model.event_type import EventType -from hathor.event.storage import EventMemoryStorage +from hathor.event.storage import EventRocksDBStorage from hathor.simulator.utils import add_new_blocks from tests import unittest from tests.utils import BURN_ADDRESS, get_genesis_key @@ -9,7 +9,9 @@ class EventReorgTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.network = 'testnet' - self.event_storage = EventMemoryStorage() + self.event_storage = EventRocksDBStorage( + rocksdb_storage=self.create_rocksdb_storage(), + ) self.manager = self.create_peer( self.network, enable_event_queue=True, diff --git a/tests/event/test_event_simulation_responses.py b/tests/event/test_event_simulation_responses.py index c2726dddc..638c5f14d 100644 --- a/tests/event/test_event_simulation_responses.py +++ b/tests/event/test_event_simulation_responses.py @@ -15,16 +15,10 @@ from hathor.event.websocket.request import AckRequest, StartStreamRequest, StopStreamRequest from hathor.event.websocket.response import InvalidRequestType from hathor.simulator.trigger import StopAfterNMinedBlocks -from tests.event.event_simulation_tester import ( - BaseEventSimulationTester, - MemoryEventSimulationTester, - RocksDBEventSimulationTester, -) +from tests.event.event_simulation_tester import BaseEventSimulationTester -class BaseEventSimulationResponsesTest(BaseEventSimulationTester): - __test__ = False - +class EventSimulationResponsesTest(BaseEventSimulationTester): def test_no_start_no_blocks(self) -> None: self.simulator.run(36000) @@ -359,11 +353,3 @@ def test_multiple_interactions(self) -> None: assert len(responses) == 4 # 4 events because of window size assert responses[0].event.id == 8 # ack=7, so we get from event 8 - - -class MemoryEventSimulationResponsesTest(BaseEventSimulationResponsesTest, MemoryEventSimulationTester): - __test__ = True - - -class RocksDBEventSimulationResponsesTest(BaseEventSimulationResponsesTest, RocksDBEventSimulationTester): - __test__ = True diff --git a/tests/event/test_event_simulation_scenarios.py b/tests/event/test_event_simulation_scenarios.py index 89cd57e42..63935d526 100644 --- a/tests/event/test_event_simulation_scenarios.py +++ b/tests/event/test_event_simulation_scenarios.py @@ -28,21 +28,15 @@ from hathor.event.model.event_type import EventType from hathor.event.websocket.request import StartStreamRequest from hathor.event.websocket.response import EventResponse -from tests.event.event_simulation_tester import ( - BaseEventSimulationTester, - MemoryEventSimulationTester, - RocksDBEventSimulationTester, -) +from tests.event.event_simulation_tester import BaseEventSimulationTester -class BaseEventSimulationScenariosTest(BaseEventSimulationTester): +class EventSimulationScenariosTest(BaseEventSimulationTester): """ NOTE: The lists of expected events used in tests below were generated by printing the event responses list to the console and then copying the output and manipulating it to create instances. """ - __test__ = False - seed_config = 6946502462188444706 def assert_response_equal(self, responses: list[EventResponse], expected: list[EventResponse]) -> None: @@ -500,11 +494,3 @@ def _remove_timestamp(responses: list[EventResponse]) -> list[EventResponse]: del response.event.timestamp return responses - - -class MemoryEventSimulationScenariosTest(BaseEventSimulationScenariosTest, MemoryEventSimulationTester): - __test__ = True - - -class RocksDBEventSimulationScenariosTest(BaseEventSimulationScenariosTest, RocksDBEventSimulationTester): - __test__ = True diff --git a/tests/event/test_event_storage.py b/tests/event/test_event_storage.py index b0b368d2f..758602129 100644 --- a/tests/event/test_event_storage.py +++ b/tests/event/test_event_storage.py @@ -1,23 +1,17 @@ -import tempfile - from hathor.event.model.base_event import BaseEvent from hathor.event.model.node_state import NodeState -from hathor.event.storage import EventStorage -from hathor.event.storage.memory_storage import EventMemoryStorage from hathor.event.storage.rocksdb_storage import EventRocksDBStorage -from hathor.storage.rocksdb_storage import RocksDBStorage from tests import unittest from tests.utils import EventMocker -class EventStorageBaseTest(unittest.TestCase): - __test__ = False - - event_storage: EventStorage - +class EventStorageTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.event_mocker = EventMocker(self.rng) + self.event_storage = EventRocksDBStorage( + rocksdb_storage=self.create_rocksdb_storage(), + ) def test_save_event_and_retrieve(self) -> None: event = self.event_mocker.generate_mocked_event() @@ -233,22 +227,3 @@ def test_reset_all_full_database(self) -> None: assert node_state is None assert event_queue_state is False - - -class EventStorageRocksDBTest(EventStorageBaseTest): - __test__ = True - - def setUp(self) -> None: - super().setUp() - self.directory = tempfile.mkdtemp() - self.tmpdirs.append(self.directory) - self.rocksdb_storage = RocksDBStorage(path=self.directory) - self.event_storage = EventRocksDBStorage(self.rocksdb_storage) - - -class EventStorageMemoryTest(EventStorageBaseTest): - __test__ = True - - def setUp(self) -> None: - super().setUp() - self.event_storage = EventMemoryStorage() diff --git a/tests/event/websocket/test_factory.py b/tests/event/websocket/test_factory.py index 24feeab98..3fbbd6711 100644 --- a/tests/event/websocket/test_factory.py +++ b/tests/event/websocket/test_factory.py @@ -17,11 +17,12 @@ import pytest from hathor.conf.get_settings import get_global_settings -from hathor.event.storage import EventMemoryStorage +from hathor.event.storage import EventRocksDBStorage from hathor.event.websocket.factory import EventWebsocketFactory from hathor.event.websocket.protocol import EventWebsocketProtocol from hathor.event.websocket.response import EventResponse, InvalidRequestType from hathor.simulator.clock import MemoryReactorHeapClock +from hathor.storage import RocksDBStorage from tests.utils import EventMocker @@ -157,7 +158,9 @@ def _get_factory( n_starting_events: int = 0, clock: MemoryReactorHeapClock = MemoryReactorHeapClock() ) -> EventWebsocketFactory: - event_storage = EventMemoryStorage() + event_storage = EventRocksDBStorage( + rocksdb_storage=RocksDBStorage.create_temp(), + ) for event_id in range(n_starting_events): event = EventMocker.create_event(event_id) diff --git a/tests/feature_activation/test_feature_service.py b/tests/feature_activation/test_feature_service.py index f042b4e45..ce0efd871 100644 --- a/tests/feature_activation/test_feature_service.py +++ b/tests/feature_activation/test_feature_service.py @@ -29,16 +29,17 @@ from hathor.feature_activation.model.feature_info import FeatureInfo from hathor.feature_activation.model.feature_state import FeatureState from hathor.feature_activation.settings import Settings as FeatureSettings -from hathor.indexes import MemoryIndexesManager from hathor.transaction import Block -from hathor.transaction.storage import TransactionMemoryStorage, TransactionStorage +from hathor.transaction.storage import TransactionStorage from hathor.transaction.validation_state import ValidationState from hathor.util import not_none +from tests.unittest import TestBuilder def get_storage(settings: HathorSettings, *, up_to_height: int) -> TransactionStorage: - indexes = MemoryIndexesManager() - storage = TransactionMemoryStorage(indexes=indexes, settings=settings) + artifacts = TestBuilder(settings).build() + storage = artifacts.tx_storage + indexes = not_none(artifacts.indexes) feature_activation_bits = [ 0b0000, # 0: boundary block 0b0010, diff --git a/tests/feature_activation/test_feature_simulation.py b/tests/feature_activation/test_feature_simulation.py index 88883742c..95f26aa31 100644 --- a/tests/feature_activation/test_feature_simulation.py +++ b/tests/feature_activation/test_feature_simulation.py @@ -663,13 +663,6 @@ def test_reorg(self) -> None: assert artifacts.bit_signaling_service.get_not_support_features() == [Feature.NOP_FEATURE_1] -class MemoryStorageFeatureSimulationTest(BaseFeatureSimulationTest): - __test__ = True - - def get_simulator_builder(self) -> Builder: - return self.simulator.get_default_builder() - - class RocksDBStorageFeatureSimulationTest(BaseFeatureSimulationTest): __test__ = True @@ -681,7 +674,7 @@ def get_rocksdb_directory(self) -> str: def get_simulator_builder_from_dir(self, rocksdb_directory: str) -> Builder: return self.simulator.get_default_builder() \ - .use_rocksdb(path=rocksdb_directory) + .set_rocksdb_path(path=rocksdb_directory) def get_simulator_builder(self) -> Builder: rocksdb_directory = self.get_rocksdb_directory() diff --git a/tests/others/test_builder.py b/tests/others/test_builder.py index b17b7c8c4..91f274d12 100644 --- a/tests/others/test_builder.py +++ b/tests/others/test_builder.py @@ -7,7 +7,6 @@ def setUp(self): super().setUp() self.reactor = self.clock self.builder = TestBuilder() - self.builder.use_memory() def test_multiple_calls_to_build(self): self.builder.build() diff --git a/tests/others/test_cli_builder.py b/tests/others/test_cli_builder.py index ebb8d09a6..29d4e65b6 100644 --- a/tests/others/test_cli_builder.py +++ b/tests/others/test_cli_builder.py @@ -3,13 +3,13 @@ from hathor.builder import CliBuilder, ResourcesBuilder from hathor.cli.run_node_args import RunNodeArgs from hathor.event import EventManager -from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage +from hathor.event.storage import EventRocksDBStorage from hathor.event.websocket import EventWebsocketFactory from hathor.exception import BuilderError -from hathor.indexes import MemoryIndexesManager, RocksDBIndexesManager +from hathor.indexes import RocksDBIndexesManager from hathor.manager import HathorManager from hathor.p2p.sync_version import SyncVersion -from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage, TransactionRocksDBStorage +from hathor.transaction.storage import TransactionCacheStorage, TransactionRocksDBStorage from hathor.wallet import HDWallet, Wallet from tests import unittest @@ -66,73 +66,51 @@ def test_disable_cache_storage(self): self.assertIsInstance(manager.tx_storage, TransactionRocksDBStorage) self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - def test_default_storage_memory_indexes(self): - data_dir = self.mkdtemp() - manager = self._build(['--memory-indexes', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionCacheStorage) - self.assertIsInstance(manager.tx_storage.store, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) - - def test_default_storage_with_rocksdb_indexes(self): - data_dir = self.mkdtemp() - manager = self._build(['--x-rocksdb-indexes', '--data', data_dir]) - self.assertIsInstance(manager.tx_storage, TransactionCacheStorage) - self.assertIsInstance(manager.tx_storage.store, TransactionRocksDBStorage) - self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - def test_rocksdb_storage(self): data_dir = self.mkdtemp() - manager = self._build(['--rocksdb-storage', '--data', data_dir]) + manager = self._build(['--data', data_dir]) self.assertIsInstance(manager.tx_storage, TransactionCacheStorage) self.assertIsInstance(manager.tx_storage.store, TransactionRocksDBStorage) self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) - def test_memory_storage(self): - manager = self._build(['--memory-storage']) - self.assertIsInstance(manager.tx_storage, TransactionMemoryStorage) - self.assertIsInstance(manager.tx_storage.indexes, MemoryIndexesManager) - - def test_memory_storage_with_rocksdb_indexes(self): - self._build_with_error(['--memory-storage', '--x-rocksdb-indexes'], 'RocksDB indexes require RocksDB data') - def test_sync_default(self): - manager = self._build(['--memory-storage']) + manager = self._build(['--temp-data']) self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_bridge(self): - self._build_with_error(['--memory-storage', '--x-sync-bridge'], '--x-sync-bridge was removed') + self._build_with_error(['--temp-data', '--x-sync-bridge'], '--x-sync-bridge was removed') def test_sync_bridge2(self): - self._build_with_error(['--memory-storage', '--sync-bridge'], '--sync-bridge was removed') + self._build_with_error(['--temp-data', '--sync-bridge'], '--sync-bridge was removed') def test_sync_v2_only(self): - manager = self._build(['--memory-storage', '--x-sync-v2-only']) + manager = self._build(['--temp-data', '--x-sync-v2-only']) self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_v2_only2(self): - manager = self._build(['--memory-storage', '--sync-v2-only']) + manager = self._build(['--temp-data', '--sync-v2-only']) self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_v1_only(self): - self._build_with_error(['--memory-storage', '--sync-v1-only'], '--sync-v1-only was removed') + self._build_with_error(['--temp-data', '--sync-v1-only'], '--sync-v1-only was removed') def test_keypair_wallet(self): - manager = self._build(['--memory-storage', '--wallet', 'keypair']) + manager = self._build(['--temp-data', '--wallet', 'keypair']) self.assertIsInstance(manager.wallet, Wallet) def test_hd_wallet(self): - manager = self._build(['--memory-storage', '--wallet', 'hd']) + manager = self._build(['--temp-data', '--wallet', 'hd']) self.assertIsInstance(manager.wallet, HDWallet) def test_invalid_wallet(self): - self._build_with_error(['--memory-storage', '--wallet', 'invalid-wallet'], 'Invalid type of wallet') + self._build_with_error(['--temp-data', '--wallet', 'invalid-wallet'], 'Invalid type of wallet') def test_status(self): self._build([ - '--memory-storage', + '--temp-data', '--status', '8080', '--utxo-index', '--enable-debug-api', @@ -142,7 +120,7 @@ def test_status(self): self.clean_pending(required_to_quiesce=False) def test_prometheus_no_data(self): - args = ['--memory-storage', '--prometheus'] + args = ['--temp-data', '--prometheus'] self._build_with_error(args, 'To run prometheus exporter you must have a data path') def test_prometheus(self): @@ -151,24 +129,11 @@ def test_prometheus(self): self.assertTrue(self.resources_builder._built_prometheus) self.clean_pending(required_to_quiesce=False) - def test_memory_and_rocksdb_indexes(self): - data_dir = self.mkdtemp() - args = ['--memory-indexes', '--x-rocksdb-indexes', '--data', data_dir] - self._build_with_error(args, 'You cannot use --memory-indexes and --x-rocksdb-indexes.') - def test_event_queue_with_rocksdb_storage(self): data_dir = self.mkdtemp() - manager = self._build(['--x-enable-event-queue', '--rocksdb-storage', '--data', data_dir]) + manager = self._build(['--x-enable-event-queue', '--data', data_dir]) self.assertIsInstance(manager._event_manager, EventManager) self.assertIsInstance(manager._event_manager._event_storage, EventRocksDBStorage) self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) self.assertTrue(manager._enable_event_queue) - - def test_event_queue_with_memory_storage(self): - manager = self._build(['--x-enable-event-queue', '--memory-storage']) - - self.assertIsInstance(manager._event_manager, EventManager) - self.assertIsInstance(manager._event_manager._event_storage, EventMemoryStorage) - self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) - self.assertTrue(manager._enable_event_queue) diff --git a/tests/others/test_metrics.py b/tests/others/test_metrics.py index 6b4c85cf3..c3d50e969 100644 --- a/tests/others/test_metrics.py +++ b/tests/others/test_metrics.py @@ -1,13 +1,15 @@ import tempfile from unittest.mock import Mock +from hathor.manager import HathorManager from hathor.p2p.manager import PeerConnectionsMetrics from hathor.p2p.peer import PrivatePeer from hathor.p2p.peer_endpoint import PeerEndpoint from hathor.p2p.protocol import HathorProtocol from hathor.pubsub import HathorEvents from hathor.simulator.utils import add_new_blocks -from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage +from hathor.transaction.storage import TransactionCacheStorage, TransactionRocksDBStorage +from hathor.transaction.vertex_parser import VertexParser from hathor.wallet import Wallet from tests import unittest @@ -21,9 +23,7 @@ def test_p2p_network_events(self): the event to set its own fields related to the network peers """ # Preparation - self.use_memory_storage = True manager = self.create_peer('testnet') - self.assertIsInstance(manager.tx_storage, TransactionMemoryStorage) pubsub = manager.pubsub # Execution @@ -49,7 +49,7 @@ def test_connections_manager_integration(self): to update the Metrics class with info from ConnectionsManager class """ # Preparation - tx_storage = TransactionMemoryStorage(settings=self._settings) + tx_storage = self.create_tx_storage() tmpdir = tempfile.mkdtemp() self.tmpdirs.append(tmpdir) wallet = Wallet(directory=tmpdir) @@ -86,14 +86,12 @@ def test_tx_storage_data_collection_with_rocksdb_storage_and_no_cache(self): The expected result is that it will successfully collect the RocksDB metrics. """ - path = tempfile.mkdtemp() - self.tmpdirs.append(path) - - def _init_manager(): + def _init_manager(path: tempfile.TemporaryDirectory | None = None) -> HathorManager: builder = self.get_builder() \ - .use_rocksdb(path, cache_capacity=100) \ - .force_memory_index() \ + .set_rocksdb_cache_capacity(100) \ .set_wallet(self._create_test_wallet(unlocked=True)) + if path: + builder.set_rocksdb_path(path) manager = self.create_peer_from_builder(builder, start_manager=False) return manager @@ -110,6 +108,14 @@ def _init_manager(): b'event': 0.0, b'event-metadata': 0.0, b'feature-activation-metadata': 0.0, + b'info-index': 0.0, + b'height-index': 0.0, + b'tips-all': 0.0, + b'tips-blocks': 0.0, + b'tips-txs': 0.0, + b'timestamp-sorted-all': 0.0, + b'timestamp-sorted-blocks': 0.0, + b'timestamp-sorted-txs': 0.0, }) manager.tx_storage.pre_init() @@ -122,7 +128,7 @@ def _init_manager(): # https://github.com/facebook/rocksdb/blob/v7.5.3/include/rocksdb/db.h#L1396 manager.tx_storage._db.close() - manager = _init_manager() + manager = _init_manager(manager.tx_storage._rocksdb_storage.temp_dir) manager.metrics._collect_data() # We don't know exactly the sizes of each column family, @@ -137,15 +143,13 @@ def test_tx_storage_data_collection_with_rocksdb_storage_and_cache(self): The expected result is that it will successfully collect the RocksDB metrics. """ - path = tempfile.mkdtemp() - self.tmpdirs.append(path) - - def _init_manager(): + def _init_manager(path: tempfile.TemporaryDirectory | None = None) -> HathorManager: builder = self.get_builder() \ - .use_rocksdb(path, cache_capacity=100) \ - .force_memory_index() \ + .set_rocksdb_cache_capacity(100) \ .set_wallet(self._create_test_wallet(unlocked=True)) \ .use_tx_storage_cache() + if path: + builder.set_rocksdb_path(path) manager = self.create_peer_from_builder(builder, start_manager=False) return manager @@ -163,6 +167,14 @@ def _init_manager(): b'event': 0.0, b'event-metadata': 0.0, b'feature-activation-metadata': 0.0, + b'info-index': 0.0, + b'height-index': 0.0, + b'tips-all': 0.0, + b'tips-blocks': 0.0, + b'tips-txs': 0.0, + b'timestamp-sorted-all': 0.0, + b'timestamp-sorted-blocks': 0.0, + b'timestamp-sorted-txs': 0.0, }) manager.tx_storage.pre_init() @@ -176,7 +188,7 @@ def _init_manager(): # https://github.com/facebook/rocksdb/blob/v7.5.3/include/rocksdb/db.h#L1396 manager.tx_storage.store._db.close() - manager = _init_manager() + manager = _init_manager(manager.tx_storage.store._rocksdb_storage.temp_dir) manager.metrics._collect_data() # We don't know exactly the sizes of each column family, @@ -184,30 +196,12 @@ def _init_manager(): self.assertTrue(manager.metrics.rocksdb_cfs_sizes[b'tx'] > 500) self.assertTrue(manager.metrics.rocksdb_cfs_sizes[b'meta'] > 1000) - def test_tx_storage_data_collection_with_memory_storage(self): - """Tests storage data collection when using Memory Storage using no cache - We don't allow using it with cache, so this is the only case - - The expected result is that nothing is done, because we currently only collect - data for RocksDB storage - """ - tx_storage = TransactionMemoryStorage(settings=self._settings) - - # All - manager = self.create_peer('testnet', tx_storage=tx_storage) - - manager.metrics._collect_data() - - self.assertEqual(manager.metrics.rocksdb_cfs_sizes, {}) - def test_peer_connections_data_collection(self): """Test if peer connections data is correctly being collected from the ConnectionsManager """ # Preparation - self.use_memory_storage = True manager = self.create_peer('testnet') - self.assertIsInstance(manager.tx_storage, TransactionMemoryStorage) my_peer = manager.my_peer @@ -260,7 +254,12 @@ def test_cache_data_collection(self): TransactionCacheStorage """ # Preparation - base_storage = TransactionMemoryStorage(settings=self._settings) + rocksdb_storage = self.create_rocksdb_storage() + base_storage = TransactionRocksDBStorage( + rocksdb_storage=rocksdb_storage, + settings=self._settings, + vertex_parser=VertexParser(settings=self._settings), + ) tx_storage = TransactionCacheStorage(base_storage, self.clock, indexes=None, settings=self._settings) manager = self.create_peer('testnet', tx_storage=tx_storage) diff --git a/tests/p2p/test_sync_v2.py b/tests/p2p/test_sync_v2.py index ea279af8a..777549fbf 100644 --- a/tests/p2p/test_sync_v2.py +++ b/tests/p2p/test_sync_v2.py @@ -63,11 +63,9 @@ def _run_restart_test(self, *, use_tx_storage_cache: bool) -> None: gen_tx1.stop() # Create a new peer and run sync for a while (but stop before getting synced). - path = self.mkdtemp() peer = PrivatePeer.auto_generated() builder2 = self.simulator.get_default_builder() \ - .set_peer(peer) \ - .use_rocksdb(path) + .set_peer(peer) manager2 = self.simulator.create_peer(builder2) conn12 = FakeConnection(manager1, manager2, latency=0.05) @@ -92,6 +90,7 @@ def _run_restart_test(self, *, use_tx_storage_cache: bool) -> None: self.simulator.remove_connection(conn12) manager2.stop() assert isinstance(manager2.tx_storage, TransactionRocksDBStorage) + temp_dir = not_none(manager2.tx_storage._rocksdb_storage.temp_dir) manager2.tx_storage._rocksdb_storage.close() del manager2 @@ -104,7 +103,7 @@ def _run_restart_test(self, *, use_tx_storage_cache: bool) -> None: # Restart full node using the same db. builder3 = self.simulator.get_default_builder() \ .set_peer(peer) \ - .use_rocksdb(path) + .set_rocksdb_path(temp_dir) if use_tx_storage_cache: builder3.use_tx_storage_cache() diff --git a/tests/poa/test_poa_simulation.py b/tests/poa/test_poa_simulation.py index 32946d4d1..384f7d1d4 100644 --- a/tests/poa/test_poa_simulation.py +++ b/tests/poa/test_poa_simulation.py @@ -324,19 +324,16 @@ def test_producer_leave_and_comeback(self) -> None: ) def test_existing_storage(self) -> None: - import tempfile - rocksdb_directory = tempfile.mkdtemp() - self.tmpdirs.append(rocksdb_directory) signer = get_signer() signer_id = signer._signer_id self.simulator.settings = get_settings(signer, time_between_blocks=10) builder = self.simulator.get_default_builder() \ .set_poa_signer(signer) \ - .use_rocksdb(path=rocksdb_directory) artifacts1 = self.simulator.create_artifacts(builder) manager1 = artifacts1.manager + rocksdb_dir = not_none(artifacts1.rocksdb_storage.temp_dir) manager1.allow_mining_without_peers() self.simulator.run(50) @@ -357,7 +354,7 @@ def test_existing_storage(self) -> None: builder = self.simulator.get_default_builder() \ .set_poa_signer(signer) \ - .use_rocksdb(path=rocksdb_directory) + .set_rocksdb_path(path=rocksdb_dir) artifacts = self.simulator.create_artifacts(builder) manager2 = artifacts.manager diff --git a/tests/resources/event/test_event.py b/tests/resources/event/test_event.py index fbf32240e..c1e2cd0ec 100644 --- a/tests/resources/event/test_event.py +++ b/tests/resources/event/test_event.py @@ -18,14 +18,17 @@ from hathor.event import EventManager from hathor.event.resources.event import EventResource -from hathor.event.storage import EventMemoryStorage +from hathor.event.storage import EventRocksDBStorage +from hathor.storage import RocksDBStorage from tests.resources.base_resource import StubSite from tests.utils import EventMocker @pytest.fixture def web(): - event_storage = EventMemoryStorage() + event_storage = EventRocksDBStorage( + rocksdb_storage=RocksDBStorage.create_temp(), + ) for i in range(3): event = EventMocker.create_event(i) diff --git a/tests/resources/transaction/test_pushtx.py b/tests/resources/transaction/test_pushtx.py index 174ee3586..3f488e429 100644 --- a/tests/resources/transaction/test_pushtx.py +++ b/tests/resources/transaction/test_pushtx.py @@ -18,9 +18,6 @@ class BasePushTxTest(_BaseResourceTest._ResourceTest): is_post: Optional[bool] = None - # XXX: we will get a "two instances of the same tx in memory" otherwise - use_memory_storage = True - def setUp(self): super().setUp() self.web = StubSite(PushTxResource(self.manager)) @@ -233,6 +230,9 @@ def test_spending_voided(self) -> Generator: data = response.json_value() self.assertTrue(data['success']) + # We have to get tx2 from the storage because the saved instance is different from the one we created here. + tx2 = self.manager.tx_storage.get_transaction(tx2.hash) + # Now we set this tx2 as voided and try to push a tx3 that spends tx2 tx_meta = tx2.get_metadata() tx_meta.voided_by = {tx2.hash} diff --git a/tests/resources/transaction/test_tx.py b/tests/resources/transaction/test_tx.py index 9419ae494..884acc30d 100644 --- a/tests/resources/transaction/test_tx.py +++ b/tests/resources/transaction/test_tx.py @@ -11,9 +11,6 @@ class TransactionTest(_BaseResourceTest._ResourceTest): - # XXX: using memory storage so that we can more easily manipulate the tokens-index for a test - use_memory_storage = True - def setUp(self): super().setUp() self.web = StubSite(TransactionResource(self.manager)) @@ -130,17 +127,11 @@ def test_get_one_known_tx(self): tx_input.set_static_metadata(TransactionStaticMetadata(min_height=0, closest_ancestor_block=b'')) self.manager.tx_storage.save_transaction(tx_input) - # XXX: this is completely dependant on MemoryTokensIndex implementation, hence use_memory_storage=True token_bytes1 = bytes.fromhex('001c382847d8440d05da95420bee2ebeb32bc437f82a9ae47b0745c8a29a7b0d') - status = self.manager.tx_storage.indexes.tokens._tokens[token_bytes1] - status.name = 'Test Coin' - status.symbol = 'TSC' + self.manager.tx_storage.indexes.tokens._create_token_info(token_bytes1, 'Test Coin', 'TSC') - # XXX: this is completely dependant on MemoryTokensIndex implementation, hence use_memory_storage=True token_bytes2 = bytes.fromhex('007231eee3cb6160d95172a409d634d0866eafc8775f5729fff6a61e7850aba5') - status2 = self.manager.tx_storage.indexes.tokens._tokens[token_bytes2] - status2.name = 'NewCoin' - status2.symbol = 'NCN' + self.manager.tx_storage.indexes.tokens._create_token_info(token_bytes2, 'NewCoin', 'NCN') response = yield self.web.get( "transaction", {b'id': b'0033784bc8443ba851fd88d81c6f06774ae529f25c1fa8f026884ad0a0e98011'}) @@ -231,11 +222,8 @@ def test_get_one_known_tx_with_authority(self): # Both inputs are the same as the last parent, so no need to manually add them - # XXX: this is completely dependant on MemoryTokensIndex implementation token_bytes1 = bytes.fromhex('000023b318c91dcfd4b967b205dc938f9f5e2fd5114256caacfb8f6dd13db330') - status = self.manager.tx_storage.indexes.tokens._tokens[token_bytes1] - status.name = 'Wat wat' - status.symbol = 'WAT' + self.manager.tx_storage.indexes.tokens._create_token_info(token_bytes1, 'Wat wat', 'WAT') response = yield self.web.get( "transaction", {b'id': b'00005f234469407614bf0abedec8f722bb5e534949ad37650f6077c899741ed7'}) @@ -290,12 +278,10 @@ def test_get_many(self): # Get last 2 blocks expected1 = blocks[-2:] - expected1.reverse() - response1 = yield self.web.get("transaction", {b'count': b'2', b'type': b'block'}) data1 = response1.json_value() - for expected, result in zip(expected1, data1['transactions']): + for expected, result in zip(reversed(expected1), data1['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) @@ -303,21 +289,18 @@ def test_get_many(self): # Get last 8 txs expected2 = txs[-8:] - expected2.reverse() - response2 = yield self.web.get("transaction", {b'count': b'8', b'type': b'tx'}) data2 = response2.json_value() - for expected, result in zip(expected2, data2['transactions']): + for expected, result in zip(reversed(expected2), data2['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) self.assertTrue(data2['has_more']) # Get older blocks with hash reference - expected3 = blocks[:2] - expected3.reverse() - + genesis_block = self.manager.tx_storage.get_genesis(self._settings.GENESIS_BLOCK_HASH) + expected3 = [genesis_block, *blocks[:2]] response3 = yield self.web.get( "transaction", { b'count': b'3', @@ -328,7 +311,7 @@ def test_get_many(self): }) data3 = response3.json_value() - for expected, result in zip(expected3, data3['transactions']): + for expected, result in zip(reversed(expected3), data3['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) @@ -345,7 +328,7 @@ def test_get_many(self): }) data4 = response4.json_value() - for expected, result in zip(expected2, data4['transactions']): + for expected, result in zip(expected2, data4['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) @@ -353,19 +336,17 @@ def test_get_many(self): # Get newer blocks with hash reference expected5 = blocks[-2:] - expected5.reverse() - response5 = yield self.web.get( "transaction", { b'count': b'3', b'type': b'block', - b'timestamp': bytes(str(expected1[-1].timestamp), 'utf-8'), - b'hash': bytes(expected1[-1].hash.hex(), 'utf-8'), + b'timestamp': bytes(str(blocks[-3].timestamp), 'utf-8'), + b'hash': bytes(blocks[-3].hash.hex(), 'utf-8'), b'page': b'previous' }) data5 = response5.json_value() - for expected, result in zip(expected5, data5['transactions']): + for expected, result in zip(expected5, data5['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) @@ -373,8 +354,6 @@ def test_get_many(self): # Get txs with hash reference expected6 = txs[:8] - expected6.reverse() - response6 = yield self.web.get( "transaction", { b'count': b'8', @@ -385,7 +364,7 @@ def test_get_many(self): }) data6 = response6.json_value() - for expected, result in zip(expected6, data6['transactions']): + for expected, result in zip(reversed(expected6), data6['transactions'], strict=True): self.assertEqual(expected.timestamp, result['timestamp']) self.assertEqual(expected.hash.hex(), result['tx_id']) @@ -462,12 +441,12 @@ def test_zero_count(self): response = yield self.web.get("transaction", {b'count': b'0', b'type': b'block'}) data = response.json_value() self.assertEqual(0, len(data['transactions'])) - self.assertFalse(data['has_more']) + self.assertTrue(data['has_more']) response = yield self.web.get("transaction", {b'count': b'0', b'type': b'tx'}) data = response.json_value() self.assertEqual(0, len(data['transactions'])) - self.assertFalse(data['has_more']) + self.assertTrue(data['has_more']) @inlineCallbacks def test_negative_count(self): diff --git a/tests/tx/test_accumulated_weight.py b/tests/tx/test_accumulated_weight.py index 783a5b07d..79d06b0e0 100644 --- a/tests/tx/test_accumulated_weight.py +++ b/tests/tx/test_accumulated_weight.py @@ -1,5 +1,4 @@ from hathor.simulator.utils import add_new_blocks -from hathor.transaction.storage import TransactionMemoryStorage from hathor.utils.weight import weight_to_work from tests import unittest from tests.utils import add_blocks_unlock_reward, add_new_transactions @@ -8,7 +7,7 @@ class AccumulatedWeightTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage(settings=self._settings) + self.tx_storage = self.create_tx_storage() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_block.py b/tests/tx/test_block.py index 1e103ed2f..a22e2eb6e 100644 --- a/tests/tx/test_block.py +++ b/tests/tx/test_block.py @@ -20,19 +20,19 @@ from hathor.conf.settings import HathorSettings from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import BlockIsMissingSignal, BlockIsSignaling, FeatureService -from hathor.indexes import MemoryIndexesManager from hathor.transaction import Block from hathor.transaction.exceptions import BlockMustSignalError from hathor.transaction.static_metadata import BlockStaticMetadata -from hathor.transaction.storage import TransactionMemoryStorage, TransactionStorage +from hathor.transaction.storage import TransactionStorage from hathor.transaction.validation_state import ValidationState from hathor.util import not_none from hathor.verification.block_verifier import BlockVerifier +from tests.unittest import TestBuilder def test_calculate_feature_activation_bit_counts_genesis(): settings = get_global_settings() - storage = TransactionMemoryStorage(settings=settings) + storage = TestBuilder().build().tx_storage genesis_block = storage.get_block(settings.GENESIS_BLOCK_HASH) result = genesis_block.static_metadata.feature_activation_bit_counts @@ -41,9 +41,9 @@ def test_calculate_feature_activation_bit_counts_genesis(): @pytest.fixture def tx_storage() -> TransactionStorage: - settings = get_global_settings() - indexes = MemoryIndexesManager() - storage = TransactionMemoryStorage(indexes=indexes, settings=settings) + artifacts = TestBuilder().build() + storage = artifacts.tx_storage + indexes = not_none(artifacts.indexes) feature_activation_bits = [ 0b0000, # 0: boundary block 0b1010, diff --git a/tests/tx/test_blockchain.py b/tests/tx/test_blockchain.py index 0aaa420a6..208257817 100644 --- a/tests/tx/test_blockchain.py +++ b/tests/tx/test_blockchain.py @@ -2,7 +2,6 @@ from hathor.daa import DifficultyAdjustmentAlgorithm, TestMode from hathor.simulator.utils import add_new_blocks -from hathor.transaction.storage import TransactionMemoryStorage from hathor.utils.weight import weight_to_work from tests import unittest from tests.utils import add_new_transactions @@ -23,7 +22,7 @@ class BlockchainTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage(settings=self._settings) + self.tx_storage = self.create_tx_storage() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_cache_storage.py b/tests/tx/test_cache_storage.py index 789011e22..d271a00b1 100644 --- a/tests/tx/test_cache_storage.py +++ b/tests/tx/test_cache_storage.py @@ -13,7 +13,6 @@ def setUp(self): super().setUp() builder = self.get_builder() \ - .use_memory() \ .use_tx_storage_cache(capacity=5) \ .set_wallet(self._create_test_wallet(unlocked=True)) self.manager = self.create_peer_from_builder(builder) diff --git a/tests/tx/test_genesis.py b/tests/tx/test_genesis.py index dbb96b8f7..f83840a57 100644 --- a/tests/tx/test_genesis.py +++ b/tests/tx/test_genesis.py @@ -2,7 +2,6 @@ from hathor.conf import HathorSettings from hathor.daa import DifficultyAdjustmentAlgorithm, TestMode -from hathor.transaction.storage import TransactionMemoryStorage from hathor.verification.verification_service import VerificationService from hathor.verification.vertex_verifier import VertexVerifier from hathor.verification.vertex_verifiers import VertexVerifiers @@ -34,7 +33,7 @@ def setUp(self) -> None: self._daa = DifficultyAdjustmentAlgorithm(settings=self._settings) verifiers = VertexVerifiers.create_defaults(settings=self._settings, daa=self._daa, feature_service=Mock()) self._verification_service = VerificationService(settings=self._settings, verifiers=verifiers) - self.storage = TransactionMemoryStorage(settings=settings) + self.storage = self.create_tx_storage() def test_pow(self): verifier = VertexVerifier(settings=self._settings) diff --git a/tests/tx/test_indexes.py b/tests/tx/test_indexes.py index 315f87198..0d13ea63a 100644 --- a/tests/tx/test_indexes.py +++ b/tests/tx/test_indexes.py @@ -687,32 +687,6 @@ def test_height_index(self): self.assertEqual(height_index.get_n_height_tips(103), height_index.get_n_height_tips(104)) -class MemoryIndexesTest(BaseIndexesTest): - __test__ = True - - def setUp(self): - from hathor.transaction.storage import TransactionMemoryStorage - - super().setUp() - self.wallet = Wallet() - self.tx_storage = TransactionMemoryStorage(settings=self._settings) - self.genesis = self.tx_storage.get_all_genesis() - self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] - self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] - - # read genesis keys - self.genesis_private_key = get_genesis_key() - self.genesis_public_key = self.genesis_private_key.public_key() - - # this makes sure we can spend the genesis outputs - self.manager = self.create_peer('testnet', tx_storage=self.tx_storage, unlock_wallet=True, wallet_index=True, - use_memory_index=True, utxo_index=True) - self.blocks = add_blocks_unlock_reward(self.manager) - self.last_block = self.blocks[-1] - - self.graphviz = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True) - - class RocksDBIndexesTest(BaseIndexesTest): __test__ = True diff --git a/tests/tx/test_indexes2.py b/tests/tx/test_indexes2.py deleted file mode 100644 index ef4357378..000000000 --- a/tests/tx/test_indexes2.py +++ /dev/null @@ -1,73 +0,0 @@ -import tempfile -from typing import TYPE_CHECKING, NamedTuple - -from tests import unittest - -if TYPE_CHECKING: # pragma: no cover - import rocksdb - - -class FakeTransaction(NamedTuple): - hash: bytes - timestamp: int - - -# XXX: sync-bridge used but it doesn't matter, it's only used to generate a random blockchain -class SimpleIndexesTestCase(unittest.TestCase): - def setUp(self): - super().setUp() - # how many transactions will be generated on the same timestamp before increasing it by 1 - self.transactions = [] - repetitions = [1, 1, 10, 10, 10, 2, 1, 0, 0, 5, 5, 5, 0, 1, 1, 10, 10, 10, 1, 2, 3, 1, 100, 100, 1, 100, 0, 1] - ts = self._settings.GENESIS_BLOCK_TIMESTAMP - for rep in repetitions: - for _ in range(rep): - tx = FakeTransaction(self.rng.randbytes(32), ts) - self.transactions.append(tx) - ts += 1 - - def create_tmp_rocksdb_db(self) -> 'rocksdb.DB': - import rocksdb - directory = tempfile.mkdtemp() - self.tmpdirs.append(directory) - options = rocksdb.Options(create_if_missing=True, error_if_exists=True) - return rocksdb.DB(directory, options) - - def test_timestamp_index(self): - # setup two indexes with different backends - from hathor.indexes.memory_timestamp_index import MemoryTimestampIndex - from hathor.indexes.rocksdb_timestamp_index import RocksDBTimestampIndex - from hathor.indexes.timestamp_index import RangeIdx, ScopeType - rocksdb_index = RocksDBTimestampIndex(self.create_tmp_rocksdb_db(), scope_type=ScopeType.ALL) - memory_index = MemoryTimestampIndex(scope_type=ScopeType.ALL) - for tx in self.transactions: - rocksdb_index.add_tx(tx) - memory_index.add_tx(tx) - - # varying count so we stop at varied points - offset_variety = set() - for count in [2, 3, 5, 10, 100]: - self.log.debug('with', count=count) - idx_rocksdb = RangeIdx(0, 0) - idx_memory = RangeIdx(0, 0) - max_iters = 1000 - while max_iters > 0: - self.log.debug('iter', idx=idx_memory) - hashes_memory, idx_memory = memory_index.get_hashes_and_next_idx(idx_memory, count) - hashes_rocksdb, idx_rocksdb = rocksdb_index.get_hashes_and_next_idx(idx_rocksdb, count) - self.assertEqual(hashes_memory, hashes_rocksdb) - self.assertEqual(idx_rocksdb, idx_memory) - # XXX: we verified they're the same, doesn't matter which we pick: - idx = idx_memory - hashes = hashes_memory - self.log.debug('indexes match', idx=idx, hashes=unittest.short_hashes(hashes)) - if idx is None: - break - offset_variety.add(idx[1]) - max_iters -= 1 - else: - self.fail('took too many iterations') - - # just making sure our tests covered enough different cases - self.log.debug('offset variety', offsets=offset_variety) - self.assertGreater(len(offset_variety), 2, msg='too little variety of offset, not enough coverage') diff --git a/tests/tx/test_indexes4.py b/tests/tx/test_indexes4.py index 8a5a98111..54664fa41 100644 --- a/tests/tx/test_indexes4.py +++ b/tests/tx/test_indexes4.py @@ -1,7 +1,6 @@ from hathor.crypto.util import decode_address from hathor.simulator.utils import add_new_blocks, gen_new_tx from hathor.transaction import Transaction -from hathor.transaction.storage import TransactionMemoryStorage from hathor.wallet.base_wallet import WalletOutputInfo from tests import unittest from tests.utils import add_blocks_unlock_reward @@ -9,9 +8,9 @@ class SimulatorIndexesTestCase(unittest.TestCase): def _build_randomized_blockchain(self, *, utxo_index=False): - tx_storage = TransactionMemoryStorage(settings=self._settings) + tx_storage = self.create_tx_storage() manager = self.create_peer('testnet', tx_storage=tx_storage, unlock_wallet=True, wallet_index=True, - use_memory_index=True, utxo_index=utxo_index) + utxo_index=utxo_index) add_new_blocks(manager, 50, advance_clock=15) @@ -53,8 +52,6 @@ def _build_randomized_blockchain(self, *, utxo_index=False): return manager def test_index_initialization(self): - from copy import deepcopy - self.manager = self._build_randomized_blockchain(utxo_index=True) # XXX: this test makes use of the internals of TipsIndex, AddressIndex and UtxoIndex @@ -74,8 +71,8 @@ def test_index_initialization(self): base_all_tips_tree = tx_storage.indexes.all_tips.tree.copy() base_block_tips_tree = tx_storage.indexes.block_tips.tree.copy() base_tx_tips_tree = tx_storage.indexes.tx_tips.tree.copy() - base_address_index = deepcopy(tx_storage.indexes.addresses.index) - base_utxo_index = deepcopy(tx_storage.indexes.utxo._index) + base_address_index = list(tx_storage.indexes.addresses.get_all_internal()) + base_utxo_index = list(tx_storage.indexes.utxo.get_all_internal()) # reset the indexes and force a re-initialization of all indexes tx_storage._manually_initialize() @@ -85,8 +82,8 @@ def test_index_initialization(self): reinit_all_tips_tree = tx_storage.indexes.all_tips.tree.copy() reinit_block_tips_tree = tx_storage.indexes.block_tips.tree.copy() reinit_tx_tips_tree = tx_storage.indexes.tx_tips.tree.copy() - reinit_address_index = deepcopy(tx_storage.indexes.addresses.index) - reinit_utxo_index = deepcopy(tx_storage.indexes.utxo._index) + reinit_address_index = list(tx_storage.indexes.addresses.get_all_internal()) + reinit_utxo_index = list(tx_storage.indexes.utxo.get_all_internal()) self.assertEqual(reinit_all_tips_tree, base_all_tips_tree) self.assertEqual(reinit_block_tips_tree, base_block_tips_tree) @@ -102,8 +99,8 @@ def test_index_initialization(self): newinit_all_tips_tree = tx_storage.indexes.all_tips.tree.copy() newinit_block_tips_tree = tx_storage.indexes.block_tips.tree.copy() newinit_tx_tips_tree = tx_storage.indexes.tx_tips.tree.copy() - newinit_address_index = deepcopy(tx_storage.indexes.addresses.index) - newinit_utxo_index = deepcopy(tx_storage.indexes.utxo._index) + newinit_address_index = list(tx_storage.indexes.addresses.get_all_internal()) + newinit_utxo_index = list(tx_storage.indexes.utxo.get_all_internal()) self.assertEqual(newinit_all_tips_tree, base_all_tips_tree) self.assertEqual(newinit_block_tips_tree, base_block_tips_tree) diff --git a/tests/tx/test_mining.py b/tests/tx/test_mining.py index 17d77450d..0e62acd30 100644 --- a/tests/tx/test_mining.py +++ b/tests/tx/test_mining.py @@ -3,7 +3,6 @@ from hathor.mining import BlockTemplate from hathor.simulator.utils import add_new_blocks from hathor.transaction import Block -from hathor.transaction.storage import TransactionMemoryStorage from hathor.utils.weight import weight_to_work from tests import unittest @@ -23,7 +22,7 @@ class MiningTest(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage(settings=self._settings) + self.tx_storage = self.create_tx_storage() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_reward_lock.py b/tests/tx/test_reward_lock.py index 55f062f56..31d4f6e69 100644 --- a/tests/tx/test_reward_lock.py +++ b/tests/tx/test_reward_lock.py @@ -7,7 +7,6 @@ from hathor.transaction import Block, Transaction, TxInput, TxOutput from hathor.transaction.exceptions import RewardLocked from hathor.transaction.scripts import P2PKH -from hathor.transaction.storage import TransactionMemoryStorage from hathor.wallet import Wallet from tests import unittest from tests.utils import add_blocks_unlock_reward, get_genesis_key @@ -23,7 +22,7 @@ def setUp(self): self.genesis_public_key = self.genesis_private_key.public_key() # this makes sure we can spend the genesis outputs - self.tx_storage = TransactionMemoryStorage(settings=self._settings) + self.tx_storage = self.create_tx_storage() self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_scripts.py b/tests/tx/test_scripts.py index 34ce6ac25..73430f729 100644 --- a/tests/tx/test_scripts.py +++ b/tests/tx/test_scripts.py @@ -56,7 +56,6 @@ op_pushdata1, ) from hathor.transaction.scripts.script_context import ScriptContext -from hathor.transaction.storage import TransactionMemoryStorage from hathor.wallet import HDWallet from tests import unittest from tests.utils import BURN_ADDRESS, get_genesis_key @@ -65,7 +64,7 @@ class TestScripts(unittest.TestCase): def setUp(self): super().setUp() - tx_storage = TransactionMemoryStorage(settings=self._settings) + tx_storage = self.create_tx_storage() self.genesis_blocks = [tx for tx in tx_storage.get_all_genesis() if tx.is_block] self.genesis_txs = [tx for tx in tx_storage.get_all_genesis() if not tx.is_block] diff --git a/tests/tx/test_stratum.py b/tests/tx/test_stratum.py index 6a2811111..4e9a935ad 100644 --- a/tests/tx/test_stratum.py +++ b/tests/tx/test_stratum.py @@ -21,7 +21,6 @@ StratumFactory, ) from hathor.transaction.block import Block -from hathor.transaction.storage import TransactionMemoryStorage from tests import unittest @@ -227,7 +226,7 @@ def test_min_share_weight(self): class StratumClientTest(unittest.TestCase): def setUp(self): super().setUp() - storage = TransactionMemoryStorage(settings=self._settings) + storage = self.create_tx_storage() self.block = storage.get_transaction(self._settings.GENESIS_BLOCK_HASH) self.transport = StringTransportWithDisconnection() self.protocol = StratumClient(reactor=self.clock) diff --git a/tests/tx/test_tokens.py b/tests/tx/test_tokens.py index 602938692..cfcc85c88 100644 --- a/tests/tx/test_tokens.py +++ b/tests/tx/test_tokens.py @@ -614,8 +614,3 @@ def test_voided_token_creation(self): self.assertEqual(1, len(melt)) tokens_index = self.manager.tx_storage.indexes.tokens.get_token_info(token_uid) print(tokens_index) - - -@pytest.mark.skipif(unittest.USE_MEMORY_STORAGE, reason='previous tests already use memory, avoid duplication') -class MemoryTokenTest(TokenTest): - use_memory_storage = True diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 6c842d656..a222ce295 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -47,7 +47,7 @@ def setUp(self): self.wallet = Wallet() # this makes sure we can spend the genesis outputs - self.manager = self.create_peer('testnet', unlock_wallet=True, wallet_index=True, use_memory_storage=True) + self.manager = self.create_peer('testnet', unlock_wallet=True, wallet_index=True) self._verifiers = self.manager.verification_service.verifiers self.tx_storage = self.manager.tx_storage diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index 87ce51bbb..105b8bfad 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -599,31 +599,11 @@ def _test_remove_tx_or_block(self, tx): self.assertFalse(self.tx_storage.store.transaction_exists(tx_hash)) -class TransactionMemoryStorageTest(BaseTransactionStorageTest): - __test__ = True - - def _config_builder(self, builder: TestBuilder) -> None: - builder.use_memory() - - -class CacheMemoryStorageTest(BaseCacheStorageTest): - __test__ = True - - def _config_builder(self, builder: TestBuilder) -> None: - builder.use_memory() - builder.use_tx_storage_cache(capacity=5) - - class TransactionRocksDBStorageTest(BaseTransactionStorageTest): __test__ = True def _config_builder(self, builder: TestBuilder) -> None: - self.directory = tempfile.mkdtemp() - builder.use_rocksdb(self.directory) - - def tearDown(self): - shutil.rmtree(self.directory) - super().tearDown() + pass def test_storage_new_blocks(self): self.tx_storage._always_use_topological_dfs = True @@ -634,10 +614,4 @@ class CacheRocksDBStorageTest(BaseCacheStorageTest): __test__ = True def _config_builder(self, builder: TestBuilder) -> None: - self.directory = tempfile.mkdtemp() - builder.use_rocksdb(self.directory) builder.use_tx_storage_cache(capacity=5) - - def tearDown(self): - shutil.rmtree(self.directory) - super().tearDown() diff --git a/tests/unittest.py b/tests/unittest.py index 1b3d6af5b..65b162f3c 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -23,6 +23,7 @@ from hathor.pubsub import PubSubManager from hathor.reactor import ReactorProtocol as Reactor, get_global_reactor from hathor.simulator.clock import MemoryReactorHeapClock +from hathor.storage import RocksDBStorage from hathor.transaction import BaseTransaction, Block, Transaction from hathor.transaction.storage.transaction_storage import TransactionStorage from hathor.types import VertexId @@ -33,7 +34,6 @@ logger = get_logger() main = ut_main -USE_MEMORY_STORAGE = os.environ.get('HATHOR_TEST_MEMORY_STORAGE', 'false').lower() == 'true' def short_hashes(container: Collection[bytes]) -> Iterable[str]: @@ -91,7 +91,6 @@ def _get_reactor(self) -> Reactor: class TestCase(unittest.TestCase): - use_memory_storage: bool = USE_MEMORY_STORAGE seed_config: Optional[int] = None def setUp(self) -> None: @@ -163,8 +162,8 @@ def get_dag_builder(self, manager: HathorManager) -> DAGBuilder: vertex_resolver=lambda x: manager.cpu_mining_service.resolve(x), ) - def get_builder(self) -> TestBuilder: - builder = TestBuilder() + def get_builder(self, settings: HathorSettings | None = None) -> TestBuilder: + builder = TestBuilder(settings) builder.set_rng(self.rng) \ .set_reactor(self.clock) return builder @@ -197,12 +196,10 @@ def create_peer( # type: ignore[no-untyped-def] checkpoints: list[Checkpoint] | None = None, utxo_index: bool = False, event_manager: EventManager | None = None, - use_memory_index: bool | None = None, start_manager: bool = True, pubsub: PubSubManager | None = None, event_storage: EventStorage | None = None, enable_event_queue: bool | None = None, - use_memory_storage: bool | None = None, enable_ipv6: bool = False, disable_ipv4: bool = False, ): # TODO: Add -> HathorManager here. It breaks the lint in a lot of places. @@ -237,25 +234,15 @@ def create_peer( # type: ignore[no-untyped-def] if enable_event_queue: builder.enable_event_queue() - if tx_storage is not None: - builder.set_tx_storage(tx_storage) - - if use_memory_storage or self.use_memory_storage: - builder.use_memory() - else: - directory = tempfile.mkdtemp() - self.tmpdirs.append(directory) - builder.use_rocksdb(directory) - - if use_memory_index is True: - builder.force_memory_index() - if wallet_index: builder.enable_wallet_index() if utxo_index: builder.enable_utxo_index() + if tx_storage is not None: + builder.set_tx_storage(tx_storage) + if capabilities is not None: builder.set_capabilities(capabilities) @@ -271,6 +258,14 @@ def create_peer( # type: ignore[no-untyped-def] return manager + def create_tx_storage(self, settings: HathorSettings | None = None) -> TransactionStorage: + artifacts = self.get_builder(settings).build() + return artifacts.tx_storage + + def create_rocksdb_storage(self, settings: HathorSettings | None = None) -> RocksDBStorage: + artifacts = self.get_builder(settings).build() + return not_none(artifacts.rocksdb_storage) + def run_to_completion(self) -> None: """ This will advance the test's clock until all calls scheduled are done. """ diff --git a/tests/utils.py b/tests/utils.py index 5be5c8510..54347004e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -219,7 +219,7 @@ def run_server( """ command = ' '.join([ 'python -m hathor run_node', - '--memory-storage', + '--temp-data', '--wallet hd', '--wallet-enable-api', '--hostname {}'.format(hostname),