From ae9fe2d3b979901dc3180c1bc494e8730d8bf15e Mon Sep 17 00:00:00 2001 From: Gabriel Levcovitz Date: Tue, 6 Aug 2024 14:20:34 -0300 Subject: [PATCH] feat(metadata): create basic static metadata structures --- hathor/builder/builder.py | 7 +- hathor/builder/cli_builder.py | 6 +- hathor/cli/quick_test.py | 5 +- hathor/feature_activation/feature_service.py | 2 +- hathor/transaction/base_transaction.py | 44 +++- hathor/transaction/block.py | 10 +- hathor/transaction/static_metadata.py | 64 ++++++ hathor/transaction/storage/cache_storage.py | 10 + hathor/transaction/storage/memory_storage.py | 15 +- hathor/transaction/storage/rocksdb_storage.py | 14 ++ .../storage/transaction_storage.py | 17 +- hathor/transaction/transaction.py | 13 +- hathor/verification/verification_service.py | 12 +- .../test_feature_service.py | 215 ++++++++---------- tests/others/test_metrics.py | 2 + tests/poa/test_poa_simulation.py | 2 +- tests/tx/test_block.py | 44 ++-- 17 files changed, 318 insertions(+), 164 deletions(-) create mode 100644 hathor/transaction/static_metadata.py diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index de8410bb2..455384852 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -563,7 +563,12 @@ def _get_or_create_verification_service(self) -> VerificationService: if self._verification_service is None: settings = self._get_or_create_settings() verifiers = self._get_or_create_vertex_verifiers() - self._verification_service = VerificationService(settings=settings, verifiers=verifiers) + storage = self._get_or_create_tx_storage() + self._verification_service = VerificationService( + settings=settings, + verifiers=verifiers, + tx_storage=storage, + ) return self._verification_service diff --git a/hathor/builder/cli_builder.py b/hathor/builder/cli_builder.py index 33acca41b..3f3304f2a 100644 --- a/hathor/builder/cli_builder.py +++ b/hathor/builder/cli_builder.py @@ -295,7 +295,11 @@ def create_manager(self, reactor: Reactor) -> HathorManager: daa=daa, feature_service=self.feature_service ) - verification_service = VerificationService(settings=settings, verifiers=vertex_verifiers) + verification_service = VerificationService( + settings=settings, + verifiers=vertex_verifiers, + tx_storage=tx_storage, + ) cpu_mining_service = CpuMiningService() diff --git a/hathor/cli/quick_test.py b/hathor/cli/quick_test.py index 30f8852ac..1ba6fd0ff 100644 --- a/hathor/cli/quick_test.py +++ b/hathor/cli/quick_test.py @@ -30,7 +30,8 @@ def create_parser(cls) -> ArgumentParser: return parser def prepare(self, *, register_resources: bool = True) -> None: - from hathor.transaction import BaseTransaction, Block + from hathor.transaction import Block + from hathor.transaction.base_transaction import GenericVertex super().prepare(register_resources=False) self._no_wait = self._args.no_wait @@ -47,7 +48,7 @@ def patched_on_new_tx(*args: Any, **kwargs: Any) -> bool: else: vertex = args[0] should_quit = False - assert isinstance(vertex, BaseTransaction) + assert isinstance(vertex, GenericVertex) if isinstance(vertex, Block): should_quit = vertex.get_height() >= self._args.quit_after_n_blocks diff --git a/hathor/feature_activation/feature_service.py b/hathor/feature_activation/feature_service.py index caadb62fb..9fa7ceb0b 100644 --- a/hathor/feature_activation/feature_service.py +++ b/hathor/feature_activation/feature_service.py @@ -222,7 +222,7 @@ def _get_ancestor_at_height(self, *, block: 'Block', ancestor_height: int) -> 'B if parent_block.get_height() == ancestor_height: return parent_block - if not parent_metadata.voided_by and (ancestor := self._tx_storage.get_transaction_by_height(ancestor_height)): + if not parent_metadata.voided_by and (ancestor := self._tx_storage.get_block_by_height(ancestor_height)): from hathor.transaction import Block assert isinstance(ancestor, Block) return ancestor diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index 607ddb539..d2bb14e50 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -24,13 +24,14 @@ from itertools import chain from math import inf, isfinite, log from struct import error as StructError, pack -from typing import TYPE_CHECKING, Any, ClassVar, Iterator, Optional +from typing import TYPE_CHECKING, Any, ClassVar, Generic, Iterator, Optional, TypeAlias, TypeVar, cast from structlog import get_logger from hathor.checkpoint import Checkpoint from hathor.conf.get_settings import get_global_settings from hathor.transaction.exceptions import InvalidOutputValue, WeightError +from hathor.transaction.static_metadata import VertexStaticMetadata from hathor.transaction.transaction_metadata import TransactionMetadata from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len from hathor.transaction.validation_state import ValidationState @@ -123,9 +124,11 @@ def get_cls(self) -> type['BaseTransaction']: _base_transaction_log = logger.new() +StaticMetadataT = TypeVar('StaticMetadataT', bound=VertexStaticMetadata, covariant=True) -class BaseTransaction(ABC): - """Hathor base transaction""" + +class GenericVertex(ABC, Generic[StaticMetadataT]): + """Hathor generic vertex""" # Even though nonce is serialized with different sizes for tx and blocks # the same size is used for hashes to enable mining algorithm compatibility @@ -134,6 +137,7 @@ class BaseTransaction(ABC): HEX_BASE = 16 _metadata: Optional[TransactionMetadata] + _static_metadata: StaticMetadataT | None # Bits extracted from the first byte of the version field. They carry extra information that may be interpreted # differently by each subclass of BaseTransaction. @@ -178,6 +182,7 @@ def __init__( self.parents = parents or [] self.storage = storage self._hash: VertexId | None = hash # Stored as bytes. + self._static_metadata = None @classproperty def log(cls): @@ -260,7 +265,7 @@ def __eq__(self, other: object) -> bool: :raises NotImplement: when one of the transactions do not have a calculated hash """ - if not isinstance(other, BaseTransaction): + if not isinstance(other, GenericVertex): return NotImplemented if self._hash and other._hash: return self.hash == other.hash @@ -762,7 +767,7 @@ def _update_feature_activation_bit_counts(self) -> None: from hathor.transaction import Block assert isinstance(self, Block) # This method lazily calculates and stores the value in metadata - self.get_feature_activation_bit_counts() + cast(Block, self).get_feature_activation_bit_counts() def _update_initial_accumulated_weight(self) -> None: """Update the vertex initial accumulated_weight.""" @@ -875,6 +880,8 @@ def clone(self, *, include_metadata: bool = True, include_storage: bool = True) :return: Transaction or Block copy """ new_tx = self.create_from_struct(self.get_struct()) + # static_metadata can be safely copied as it is a frozen dataclass + new_tx.set_static_metadata(self._static_metadata) if hasattr(self, '_metadata') and include_metadata: assert self._metadata is not None # FIXME: is this actually true or do we have to check if not None new_tx._metadata = self._metadata.clone() @@ -897,6 +904,33 @@ def is_ready_for_validation(self) -> bool: return False return True + @property + def static_metadata(self) -> StaticMetadataT: + """Get this vertex's static metadata. Assumes it has been initialized.""" + assert self._static_metadata is not None + return self._static_metadata + + @abstractmethod + def init_static_metadata_from_storage(self, storage: 'TransactionStorage') -> None: + """Initialize this vertex's static metadata using dependencies from a storage. This can be called multiple + times, provided the dependencies don't change.""" + raise NotImplementedError + + def set_static_metadata(self, static_metadata: StaticMetadataT | None) -> None: + """Set this vertex's static metadata. After it's set, it can only be set again to the same value.""" + assert not self._static_metadata or self._static_metadata == static_metadata, ( + 'trying to set static metadata with different values' + ) + self._static_metadata = static_metadata + + +""" +Type aliases for easily working with `GenericVertex`. A `Vertex` is a superclass that includes all specific +vertex subclasses, and a `BaseTransaction` is simply an alias to `Vertex` for backwards compatibility. +""" +Vertex: TypeAlias = GenericVertex[VertexStaticMetadata] +BaseTransaction: TypeAlias = Vertex + class TxInput: _tx: BaseTransaction # XXX: used for caching on hathor.transaction.Transaction.get_spent_tx diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 22e6d61ac..617458a8a 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -20,13 +20,15 @@ from struct import pack from typing import TYPE_CHECKING, Any, Iterator, Optional -from typing_extensions import Self +from typing_extensions import Self, override from hathor.checkpoint import Checkpoint from hathor.feature_activation.feature import Feature from hathor.feature_activation.model.feature_state import FeatureState from hathor.transaction import BaseTransaction, TxOutput, TxVersion +from hathor.transaction.base_transaction import GenericVertex from hathor.transaction.exceptions import CheckpointError +from hathor.transaction.static_metadata import BlockStaticMetadata from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len from hathor.util import not_none from hathor.utils.int import get_bit_list @@ -42,7 +44,7 @@ _SIGHASH_ALL_FORMAT_STRING = '!BBBB' -class Block(BaseTransaction): +class Block(GenericVertex[BlockStaticMetadata]): SERIALIZATION_NONCE_SIZE = 16 def __init__( @@ -427,3 +429,7 @@ def iter_transactions_in_this_block(self) -> Iterator[BaseTransaction]: bfs.skip_neighbors(tx) continue yield tx + + @override + def init_static_metadata_from_storage(self, storage: 'TransactionStorage') -> None: + raise NotImplementedError('this will be implemented') diff --git a/hathor/transaction/static_metadata.py b/hathor/transaction/static_metadata.py new file mode 100644 index 000000000..43114c0fb --- /dev/null +++ b/hathor/transaction/static_metadata.py @@ -0,0 +1,64 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +from abc import ABC +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from hathor.util import json_dumpb, json_loadb + +if TYPE_CHECKING: + from hathor.transaction import BaseTransaction + + +@dataclass(slots=True, frozen=True, kw_only=True) +class VertexStaticMetadata(ABC): + """ + Static Metadata represents vertex attributes that are not intrinsic to the vertex data, but can be calculated from + only the vertex itself and its dependencies, and whose values never change. + + This class is an abstract base class for all static metadata types that includes attributes common to all vertex + types. + """ + min_height: int + + def to_bytes(self) -> bytes: + """Convert this static metadata instance to a json bytes representation.""" + return json_dumpb(dataclasses.asdict(self)) + + @classmethod + def from_bytes(cls, data: bytes, *, target: 'BaseTransaction') -> 'VertexStaticMetadata': + """Create a static metadata instance from a json bytes representation, with a known vertex type target.""" + from hathor.transaction import Block, Transaction + json_dict = json_loadb(data) + + if isinstance(target, Block): + return BlockStaticMetadata(**json_dict) + + if isinstance(target, Transaction): + return TransactionStaticMetadata(**json_dict) + + raise NotImplementedError + + +@dataclass(slots=True, frozen=True, kw_only=True) +class BlockStaticMetadata(VertexStaticMetadata): + height: int + feature_activation_bit_counts: list[int] + + +@dataclass(slots=True, frozen=True, kw_only=True) +class TransactionStaticMetadata(VertexStaticMetadata): + pass diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index 63b9af6b3..8f9536358 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -16,11 +16,13 @@ from typing import Any, Iterator, Optional from twisted.internet import threads +from typing_extensions import override from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction +from hathor.transaction.static_metadata import VertexStaticMetadata from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.transaction.storage.tx_allow_scope import TxAllowScope @@ -164,6 +166,14 @@ def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False # call super which adds to index if needed super().save_transaction(tx, only_metadata=only_metadata) + @override + def _save_static_metadata(self, tx: BaseTransaction) -> None: + self.store._save_static_metadata(tx) + + @override + def _get_static_metadata(self, vertex: BaseTransaction) -> VertexStaticMetadata | None: + return self.store._get_static_metadata(vertex) + def get_all_genesis(self) -> set[BaseTransaction]: return self.store.get_all_genesis() diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index efb47b1ba..861e19e65 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -14,12 +14,15 @@ from typing import Any, Iterator, Optional, TypeVar +from typing_extensions import override + from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager +from hathor.transaction import BaseTransaction +from hathor.transaction.static_metadata import VertexStaticMetadata from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage -from hathor.transaction.transaction import BaseTransaction from hathor.transaction.transaction_metadata import TransactionMetadata _Clonable = TypeVar('_Clonable', BaseTransaction, TransactionMetadata) @@ -40,6 +43,7 @@ def __init__( """ self.transactions: dict[bytes, BaseTransaction] = {} self.metadata: dict[bytes, TransactionMetadata] = {} + self._static_metadata: dict[bytes, VertexStaticMetadata] = {} # Store custom key/value attributes self.attributes: dict[str, Any] = {} self._clone_if_needed = _clone_if_needed @@ -71,6 +75,7 @@ def remove_transaction(self, tx: BaseTransaction) -> None: super().remove_transaction(tx) self.transactions.pop(tx.hash, None) self.metadata.pop(tx.hash, None) + self._static_metadata.pop(tx.hash, None) def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: super().save_transaction(tx, only_metadata=only_metadata) @@ -83,6 +88,14 @@ def _save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) if meta: self.metadata[tx.hash] = self._clone(meta) + @override + def _save_static_metadata(self, tx: BaseTransaction) -> None: + self._static_metadata[tx.hash] = tx.static_metadata + + @override + def _get_static_metadata(self, vertex: BaseTransaction) -> VertexStaticMetadata | None: + return self._static_metadata.get(vertex.hash) + def transaction_exists(self, hash_bytes: bytes) -> bool: return hash_bytes in self.transactions diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index 50c7be615..faa97b590 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -15,10 +15,12 @@ from typing import TYPE_CHECKING, Iterator, Optional from structlog import get_logger +from typing_extensions import override from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager from hathor.storage import RocksDBStorage +from hathor.transaction.static_metadata import VertexStaticMetadata from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage @@ -35,6 +37,7 @@ _DB_NAME = 'data_v2.db' _CF_NAME_TX = b'tx' _CF_NAME_META = b'meta' +_CF_NAME_STATIC_META = b'static-meta' _CF_NAME_ATTR = b'attr' _CF_NAME_MIGRATIONS = b'migrations' @@ -55,6 +58,7 @@ def __init__( ) -> None: self._cf_tx = rocksdb_storage.get_or_create_column_family(_CF_NAME_TX) self._cf_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_META) + self._cf_static_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_STATIC_META) self._cf_attr = rocksdb_storage.get_or_create_column_family(_CF_NAME_ATTR) self._cf_migrations = rocksdb_storage.get_or_create_column_family(_CF_NAME_MIGRATIONS) @@ -93,6 +97,7 @@ def remove_transaction(self, tx: 'BaseTransaction') -> None: super().remove_transaction(tx) self._db.delete((self._cf_tx, tx.hash)) self._db.delete((self._cf_meta, tx.hash)) + self._db.delete((self._cf_static_meta, tx.hash)) self._remove_from_weakref(tx) def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: @@ -108,6 +113,15 @@ def _save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = Fals meta_data = self._meta_to_bytes(tx.get_metadata(use_storage=False)) self._db.put((self._cf_meta, key), meta_data) + @override + def _save_static_metadata(self, tx: 'BaseTransaction') -> None: + self._db.put((self._cf_static_meta, tx.hash), tx.static_metadata.to_bytes()) + + @override + def _get_static_metadata(self, vertex: 'BaseTransaction') -> VertexStaticMetadata | None: + data = self._db.get((self._cf_static_meta, vertex.hash)) + return VertexStaticMetadata.from_bytes(data, target=vertex) if data else None + def transaction_exists(self, hash_bytes: bytes) -> bool: may_exist, _ = self._db.key_may_exist((self._cf_tx, hash_bytes)) if not may_exist: diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 85a978b2c..32326c02c 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -32,6 +32,7 @@ from hathor.transaction.base_transaction import BaseTransaction, TxOutput from hathor.transaction.block import Block from hathor.transaction.exceptions import RewardLocked +from hathor.transaction.static_metadata import VertexStaticMetadata from hathor.transaction.storage.exceptions import ( TransactionDoesNotExist, TransactionIsNotABlock, @@ -425,6 +426,11 @@ def save_transaction(self: 'TransactionStorage', tx: BaseTransaction, *, only_me meta = tx.get_metadata() self.pre_save_validation(tx, meta) + @abstractmethod + def _save_static_metadata(self, vertex: BaseTransaction) -> None: + """Save a vertex's static metadata to this storage.""" + raise NotImplementedError + def pre_save_validation(self, tx: BaseTransaction, tx_meta: TransactionMetadata) -> None: """ Must be run before every save, will raise AssertionError or TransactionNotInAllowedScopeError @@ -545,12 +551,12 @@ def get_transaction(self, hash_bytes: bytes) -> BaseTransaction: self.post_get_validation(tx) return tx - def get_transaction_by_height(self, height: int) -> Optional[BaseTransaction]: - """Returns a transaction from the height index. This is fast.""" + def get_block_by_height(self, height: int) -> Optional[Block]: + """Return a block in the best blockchain from the height index. This is fast.""" assert self.indexes is not None ancestor_hash = self.indexes.height.get(height) - return None if ancestor_hash is None else self.get_transaction(ancestor_hash) + return None if ancestor_hash is None else self.get_block(ancestor_hash) def get_metadata(self, hash_bytes: bytes) -> Optional[TransactionMetadata]: """Returns the transaction metadata with hash `hash_bytes`. @@ -564,6 +570,11 @@ def get_metadata(self, hash_bytes: bytes) -> Optional[TransactionMetadata]: except TransactionDoesNotExist: return None + @abstractmethod + def _get_static_metadata(self, vertex: BaseTransaction) -> VertexStaticMetadata | None: + """Get a vertex's static metadata from this storage.""" + raise NotImplementedError + def get_all_transactions(self) -> Iterator[BaseTransaction]: """Return all vertices (transactions and blocks) within the allowed scope. """ diff --git a/hathor/transaction/transaction.py b/hathor/transaction/transaction.py index a787339d8..a826b6a6b 100644 --- a/hathor/transaction/transaction.py +++ b/hathor/transaction/transaction.py @@ -19,12 +19,15 @@ from struct import pack from typing import TYPE_CHECKING, Any, NamedTuple, Optional +from typing_extensions import override + from hathor.checkpoint import Checkpoint from hathor.exception import InvalidNewTransaction from hathor.reward_lock import iter_spent_rewards -from hathor.transaction import BaseTransaction, TxInput, TxOutput, TxVersion -from hathor.transaction.base_transaction import TX_HASH_SIZE +from hathor.transaction import TxInput, TxOutput, TxVersion +from hathor.transaction.base_transaction import TX_HASH_SIZE, GenericVertex from hathor.transaction.exceptions import InvalidToken +from hathor.transaction.static_metadata import TransactionStaticMetadata from hathor.transaction.util import VerboseCallback, unpack, unpack_len from hathor.types import TokenUid, VertexId from hathor.util import not_none @@ -51,7 +54,7 @@ class RewardLockedInfo(NamedTuple): blocks_needed: int -class Transaction(BaseTransaction): +class Transaction(GenericVertex[TransactionStaticMetadata]): SERIALIZATION_NONCE_SIZE = 4 @@ -389,3 +392,7 @@ def is_spending_voided_tx(self) -> bool: if meta.voided_by: return True return False + + @override + def init_static_metadata_from_storage(self, storage: 'TransactionStorage') -> None: + raise NotImplementedError('this will be implemented') diff --git a/hathor/verification/verification_service.py b/hathor/verification/verification_service.py index 3f75c0b9b..09b3563f9 100644 --- a/hathor/verification/verification_service.py +++ b/hathor/verification/verification_service.py @@ -18,6 +18,7 @@ from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, Block, MergeMinedBlock, Transaction, TxVersion from hathor.transaction.poa import PoaBlock +from hathor.transaction.storage import TransactionStorage from hathor.transaction.token_creation_tx import TokenCreationTransaction from hathor.transaction.transaction import TokenInfo from hathor.transaction.validation_state import ValidationState @@ -28,11 +29,18 @@ class VerificationService: - __slots__ = ('_settings', 'verifiers') + __slots__ = ('_settings', 'verifiers', '_tx_storage') - def __init__(self, *, settings: HathorSettings, verifiers: VertexVerifiers) -> None: + def __init__( + self, + *, + settings: HathorSettings, + verifiers: VertexVerifiers, + tx_storage: TransactionStorage | None = None, + ) -> None: self._settings = settings self.verifiers = verifiers + self._tx_storage = tx_storage def validate_basic(self, vertex: BaseTransaction, *, skip_block_weight_verification: bool = False) -> bool: """ Run basic validations (all that are possible without dependencies) and update the validation state. diff --git a/tests/feature_activation/test_feature_service.py b/tests/feature_activation/test_feature_service.py index 60c76d8bc..b81a6c812 100644 --- a/tests/feature_activation/test_feature_service.py +++ b/tests/feature_activation/test_feature_service.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import cast from unittest.mock import Mock, patch import pytest @@ -29,15 +28,18 @@ from hathor.feature_activation.model.feature_description import FeatureDescription from hathor.feature_activation.model.feature_state import FeatureState from hathor.feature_activation.settings import Settings as FeatureSettings -from hathor.transaction import Block, TransactionMetadata -from hathor.transaction.storage import TransactionStorage +from hathor.indexes import MemoryIndexesManager +from hathor.transaction import Block +from hathor.transaction.storage import TransactionMemoryStorage, TransactionStorage from hathor.transaction.validation_state import ValidationState +from hathor.util import not_none -def _get_blocks_and_storage() -> tuple[list[Block], TransactionStorage]: +@pytest.fixture +def storage() -> TransactionStorage: settings = get_global_settings() - genesis_hash = settings.GENESIS_BLOCK_HASH - blocks: list[Block] = [] + indexes = MemoryIndexesManager() + storage = TransactionMemoryStorage(indexes=indexes, settings=settings) feature_activation_bits = [ 0b0000, # 0: boundary block 0b0010, @@ -72,37 +74,18 @@ def _get_blocks_and_storage() -> tuple[list[Block], TransactionStorage]: 0b0000, # 24: boundary block 0b0000, ] - storage = Mock() for height, bits in enumerate(feature_activation_bits): - block_hash = genesis_hash if height == 0 else int.to_bytes(height, length=1, byteorder='big') - block = Block(hash=block_hash, storage=storage, signal_bits=bits) - blocks.append(block) - parent_hash = blocks[height - 1].hash - assert parent_hash is not None - block.parents = [parent_hash] - block._metadata = TransactionMetadata(height=height) - block._metadata.validation = ValidationState.FULL - - block_by_hash = {block.hash: block for block in blocks} - storage.get_transaction = Mock(side_effect=lambda hash_bytes: block_by_hash[hash_bytes]) - storage.get_transaction_by_height = Mock(side_effect=lambda h: blocks[h]) + if height == 0: + continue + parent = not_none(storage.get_block_by_height(height - 1)) + block = Block(signal_bits=bits, parents=[parent.hash], storage=storage) + block.update_hash() + block.get_metadata().validation = ValidationState.FULL + storage.save_transaction(block) + indexes.height.add_new(height, block.hash, block.timestamp) - return blocks, storage - - -@pytest.fixture -def block_mocks() -> list[Block]: - blocks, _ = _get_blocks_and_storage() - - return blocks - - -@pytest.fixture -def tx_storage() -> TransactionStorage: - _, tx_storage = _get_blocks_and_storage() - - return tx_storage + return storage @pytest.fixture @@ -114,26 +97,26 @@ def feature_settings() -> FeatureSettings: @pytest.fixture -def service(feature_settings: FeatureSettings, tx_storage: TransactionStorage) -> FeatureService: +def service(feature_settings: FeatureSettings, storage: TransactionStorage) -> FeatureService: service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() return service -def test_get_state_genesis(block_mocks: list[Block], service: FeatureService) -> None: - block = block_mocks[0] +def test_get_state_genesis(storage: TransactionStorage, service: FeatureService) -> None: + block = not_none(storage.get_block_by_height(0)) result = service.get_state(block=block, feature=Mock()) assert result == FeatureState.DEFINED @pytest.mark.parametrize('block_height', [0, 1, 2, 3]) -def test_get_state_first_interval(block_mocks: list[Block], service: FeatureService, block_height: int) -> None: - block = block_mocks[block_height] +def test_get_state_first_interval(storage: TransactionStorage, service: FeatureService, block_height: int) -> None: + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Mock()) assert result == FeatureState.DEFINED @@ -149,8 +132,7 @@ def test_get_state_first_interval(block_mocks: list[Block], service: FeatureServ ] ) def test_get_state_from_defined( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, start_height: int, expected_state: FeatureState @@ -168,10 +150,10 @@ def test_get_state_from_defined( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -181,8 +163,7 @@ def test_get_state_from_defined( @pytest.mark.parametrize('block_height', [12, 13, 14, 15, 16, 17]) @pytest.mark.parametrize('timeout_height', [8, 12]) def test_get_state_from_started_to_failed( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, timeout_height: int, ) -> None: @@ -200,10 +181,10 @@ def test_get_state_from_started_to_failed( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -213,8 +194,7 @@ def test_get_state_from_started_to_failed( @pytest.mark.parametrize('block_height', [8, 9, 10, 11]) @pytest.mark.parametrize('timeout_height', [8, 12]) def test_get_state_from_started_to_must_signal_on_timeout( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, timeout_height: int, ) -> None: @@ -232,10 +212,10 @@ def test_get_state_from_started_to_must_signal_on_timeout( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -246,8 +226,7 @@ def test_get_state_from_started_to_must_signal_on_timeout( @pytest.mark.parametrize('block_height', [8, 9, 10, 11]) @pytest.mark.parametrize('default_threshold', [0, 1, 2, 3]) def test_get_state_from_started_to_locked_in_on_default_threshold( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, default_threshold: int ) -> None: @@ -266,10 +245,10 @@ def test_get_state_from_started_to_locked_in_on_default_threshold( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -279,8 +258,7 @@ def test_get_state_from_started_to_locked_in_on_default_threshold( @pytest.mark.parametrize('block_height', [8, 9, 10, 11]) @pytest.mark.parametrize('custom_threshold', [0, 1, 2, 3]) def test_get_state_from_started_to_locked_in_on_custom_threshold( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, custom_threshold: int ) -> None: @@ -298,10 +276,10 @@ def test_get_state_from_started_to_locked_in_on_custom_threshold( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -318,8 +296,7 @@ def test_get_state_from_started_to_locked_in_on_custom_threshold( ] ) def test_get_state_from_started_to_started( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, lock_in_on_timeout: bool, timeout_height: int, @@ -338,10 +315,10 @@ def test_get_state_from_started_to_started( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -350,8 +327,7 @@ def test_get_state_from_started_to_started( @pytest.mark.parametrize('block_height', [12, 13, 14, 15]) def test_get_state_from_must_signal_to_locked_in( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, ) -> None: feature_settings = FeatureSettings.construct( @@ -368,10 +344,10 @@ def test_get_state_from_must_signal_to_locked_in( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -381,8 +357,7 @@ def test_get_state_from_must_signal_to_locked_in( @pytest.mark.parametrize('block_height', [16, 17, 18, 19]) @pytest.mark.parametrize('minimum_activation_height', [0, 4, 8, 12, 16]) def test_get_state_from_locked_in_to_active( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, minimum_activation_height: int, ) -> None: @@ -401,10 +376,10 @@ def test_get_state_from_locked_in_to_active( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -414,8 +389,7 @@ def test_get_state_from_locked_in_to_active( @pytest.mark.parametrize('block_height', [16, 17, 18, 19]) @pytest.mark.parametrize('minimum_activation_height', [17, 20, 100]) def test_get_state_from_locked_in_to_locked_in( - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, minimum_activation_height: int, ) -> None: @@ -434,10 +408,10 @@ def test_get_state_from_locked_in_to_locked_in( ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -445,7 +419,7 @@ def test_get_state_from_locked_in_to_locked_in( @pytest.mark.parametrize('block_height', [20, 21, 22, 23]) -def test_get_state_from_active(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: +def test_get_state_from_active(storage: TransactionStorage, block_height: int) -> None: feature_settings = FeatureSettings.construct( evaluation_interval=4, features={ @@ -460,10 +434,10 @@ def test_get_state_from_active(block_mocks: list[Block], tx_storage: Transaction ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) @@ -471,7 +445,7 @@ def test_get_state_from_active(block_mocks: list[Block], tx_storage: Transaction @pytest.mark.parametrize('block_height', [16, 17, 18, 19]) -def test_caching_mechanism(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: +def test_caching_mechanism(storage: TransactionStorage, block_height: int) -> None: feature_settings = FeatureSettings.construct( evaluation_interval=4, features={ @@ -484,9 +458,9 @@ def test_caching_mechanism(block_mocks: list[Block], tx_storage: TransactionStor ) } ) - service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + service = FeatureService(feature_settings=feature_settings, tx_storage=storage) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) calculate_new_state_mock = Mock(wraps=service._calculate_new_state) with patch.object(FeatureService, '_calculate_new_state', calculate_new_state_mock): @@ -503,7 +477,7 @@ def test_caching_mechanism(block_mocks: list[Block], tx_storage: TransactionStor @pytest.mark.parametrize('block_height', [16, 17, 18, 19]) -def test_is_feature_active(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: +def test_is_feature_active(storage: TransactionStorage, block_height: int) -> None: feature_settings = FeatureSettings.construct( evaluation_interval=4, features={ @@ -518,10 +492,10 @@ def test_is_feature_active(block_mocks: list[Block], tx_storage: TransactionStor ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.is_feature_active(block=block, feature=Feature.NOP_FEATURE_1) @@ -529,7 +503,7 @@ def test_is_feature_active(block_mocks: list[Block], tx_storage: TransactionStor @pytest.mark.parametrize('block_height', [12, 13, 14, 15]) -def test_get_state_from_failed(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: +def test_get_state_from_failed(storage: TransactionStorage, block_height: int) -> None: feature_settings = FeatureSettings.construct( evaluation_interval=4, features={ @@ -543,25 +517,24 @@ def test_get_state_from_failed(block_mocks: list[Block], tx_storage: Transaction ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) assert result == FeatureState.FAILED -def test_get_state_undefined_feature(block_mocks: list[Block], service: FeatureService) -> None: - block = block_mocks[10] - +def test_get_state_undefined_feature(storage: TransactionStorage, service: FeatureService) -> None: + block = not_none(storage.get_block_by_height(10)) result = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) assert result == FeatureState.DEFINED -def test_get_bits_description(tx_storage: TransactionStorage) -> None: +def test_get_bits_description(storage: TransactionStorage) -> None: criteria_mock_1 = Criteria.construct(bit=Mock(), start_height=Mock(), timeout_height=Mock(), version=Mock()) criteria_mock_2 = Criteria.construct(bit=Mock(), start_height=Mock(), timeout_height=Mock(), version=Mock()) feature_settings = FeatureSettings.construct( @@ -572,7 +545,7 @@ def test_get_bits_description(tx_storage: TransactionStorage) -> None: ) service = FeatureService( feature_settings=feature_settings, - tx_storage=tx_storage + tx_storage=storage ) service.bit_signaling_service = Mock() @@ -606,14 +579,13 @@ def get_state(self: FeatureService, *, block: Block, feature: Feature) -> Featur ) def test_get_ancestor_at_height_invalid( feature_settings: FeatureSettings, - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, ancestor_height: int ) -> None: - service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + service = FeatureService(feature_settings=feature_settings, tx_storage=storage) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) with pytest.raises(AssertionError) as e: service._get_ancestor_at_height(block=block, ancestor_height=ancestor_height) @@ -636,21 +608,23 @@ def test_get_ancestor_at_height_invalid( ) def test_get_ancestor_at_height( feature_settings: FeatureSettings, - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, ancestor_height: int ) -> None: - service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + service = FeatureService(feature_settings=feature_settings, tx_storage=storage) service.bit_signaling_service = Mock() - block = block_mocks[block_height] - result = service._get_ancestor_at_height(block=block, ancestor_height=ancestor_height) + block = not_none(storage.get_block_by_height(block_height)) + + get_block_by_height_wrapped = Mock(wraps=storage.get_block_by_height) + with patch.object(storage, 'get_block_by_height', get_block_by_height_wrapped): + result = service._get_ancestor_at_height(block=block, ancestor_height=ancestor_height) - assert result == block_mocks[ancestor_height] - assert result.get_height() == ancestor_height - assert cast(Mock, tx_storage.get_transaction_by_height).call_count == ( - 0 if block_height - ancestor_height <= 1 else 1 - ), 'this should only be called if the ancestor is deeper than one parent away' + assert get_block_by_height_wrapped.call_count == ( + 0 if block_height - ancestor_height <= 1 else 1 + ), 'this should only be called if the ancestor is deeper than one parent away' + assert result == storage.get_block_by_height(ancestor_height) + assert result.get_height() == ancestor_height @pytest.mark.parametrize( @@ -665,21 +639,23 @@ def test_get_ancestor_at_height( ) def test_get_ancestor_at_height_voided( feature_settings: FeatureSettings, - block_mocks: list[Block], - tx_storage: TransactionStorage, + storage: TransactionStorage, block_height: int, ancestor_height: int ) -> None: - service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + service = FeatureService(feature_settings=feature_settings, tx_storage=storage) service.bit_signaling_service = Mock() - block = block_mocks[block_height] - parent_block = block_mocks[block_height - 1] + block = not_none(storage.get_block_by_height(block_height)) + parent_block = not_none(storage.get_block_by_height(block_height - 1)) parent_block.get_metadata().voided_by = {b'some'} - result = service._get_ancestor_at_height(block=block, ancestor_height=ancestor_height) - assert result == block_mocks[ancestor_height] - assert result.get_height() == ancestor_height - assert cast(Mock, tx_storage.get_transaction_by_height).call_count == 0 + get_block_by_height_wrapped = Mock(wraps=storage.get_block_by_height) + with patch.object(storage, 'get_block_by_height', get_block_by_height_wrapped): + result = service._get_ancestor_at_height(block=block, ancestor_height=ancestor_height) + + assert get_block_by_height_wrapped.call_count == 0 + assert result == storage.get_block_by_height(ancestor_height) + assert result.get_height() == ancestor_height @pytest.mark.parametrize( @@ -709,8 +685,7 @@ def test_get_ancestor_at_height_voided( ] ) def test_check_must_signal( - tx_storage: TransactionStorage, - block_mocks: list[Block], + storage: TransactionStorage, bit: int, threshold: int, block_height: int, @@ -729,9 +704,9 @@ def test_check_must_signal( ) } ) - service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + service = FeatureService(feature_settings=feature_settings, tx_storage=storage) service.bit_signaling_service = Mock() - block = block_mocks[block_height] + block = not_none(storage.get_block_by_height(block_height)) result = service.is_signaling_mandatory_features(block) diff --git a/tests/others/test_metrics.py b/tests/others/test_metrics.py index 2d21d0c57..6573c43a2 100644 --- a/tests/others/test_metrics.py +++ b/tests/others/test_metrics.py @@ -108,6 +108,7 @@ def _init_manager(): b'meta': 0.0, b'attr': 0.0, b'migrations': 0.0, + b'static-meta': 0.0, b'event': 0.0, b'event-metadata': 0.0, b'feature-activation-metadata': 0.0, @@ -161,6 +162,7 @@ def _init_manager(): b'meta': 0.0, b'attr': 0.0, b'migrations': 0.0, + b'static-meta': 0.0, b'event': 0.0, b'event-metadata': 0.0, b'feature-activation-metadata': 0.0, diff --git a/tests/poa/test_poa_simulation.py b/tests/poa/test_poa_simulation.py index 33c455426..9941dabf0 100644 --- a/tests/poa/test_poa_simulation.py +++ b/tests/poa/test_poa_simulation.py @@ -185,7 +185,7 @@ def test_two_producers(self) -> None: assert set(manager1_blocks_by_height[1]) == set(manager2_blocks_by_height[1]) # but only the block from signer2 becomes non-voided, as it is in turn - non_voided_block1 = manager1.tx_storage.get_transaction_by_height(1) + non_voided_block1 = manager1.tx_storage.get_block_by_height(1) assert isinstance(non_voided_block1, PoaBlock) _assert_block_in_turn(non_voided_block1, signer2) diff --git a/tests/tx/test_block.py b/tests/tx/test_block.py index 7bef0f834..9996d9f55 100644 --- a/tests/tx/test_block.py +++ b/tests/tx/test_block.py @@ -20,26 +20,29 @@ from hathor.conf.settings import HathorSettings from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import BlockIsMissingSignal, BlockIsSignaling, FeatureService +from hathor.indexes import MemoryIndexesManager from hathor.transaction import Block, TransactionMetadata from hathor.transaction.exceptions import BlockMustSignalError from hathor.transaction.storage import TransactionMemoryStorage, TransactionStorage +from hathor.transaction.validation_state import ValidationState +from hathor.util import not_none from hathor.verification.block_verifier import BlockVerifier def test_calculate_feature_activation_bit_counts_genesis(): settings = get_global_settings() storage = TransactionMemoryStorage(settings=settings) - genesis_block = storage.get_transaction(settings.GENESIS_BLOCK_HASH) - assert isinstance(genesis_block, Block) + genesis_block = storage.get_block(settings.GENESIS_BLOCK_HASH) result = genesis_block.get_feature_activation_bit_counts() assert result == [0, 0, 0, 0] @pytest.fixture -def block_mocks() -> list[Block]: +def tx_storage() -> TransactionStorage: settings = get_global_settings() - blocks: list[Block] = [] + indexes = MemoryIndexesManager() + storage = TransactionMemoryStorage(indexes=indexes, settings=settings) feature_activation_bits = [ 0b0000, # 0: boundary block 0b1010, @@ -55,20 +58,19 @@ def block_mocks() -> list[Block]: 0b0000, ] - for i, bits in enumerate(feature_activation_bits): - genesis_hash = settings.GENESIS_BLOCK_HASH - block_hash = genesis_hash if i == 0 else b'some_hash' + for height, bits in enumerate(feature_activation_bits): + if height == 0: + continue + parent = not_none(storage.get_block_by_height(height - 1)) + block = Block(signal_bits=bits, parents=[parent.hash], storage=storage) + block.update_hash() + meta = block.get_metadata() + meta.validation = ValidationState.FULL + meta.height = height + storage.save_transaction(block) + indexes.height.add_new(height, block.hash, block.timestamp) - storage = Mock(spec_set=TransactionStorage) - storage.get_metadata = Mock(return_value=None) - - block = Block(hash=block_hash, storage=storage, signal_bits=bits) - blocks.append(block) - - get_block_parent_mock = Mock(return_value=blocks[i - 1]) - setattr(block, 'get_block_parent', get_block_parent_mock) - - return blocks + return storage @pytest.mark.parametrize( @@ -87,14 +89,12 @@ def block_mocks() -> list[Block]: ] ) def test_calculate_feature_activation_bit_counts( - block_mocks: list[Block], + tx_storage: TransactionStorage, block_height: int, expected_counts: list[int] ) -> None: - block = block_mocks[block_height] - result = block.get_feature_activation_bit_counts() - - assert result == expected_counts + block = not_none(tx_storage.get_block_by_height(block_height)) + assert block.get_feature_activation_bit_counts() == expected_counts def test_get_height():