feat(metadata): create basic static metadata structures [part 1/12]

hathor/builder/builder.py

@@ -563,7 +563,12 @@ def _get_or_create_verification_service(self) -> VerificationService:
         if self._verification_service is None:
             settings = self._get_or_create_settings()
             verifiers = self._get_or_create_vertex_verifiers()
-            self._verification_service = VerificationService(settings=settings, verifiers=verifiers)
+            storage = self._get_or_create_tx_storage()
+            self._verification_service = VerificationService(
+                settings=settings,
+                verifiers=verifiers,
+                tx_storage=storage,
+            )
 
         return self._verification_service
 

hathor/builder/cli_builder.py

@@ -295,7 +295,11 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             daa=daa,
             feature_service=self.feature_service
         )
-        verification_service = VerificationService(settings=settings, verifiers=vertex_verifiers)
+        verification_service = VerificationService(
+            settings=settings,
+            verifiers=vertex_verifiers,
+            tx_storage=tx_storage,
+        )
 
         cpu_mining_service = CpuMiningService()
 

hathor/cli/quick_test.py

@@ -30,7 +30,8 @@ def create_parser(cls) -> ArgumentParser:
         return parser
 
     def prepare(self, *, register_resources: bool = True) -> None:
-        from hathor.transaction import BaseTransaction, Block
+        from hathor.transaction import Block
+        from hathor.transaction.base_transaction import GenericVertex
         super().prepare(register_resources=False)
         self._no_wait = self._args.no_wait
 
@@ -47,7 +48,7 @@ def patched_on_new_tx(*args: Any, **kwargs: Any) -> bool:
             else:
                 vertex = args[0]
                 should_quit = False
-                assert isinstance(vertex, BaseTransaction)
+                assert isinstance(vertex, GenericVertex)
 
                 if isinstance(vertex, Block):
                     should_quit = vertex.get_height() >= self._args.quit_after_n_blocks

hathor/feature_activation/feature_service.py

@@ -222,7 +222,7 @@ def _get_ancestor_at_height(self, *, block: 'Block', ancestor_height: int) -> 'B
         if parent_block.get_height() == ancestor_height:
             return parent_block
 
-        if not parent_metadata.voided_by and (ancestor := self._tx_storage.get_transaction_by_height(ancestor_height)):
+        if not parent_metadata.voided_by and (ancestor := self._tx_storage.get_block_by_height(ancestor_height)):
             from hathor.transaction import Block
             assert isinstance(ancestor, Block)
             return ancestor

hathor/transaction/base_transaction.py

@@ -24,13 +24,14 @@
 from itertools import chain
 from math import inf, isfinite, log
 from struct import error as StructError, pack
-from typing import TYPE_CHECKING, Any, ClassVar, Iterator, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, Iterator, Optional, TypeAlias, TypeVar, cast
 
 from structlog import get_logger
 
 from hathor.checkpoint import Checkpoint
 from hathor.conf.get_settings import get_global_settings
 from hathor.transaction.exceptions import InvalidOutputValue, WeightError
+from hathor.transaction.static_metadata import VertexStaticMetadata
 from hathor.transaction.transaction_metadata import TransactionMetadata
 from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len
 from hathor.transaction.validation_state import ValidationState
@@ -123,9 +124,11 @@ def get_cls(self) -> type['BaseTransaction']:
 
 _base_transaction_log = logger.new()
 
+StaticMetadataT = TypeVar('StaticMetadataT', bound=VertexStaticMetadata, covariant=True)
 
-class BaseTransaction(ABC):
-    """Hathor base transaction"""
+
+class GenericVertex(ABC, Generic[StaticMetadataT]):
+    """Hathor generic vertex"""
 
     # Even though nonce is serialized with different sizes for tx and blocks
     # the same size is used for hashes to enable mining algorithm compatibility
@@ -134,6 +137,7 @@ class BaseTransaction(ABC):
     HEX_BASE = 16
 
     _metadata: Optional[TransactionMetadata]
+    _static_metadata: StaticMetadataT | None
 
     # Bits extracted from the first byte of the version field. They carry extra information that may be interpreted
     # differently by each subclass of BaseTransaction.
@@ -178,6 +182,7 @@ def __init__(
         self.parents = parents or []
         self.storage = storage
         self._hash: VertexId | None = hash  # Stored as bytes.
+        self._static_metadata = None
 
     @classproperty
     def log(cls):
@@ -260,7 +265,7 @@ def __eq__(self, other: object) -> bool:
 
         :raises NotImplement: when one of the transactions do not have a calculated hash
         """
-        if not isinstance(other, BaseTransaction):
+        if not isinstance(other, GenericVertex):
             return NotImplemented
         if self._hash and other._hash:
             return self.hash == other.hash
@@ -762,7 +767,7 @@ def _update_feature_activation_bit_counts(self) -> None:
         from hathor.transaction import Block
         assert isinstance(self, Block)
         # This method lazily calculates and stores the value in metadata
-        self.get_feature_activation_bit_counts()
+        cast(Block, self).get_feature_activation_bit_counts()
 
     def _update_initial_accumulated_weight(self) -> None:
         """Update the vertex initial accumulated_weight."""
@@ -875,6 +880,8 @@ def clone(self, *, include_metadata: bool = True, include_storage: bool = True)
         :return: Transaction or Block copy
         """
         new_tx = self.create_from_struct(self.get_struct())
+        # static_metadata can be safely copied as it is a frozen dataclass
+        new_tx.set_static_metadata(self._static_metadata)
         if hasattr(self, '_metadata') and include_metadata:
             assert self._metadata is not None  # FIXME: is this actually true or do we have to check if not None
             new_tx._metadata = self._metadata.clone()
@@ -897,6 +904,33 @@ def is_ready_for_validation(self) -> bool:
                 return False
         return True
 
+    @property
+    def static_metadata(self) -> StaticMetadataT:
+        """Get this vertex's static metadata. Assumes it has been initialized."""
+        assert self._static_metadata is not None
+        return self._static_metadata
+
+    @abstractmethod
+    def init_static_metadata_from_storage(self, storage: 'TransactionStorage') -> None:
+        """Initialize this vertex's static metadata using dependencies from a storage. This can be called multiple
+        times, provided the dependencies don't change."""
+        raise NotImplementedError
+
+    def set_static_metadata(self, static_metadata: StaticMetadataT | None) -> None:
+        """Set this vertex's static metadata. After it's set, it can only be set again to the same value."""
+        assert not self._static_metadata or self._static_metadata == static_metadata, (
+            'trying to set static metadata with different values'
+        )
+        self._static_metadata = static_metadata
+
+
+"""
+Type aliases for easily working with `GenericVertex`. A `Vertex` is a superclass that includes all specific
+vertex subclasses, and a `BaseTransaction` is simply an alias to `Vertex` for backwards compatibility.
+"""
+Vertex: TypeAlias = GenericVertex[VertexStaticMetadata]
+BaseTransaction: TypeAlias = Vertex
+
 
 class TxInput:
     _tx: BaseTransaction  # XXX: used for caching on hathor.transaction.Transaction.get_spent_tx

hathor/transaction/block.py

@@ -20,13 +20,15 @@
 from struct import pack
 from typing import TYPE_CHECKING, Any, Iterator, Optional
 
-from typing_extensions import Self
+from typing_extensions import Self, override
 
 from hathor.checkpoint import Checkpoint
 from hathor.feature_activation.feature import Feature
 from hathor.feature_activation.model.feature_state import FeatureState
 from hathor.transaction import BaseTransaction, TxOutput, TxVersion
+from hathor.transaction.base_transaction import GenericVertex
 from hathor.transaction.exceptions import CheckpointError
+from hathor.transaction.static_metadata import BlockStaticMetadata
 from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len
 from hathor.util import not_none
 from hathor.utils.int import get_bit_list
@@ -42,7 +44,7 @@
 _SIGHASH_ALL_FORMAT_STRING = '!BBBB'
 
 
-class Block(BaseTransaction):
+class Block(GenericVertex[BlockStaticMetadata]):
     SERIALIZATION_NONCE_SIZE = 16
 
     def __init__(
@@ -427,3 +429,7 @@ def iter_transactions_in_this_block(self) -> Iterator[BaseTransaction]:
                 bfs.skip_neighbors(tx)
                 continue
             yield tx
+
+    @override
+    def init_static_metadata_from_storage(self, storage: 'TransactionStorage') -> None:
+        raise NotImplementedError('this will be implemented')

hathor/transaction/static_metadata.py

@@ -0,0 +1,64 @@
+#  Copyright 2024 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import dataclasses
+from abc import ABC
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from hathor.util import json_dumpb, json_loadb
+
+if TYPE_CHECKING:
+    from hathor.transaction import BaseTransaction
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class VertexStaticMetadata(ABC):
+    """
+    Static Metadata represents vertex attributes that are not intrinsic to the vertex data, but can be calculated from
+    only the vertex itself and its dependencies, and whose values never change.
+
+    This class is an abstract base class for all static metadata types that includes attributes common to all vertex
+    types.
+    """
+    min_height: int
+
+    def to_bytes(self) -> bytes:
+        """Convert this static metadata instance to a json bytes representation."""
+        return json_dumpb(dataclasses.asdict(self))
+
+    @classmethod
+    def from_bytes(cls, data: bytes, *, target: 'BaseTransaction') -> 'VertexStaticMetadata':
+        """Create a static metadata instance from a json bytes representation, with a known vertex type target."""
+        from hathor.transaction import Block, Transaction
+        json_dict = json_loadb(data)
+
+        if isinstance(target, Block):
+            return BlockStaticMetadata(**json_dict)
+
+        if isinstance(target, Transaction):
+            return TransactionStaticMetadata(**json_dict)
+
+        raise NotImplementedError
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class BlockStaticMetadata(VertexStaticMetadata):
+    height: int
+    feature_activation_bit_counts: list[int]
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class TransactionStaticMetadata(VertexStaticMetadata):
+    pass

hathor/transaction/storage/cache_storage.py

@@ -16,11 +16,13 @@
 from typing import Any, Iterator, Optional
 
 from twisted.internet import threads
+from typing_extensions import override
 
 from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
 from hathor.reactor import ReactorProtocol as Reactor
 from hathor.transaction import BaseTransaction
+from hathor.transaction.static_metadata import VertexStaticMetadata
 from hathor.transaction.storage.migrations import MigrationState
 from hathor.transaction.storage.transaction_storage import BaseTransactionStorage
 from hathor.transaction.storage.tx_allow_scope import TxAllowScope
@@ -164,6 +166,14 @@ def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False
         # call super which adds to index if needed
         super().save_transaction(tx, only_metadata=only_metadata)
 
+    @override
+    def _save_static_metadata(self, tx: BaseTransaction) -> None:
+        self.store._save_static_metadata(tx)
+
+    @override
+    def _get_static_metadata(self, vertex: BaseTransaction) -> VertexStaticMetadata | None:
+        return self.store._get_static_metadata(vertex)
+
     def get_all_genesis(self) -> set[BaseTransaction]:
         return self.store.get_all_genesis()
 

hathor/transaction/storage/memory_storage.py

@@ -14,12 +14,15 @@
 
 from typing import Any, Iterator, Optional, TypeVar
 
+from typing_extensions import override
+
 from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
+from hathor.transaction import BaseTransaction
+from hathor.transaction.static_metadata import VertexStaticMetadata
 from hathor.transaction.storage.exceptions import TransactionDoesNotExist
 from hathor.transaction.storage.migrations import MigrationState
 from hathor.transaction.storage.transaction_storage import BaseTransactionStorage
-from hathor.transaction.transaction import BaseTransaction
 from hathor.transaction.transaction_metadata import TransactionMetadata
 
 _Clonable = TypeVar('_Clonable', BaseTransaction, TransactionMetadata)
@@ -40,6 +43,7 @@ def __init__(
         """
         self.transactions: dict[bytes, BaseTransaction] = {}
         self.metadata: dict[bytes, TransactionMetadata] = {}
+        self._static_metadata: dict[bytes, VertexStaticMetadata] = {}
         # Store custom key/value attributes
         self.attributes: dict[str, Any] = {}
         self._clone_if_needed = _clone_if_needed
@@ -71,6 +75,7 @@ def remove_transaction(self, tx: BaseTransaction) -> None:
         super().remove_transaction(tx)
         self.transactions.pop(tx.hash, None)
         self.metadata.pop(tx.hash, None)
+        self._static_metadata.pop(tx.hash, None)
 
     def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None:
         super().save_transaction(tx, only_metadata=only_metadata)
@@ -83,6 +88,14 @@ def _save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False)
         if meta:
             self.metadata[tx.hash] = self._clone(meta)
 
+    @override
+    def _save_static_metadata(self, tx: BaseTransaction) -> None:
+        self._static_metadata[tx.hash] = tx.static_metadata
+
+    @override
+    def _get_static_metadata(self, vertex: BaseTransaction) -> VertexStaticMetadata | None:
+        return self._static_metadata.get(vertex.hash)
+
     def transaction_exists(self, hash_bytes: bytes) -> bool:
         return hash_bytes in self.transactions
 

hathor/transaction/storage/rocksdb_storage.py

@@ -15,10 +15,12 @@
 from typing import TYPE_CHECKING, Iterator, Optional
 
 from structlog import get_logger
+from typing_extensions import override
 
 from hathor.conf.settings import HathorSettings
 from hathor.indexes import IndexesManager
 from hathor.storage import RocksDBStorage
+from hathor.transaction.static_metadata import VertexStaticMetadata
 from hathor.transaction.storage.exceptions import TransactionDoesNotExist
 from hathor.transaction.storage.migrations import MigrationState
 from hathor.transaction.storage.transaction_storage import BaseTransactionStorage
@@ -35,6 +37,7 @@
 _DB_NAME = 'data_v2.db'
 _CF_NAME_TX = b'tx'
 _CF_NAME_META = b'meta'
+_CF_NAME_STATIC_META = b'static-meta'
 _CF_NAME_ATTR = b'attr'
 _CF_NAME_MIGRATIONS = b'migrations'
 
@@ -55,6 +58,7 @@ def __init__(
     ) -> None:
         self._cf_tx = rocksdb_storage.get_or_create_column_family(_CF_NAME_TX)
         self._cf_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_META)
+        self._cf_static_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_STATIC_META)
         self._cf_attr = rocksdb_storage.get_or_create_column_family(_CF_NAME_ATTR)
         self._cf_migrations = rocksdb_storage.get_or_create_column_family(_CF_NAME_MIGRATIONS)
 
@@ -93,6 +97,7 @@ def remove_transaction(self, tx: 'BaseTransaction') -> None:
         super().remove_transaction(tx)
         self._db.delete((self._cf_tx, tx.hash))
         self._db.delete((self._cf_meta, tx.hash))
+        self._db.delete((self._cf_static_meta, tx.hash))
         self._remove_from_weakref(tx)
 
     def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None:
@@ -108,6 +113,15 @@ def _save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = Fals
         meta_data = self._meta_to_bytes(tx.get_metadata(use_storage=False))
         self._db.put((self._cf_meta, key), meta_data)
 
+    @override
+    def _save_static_metadata(self, tx: 'BaseTransaction') -> None:
+        self._db.put((self._cf_static_meta, tx.hash), tx.static_metadata.to_bytes())
+
+    @override
+    def _get_static_metadata(self, vertex: 'BaseTransaction') -> VertexStaticMetadata | None:
+        data = self._db.get((self._cf_static_meta, vertex.hash))
+        return VertexStaticMetadata.from_bytes(data, target=vertex) if data else None
+
     def transaction_exists(self, hash_bytes: bytes) -> bool:
         may_exist, _ = self._db.key_may_exist((self._cf_tx, hash_bytes))
         if not may_exist: