diff --git a/hathor/consensus/consensus.py b/hathor/consensus/consensus.py index fcb030201..3e419faca 100644 --- a/hathor/consensus/consensus.py +++ b/hathor/consensus/consensus.py @@ -70,6 +70,8 @@ def create_context(self) -> ConsensusAlgorithmContext: @cpu.profiler(key=lambda self, base: 'consensus!{}'.format(base.hash.hex())) def update(self, base: BaseTransaction) -> None: + assert base.storage is not None + assert base.storage.is_only_valid_allowed() try: self._unsafe_update(base) except Exception: @@ -107,11 +109,16 @@ def _unsafe_update(self, base: BaseTransaction) -> None: if new_best_height < best_height: self.log.warn('height decreased, re-checking mempool', prev_height=best_height, new_height=new_best_height, prev_block_tip=best_tip.hex(), new_block_tip=new_best_tip.hex()) - to_remove = storage.get_transactions_that_became_invalid() + # XXX: this method will mark as INVALID all transactions in the mempool that became invalid because of a + # reward lock + to_remove = storage.compute_transactions_that_became_invalid() if to_remove: self.log.warn('some transactions on the mempool became invalid and will be removed', count=len(to_remove)) - storage.remove_transactions(to_remove) + # XXX: because transactions in `to_remove` are marked as invalid, we need this context to be able to + # remove them + with storage.allow_invalid_context(): + storage.remove_transactions(to_remove) for tx_removed in to_remove: context.pubsub.publish(HathorEvents.CONSENSUS_TX_REMOVED, tx_hash=tx_removed.hash) diff --git a/hathor/manager.py b/hathor/manager.py index 108a6001b..a3065a55d 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -413,8 +413,9 @@ def _initialize_components(self) -> None: # self.start_profiler() if self._full_verification: self.log.debug('reset all metadata') - for tx in self.tx_storage.get_all_transactions(): - tx.reset_metadata() + with self.tx_storage.allow_partially_validated_context(): + for tx in self.tx_storage.get_all_transactions(): + tx.reset_metadata() self.log.debug('load blocks and transactions') for tx in self.tx_storage._topological_sort_dfs(): @@ -459,9 +460,11 @@ def _initialize_components(self) -> None: self.tx_storage.indexes.mempool_tips.update(tx) # XXX: move to indexes.update if self.tx_storage.indexes.deps is not None: self.sync_v2_step_validations([tx]) + self.tx_storage.save_transaction(tx, only_metadata=True) else: assert tx.validate_basic(skip_block_weight_verification=skip_block_weight_verification) - self.tx_storage.save_transaction(tx, only_metadata=True) + with self.tx_storage.allow_partially_validated_context(): + self.tx_storage.save_transaction(tx, only_metadata=True) else: # TODO: deal with invalid tx if not tx_meta.validation.is_final(): diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index c983ef739..8615b099f 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -42,7 +42,7 @@ TxValidationError, WeightError, ) -from hathor.transaction.transaction_metadata import TransactionMetadata +from hathor.transaction.transaction_metadata import TransactionMetadata, ValidationState from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len from hathor.util import classproperty @@ -482,19 +482,25 @@ def can_validate_full(self) -> bool: return True return all_exist and all_valid + def set_validation(self, validation: ValidationState) -> None: + """ This method will set the internal validation state AND the appropriate voided_by marker. + + NOTE: THIS METHOD WILL NOT SAVE THE TRANSACTION + """ + meta = self.get_metadata() + meta.validation = validation + if validation.is_fully_connected(): + self._unmark_partially_validated() + else: + self._mark_partially_validated() + def validate_checkpoint(self, checkpoints: List[Checkpoint]) -> bool: """ Run checkpoint validations and update the validation state. If no exception is raised, the ValidationState will end up as `CHECKPOINT` and return `True`. """ - from hathor.transaction.transaction_metadata import ValidationState - - meta = self.get_metadata() - self.verify_checkpoint(checkpoints) - - meta.validation = ValidationState.CHECKPOINT - self.mark_partially_validated() + self.set_validation(ValidationState.CHECKPOINT) return True def validate_basic(self, skip_block_weight_verification: bool = False) -> bool: @@ -502,14 +508,8 @@ def validate_basic(self, skip_block_weight_verification: bool = False) -> bool: If no exception is raised, the ValidationState will end up as `BASIC` and return `True`. """ - from hathor.transaction.transaction_metadata import ValidationState - - meta = self.get_metadata() - self.verify_basic(skip_block_weight_verification=skip_block_weight_verification) - - meta.validation = ValidationState.BASIC - self.mark_partially_validated() + self.set_validation(ValidationState.BASIC) return True def validate_full(self, skip_block_weight_verification: bool = False, sync_checkpoints: bool = False, @@ -523,9 +523,7 @@ def validate_full(self, skip_block_weight_verification: bool = False, sync_check meta = self.get_metadata() # skip full validation when it is a checkpoint if meta.validation.is_checkpoint(): - meta.validation = ValidationState.CHECKPOINT_FULL - # at last, remove the partially validated mark - self.unmark_partially_validated() + self.set_validation(ValidationState.CHECKPOINT_FULL) return True # XXX: in some cases it might be possible that this transaction is verified by a checkpoint but we went @@ -536,16 +534,11 @@ def validate_full(self, skip_block_weight_verification: bool = False, sync_check self.verify_basic(skip_block_weight_verification=skip_block_weight_verification) self.verify(reject_locked_reward=reject_locked_reward) - if sync_checkpoints: - meta.validation = ValidationState.CHECKPOINT_FULL - else: - meta.validation = ValidationState.FULL - - # at last, remove the partially validated mark - self.unmark_partially_validated() + validation = ValidationState.CHECKPOINT_FULL if sync_checkpoints else ValidationState.FULL + self.set_validation(validation) return True - def mark_partially_validated(self) -> None: + def _mark_partially_validated(self) -> None: """ This function is used to add the partially-validated mark from the voided-by metadata. It is idempotent: calling it multiple time has the same effect as calling it once. But it must only be called @@ -555,7 +548,7 @@ def mark_partially_validated(self) -> None: assert not tx_meta.validation.is_fully_connected() tx_meta.add_voided_by(settings.PARTIALLY_VALIDATED_ID) - def unmark_partially_validated(self) -> None: + def _unmark_partially_validated(self) -> None: """ This function is used to remove the partially-validated mark from the voided-by metadata. It is idempotent: calling it multiple time has the same effect as calling it once. But it must only be called diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index f662b22f5..dfc6b0fec 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -13,7 +13,7 @@ # limitations under the License. from collections import OrderedDict -from typing import Any, Optional, Set +from typing import Any, Iterator, Optional, Set from twisted.internet import threads @@ -208,9 +208,11 @@ def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: assert tx is not None return tx - def get_all_transactions(self): + def _get_all_transactions(self) -> Iterator[BaseTransaction]: self._flush_to_storage(self.dirty_txs.copy()) - for tx in self.store.get_all_transactions(): + # XXX: explicitly use _get_all_transaction instead of get_all_transactions because there will already be a + # TransactionCacheStorage.get_all_transactions outer method + for tx in self.store._get_all_transactions(): tx.storage = self self._save_to_weakref(tx) yield tx diff --git a/hathor/transaction/storage/exceptions.py b/hathor/transaction/storage/exceptions.py index b587f426c..1f7ff5ae0 100644 --- a/hathor/transaction/storage/exceptions.py +++ b/hathor/transaction/storage/exceptions.py @@ -41,3 +41,7 @@ class PartialMigrationError(HathorError): class OutOfOrderMigrationError(HathorError): """A migration was run before another that was before it""" + + +class TransactionNotInAllowedScopeError(TransactionDoesNotExist): + """You are trying to get a transaction that is not allowed in the current scope, treated as non-existent""" diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index af07758f9..e4cd2cf7e 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -90,7 +90,7 @@ def _get_transaction(self, hash_bytes: bytes) -> BaseTransaction: else: raise TransactionDoesNotExist(hash_bytes.hex()) - def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: + def _get_all_transactions(self) -> Iterator[BaseTransaction]: for tx in self.transactions.values(): tx = self._clone(tx) if tx.hash in self.metadata: diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index 1a03df316..5daa51815 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -122,6 +122,7 @@ def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': if not tx: raise TransactionDoesNotExist(hash_bytes.hex()) + assert tx._metadata is not None assert tx.hash == hash_bytes self._save_to_weakref(tx) @@ -146,7 +147,7 @@ def _get_tx(self, hash_bytes: bytes, tx_data: bytes) -> 'BaseTransaction': self._save_to_weakref(tx) return tx - def get_all_transactions(self, *, include_partial: bool = False) -> Iterator['BaseTransaction']: + def _get_all_transactions(self) -> Iterator['BaseTransaction']: tx: Optional['BaseTransaction'] items = self._db.iteritems(self._cf_tx) @@ -163,10 +164,6 @@ def get_all_transactions(self, *, include_partial: bool = False) -> Iterator['Ba tx = self._get_tx(hash_bytes, tx_data) assert tx is not None - if not include_partial: - assert tx._metadata is not None - if not tx._metadata.validation.is_fully_connected(): - continue yield tx def is_empty(self) -> bool: diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 476557102..14ae9b2a5 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -15,6 +15,7 @@ import hashlib from abc import ABC, abstractmethod, abstractproperty from collections import defaultdict, deque +from contextlib import AbstractContextManager from threading import Lock from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Set, Tuple, Type, cast from weakref import WeakValueDictionary @@ -28,8 +29,13 @@ from hathor.pubsub import PubSubManager from hathor.transaction.base_transaction import BaseTransaction from hathor.transaction.block import Block -from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionIsNotABlock +from hathor.transaction.storage.exceptions import ( + TransactionDoesNotExist, + TransactionIsNotABlock, + TransactionNotInAllowedScopeError, +) from hathor.transaction.storage.migrations import BaseMigration, MigrationState, add_min_height_metadata +from hathor.transaction.storage.tx_allow_scope import TxAllowScope, tx_allow_context from hathor.transaction.transaction import Transaction from hathor.transaction.transaction_metadata import TransactionMetadata from hathor.util import not_none @@ -94,6 +100,9 @@ def __init__(self) -> None: # This is a global lock used to prevent concurrent access when getting the tx lock in the dict above self._weakref_lock: Lock = Lock() + # Flag to allow/disallow partially validated vertices. + self.allow_scope: TxAllowScope = TxAllowScope.VALID + # Cache for the best block tips # This cache is updated in the consensus algorithm. self._best_block_tips_cache: Optional[List[bytes]] = None @@ -330,6 +339,43 @@ def get_transaction_from_weakref(self, hash_bytes: bytes) -> Optional[BaseTransa return None return self._tx_weakref.get(hash_bytes, None) + # TODO: check if the method bellow is currently needed + def allow_only_valid_context(self) -> AbstractContextManager[None]: + """This method is used to temporarily reset the storage back to only allow valid transactions. + + The implementation will OVERRIDE the current scope to allowing only valid transactions on the observed + storage. + """ + return tx_allow_context(self, allow_scope=TxAllowScope.VALID) + + def allow_partially_validated_context(self) -> AbstractContextManager[None]: + """This method is used to temporarily make the storage allow partially validated transactions. + + The implementation will INCLUDE allowing partially valid transactions to the current allow scope. + """ + new_allow_scope = self.allow_scope | TxAllowScope.PARTIAL + return tx_allow_context(self, allow_scope=new_allow_scope) + + def allow_invalid_context(self) -> AbstractContextManager[None]: + """This method is used to temporarily make the storage allow invalid transactions. + + The implementation will INCLUDE allowing invalid transactions to the current allow scope. + """ + new_allow_scope = self.allow_scope | TxAllowScope.INVALID + return tx_allow_context(self, allow_scope=new_allow_scope) + + def is_only_valid_allowed(self) -> bool: + """Whether only valid transactions are allowed to be returned/accepted by the storage, the default state.""" + return self.allow_scope is TxAllowScope.VALID + + def is_partially_validated_allowed(self) -> bool: + """Whether partially validated transactions are allowed to be returned/accepted by the storage.""" + return TxAllowScope.PARTIAL in self.allow_scope + + def is_invalid_allowed(self) -> bool: + """Whether invalid transactions are allowed to be returned/accepted by the storage.""" + return TxAllowScope.INVALID in self.allow_scope + def _enable_weakref(self) -> None: """ Weakref should never be disabled unless you know exactly what you are doing. """ @@ -354,7 +400,7 @@ def save_transaction(self: 'TransactionStorage', tx: BaseTransaction, *, only_me self.pre_save_validation(tx, meta) def pre_save_validation(self, tx: BaseTransaction, tx_meta: TransactionMetadata) -> None: - """ Must be run before every save, only raises AssertionError. + """ Must be run before every save, will raise AssertionError or TransactionNotInAllowedScopeError A failure means there is a bug in the code that allowed the condition to reach the "save" code. This is a last second measure to prevent persisting a bad transaction/metadata. @@ -365,6 +411,20 @@ def pre_save_validation(self, tx: BaseTransaction, tx_meta: TransactionMetadata) assert tx.hash is not None assert tx_meta.hash is not None assert tx.hash == tx_meta.hash, f'{tx.hash.hex()} != {tx_meta.hash.hex()}' + self._validate_partial_marker_consistency(tx_meta) + self._validate_transaction_in_scope(tx) + + def post_get_validation(self, tx: BaseTransaction) -> None: + """ Must be run before every save, will raise AssertionError or TransactionNotInAllowedScopeError + + A failure means there is a bug in the code that allowed the condition to reach the "get" code. This is a last + second measure to prevent getting a transaction while using the wrong scope. + """ + tx_meta = tx.get_metadata() + self._validate_partial_marker_consistency(tx_meta) + self._validate_transaction_in_scope(tx) + + def _validate_partial_marker_consistency(self, tx_meta: TransactionMetadata) -> None: voided_by = tx_meta.get_frozen_voided_by() # XXX: PARTIALLY_VALIDATED_ID must be included if the tx is fully connected and must not be included otherwise has_partially_validated_marker = settings.PARTIALLY_VALIDATED_ID in voided_by @@ -372,6 +432,11 @@ def pre_save_validation(self, tx: BaseTransaction, tx_meta: TransactionMetadata) assert (not has_partially_validated_marker) == validation_is_fully_connected, \ 'Inconsistent ValidationState and voided_by' + def _validate_transaction_in_scope(self, tx: BaseTransaction) -> None: + if not self.allow_scope.is_allowed(tx): + tx_meta = tx.get_metadata() + raise TransactionNotInAllowedScopeError(tx.hash_hex, self.allow_scope.name, tx_meta.validation.name) + @abstractmethod def remove_transaction(self, tx: BaseTransaction) -> None: """Remove the tx. @@ -483,6 +548,7 @@ def get_transaction(self, hash_bytes: bytes) -> BaseTransaction: tx = self._get_transaction(hash_bytes) else: tx = self._get_transaction(hash_bytes) + self.post_get_validation(tx) return tx def get_metadata(self, hash_bytes: bytes) -> Optional[TransactionMetadata]: @@ -497,10 +563,16 @@ def get_metadata(self, hash_bytes: bytes) -> Optional[TransactionMetadata]: except TransactionDoesNotExist: return None + def get_all_transactions(self) -> Iterator[BaseTransaction]: + """Return all vertices (transactions and blocks) within the allowed scope. + """ + for tx in self._get_all_transactions(): + if self.allow_scope.is_allowed(tx): + yield tx + @abstractmethod - def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: - # TODO: verify the following claim: - """Return all transactions that are not blocks. + def _get_all_transactions(self) -> Iterator[BaseTransaction]: + """Internal implementation that iterates over all transactions/blocks. """ raise NotImplementedError @@ -950,14 +1022,14 @@ def iter_mempool_from_best_index(self) -> Iterator[Transaction]: else: yield from self.iter_mempool_from_tx_tips() - def get_transactions_that_became_invalid(self) -> List[BaseTransaction]: + def compute_transactions_that_became_invalid(self) -> List[BaseTransaction]: """ This method will look for transactions in the mempool that have became invalid due to the reward lock. """ from hathor.transaction.transaction_metadata import ValidationState to_remove: List[BaseTransaction] = [] for tx in self.iter_mempool_from_best_index(): if tx.is_spent_reward_locked(): - tx.get_metadata().validation = ValidationState.INVALID + tx.set_validation(ValidationState.INVALID) to_remove.append(tx) return to_remove @@ -1001,6 +1073,8 @@ def reset_indexes(self) -> None: """Reset all indexes. This function should not be called unless you know what you are doing.""" assert self.indexes is not None, 'Cannot reset indexes because they have not been enabled.' self.indexes.force_clear_all() + self.update_best_block_tips_cache(None) + self._all_tips_cache = None def remove_cache(self) -> None: """Remove all caches in case we don't need it.""" diff --git a/hathor/transaction/storage/tx_allow_scope.py b/hathor/transaction/storage/tx_allow_scope.py new file mode 100644 index 000000000..f490abd0d --- /dev/null +++ b/hathor/transaction/storage/tx_allow_scope.py @@ -0,0 +1,65 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager +from enum import Flag, auto +from typing import TYPE_CHECKING, Generator + +from hathor.conf import HathorSettings +from hathor.transaction.base_transaction import BaseTransaction + +if TYPE_CHECKING: + from hathor.transaction.storage import TransactionStorage # noqa: F401 + + +settings = HathorSettings() + + +class TxAllowScope(Flag): + """ This enum is used internally to mark which "type" of transactions to allow the database to read/write + + In this context "type" it means validation level, the supported "types" are enumerated in this class and for the + purpose of filtering it can be any combination of the supported types. + """ + VALID = auto() + PARTIAL = auto() + INVALID = auto() + + def is_allowed(self, tx: BaseTransaction) -> bool: + """True means it is allowed to be used in the storage (as argument or as return), False means not allowed.""" + tx_meta = tx.get_metadata() + # XXX: partial/invalid/fully_connected never overlap and cover all possible validation states + # see hathor.transaction.transaction_metadata.ValidationState for more details + validation = tx_meta.validation + if validation.is_partial() and TxAllowScope.PARTIAL not in self: + return False + if validation.is_invalid() and TxAllowScope.INVALID not in self: + return False + # XXX: not allowing valid transactions is really specific, should we allow it? + if validation.is_fully_connected() and TxAllowScope.VALID not in self: + return False + return True + + +@contextmanager +def tx_allow_context(tx_storage: 'TransactionStorage', *, allow_scope: TxAllowScope) -> Generator[None, None, None]: + """This is used to wrap the storage with a temporary allow-scope that is reverted when the context exits""" + from hathor.transaction.storage import TransactionStorage + assert isinstance(tx_storage, TransactionStorage) + previous_allow_scope = tx_storage.allow_scope + try: + tx_storage.allow_scope = allow_scope + yield + finally: + tx_storage.allow_scope = previous_allow_scope diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index effb32a45..f2cec5ad6 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -69,7 +69,7 @@ def is_at_least_basic(self) -> bool: def is_valid(self) -> bool: """Short-hand property.""" - return self in {ValidationState.FULL, ValidationState.CHECKPOINT} + return self in {ValidationState.FULL, ValidationState.CHECKPOINT, ValidationState.CHECKPOINT_FULL} def is_checkpoint(self) -> bool: """Short-hand property.""" @@ -79,6 +79,10 @@ def is_fully_connected(self) -> bool: """Short-hand property.""" return self in {ValidationState.FULL, ValidationState.CHECKPOINT_FULL} + def is_partial(self) -> bool: + """Short-hand property.""" + return self in {ValidationState.INITIAL, ValidationState.BASIC, ValidationState.CHECKPOINT} + def is_invalid(self) -> bool: """Short-hand property.""" return self is ValidationState.INVALID diff --git a/tests/others/test_init_manager.py b/tests/others/test_init_manager.py index 2fe481543..76d1d2f2e 100644 --- a/tests/others/test_init_manager.py +++ b/tests/others/test_init_manager.py @@ -25,12 +25,12 @@ def __init__(self, *args, **kwargs): def set_first_tx(self, tx: BaseTransaction) -> None: self._first_tx = tx - def get_all_transactions(self, *, include_partial: bool = False) -> Iterator[BaseTransaction]: + def _get_all_transactions(self) -> Iterator[BaseTransaction]: skip_hash = None if self._first_tx: yield self._first_tx skip_hash = self._first_tx.hash - for tx in super().get_all_transactions(include_partial=include_partial): + for tx in super()._get_all_transactions(): if tx.hash != skip_hash: yield tx diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index 97ef9d9dc..e5a749959 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -226,17 +226,165 @@ def test_save_tx(self): def test_pre_save_validation_invalid_tx_1(self): self.tx.get_metadata().validation = ValidationState.BASIC with self.assertRaises(AssertionError): - self.validate_save(self.tx) + # XXX: avoid using validate_save because an exception could be raised for other reasons + self.tx_storage.save_transaction(self.tx) def test_pre_save_validation_invalid_tx_2(self): self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) with self.assertRaises(AssertionError): - self.validate_save(self.tx) + with self.tx_storage.allow_partially_validated_context(): + # XXX: avoid using validate_save because an exception could be raised for other reasons + self.tx_storage.save_transaction(self.tx) def test_pre_save_validation_success(self): self.tx.get_metadata().validation = ValidationState.BASIC self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) + with self.tx_storage.allow_partially_validated_context(): + # XXX: it's good to use validate_save now since we don't expect any exceptions to be raised + self.validate_save(self.tx) + + def test_allow_scope_get_all_transactions(self): + self.tx.get_metadata().validation = ValidationState.BASIC + self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) + with self.tx_storage.allow_partially_validated_context(): + self.tx_storage.save_transaction(self.tx) + only_valid_txs = list(self.tx_storage.get_all_transactions()) + self.assertNotIn(self.tx, only_valid_txs) + with self.tx_storage.allow_partially_validated_context(): + txs_that_may_be_partial = list(self.tx_storage.get_all_transactions()) + self.assertIn(self.tx, txs_that_may_be_partial) + + def test_allow_scope_topological_sort_dfs(self): + self.tx.get_metadata().validation = ValidationState.BASIC + self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) + with self.tx_storage.allow_partially_validated_context(): + self.tx_storage.save_transaction(self.tx) + only_valid_txs = list(self.tx_storage._topological_sort_dfs()) + self.assertNotIn(self.tx, only_valid_txs) + with self.tx_storage.allow_partially_validated_context(): + txs_that_may_be_partial = list(self.tx_storage._topological_sort_dfs()) + self.assertIn(self.tx, txs_that_may_be_partial) + + def test_allow_partially_validated_context(self): + from hathor.transaction.storage.exceptions import TransactionNotInAllowedScopeError + self.tx.get_metadata().validation = ValidationState.BASIC + self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + # should fail because it is out of the allowed scope + with self.assertRaises(TransactionNotInAllowedScopeError): + # XXX: avoid using validate_save because an exception could be raised for other reasons + self.tx_storage.save_transaction(self.tx) + # should succeed because a custom scope is being used + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.validate_save(self.tx) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + # should fail because it is out of the allowed scope + with self.assertRaises(TransactionNotInAllowedScopeError): + self.tx_storage.get_transaction(self.tx.hash) + # should return None since TransactionNotInAllowedScopeError inherits TransactionDoesNotExist + self.assertIsNone(self.tx_storage.get_metadata(self.tx.hash)) + # should succeed because a custom scope is being used + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.tx_storage.get_transaction(self.tx.hash) + self.assertIsNotNone(self.tx_storage.get_metadata(self.tx.hash)) + + def test_allow_invalid_context(self): + from hathor.transaction.storage.exceptions import TransactionNotInAllowedScopeError self.validate_save(self.tx) + self.tx.get_metadata().validation = ValidationState.INVALID + # XXX: should this apply to invalid too? note that we never save invalid transactions so using the + # PARTIALLY_VALIDATED_ID marker is artificial just for testing + self.tx.get_metadata().add_voided_by(settings.PARTIALLY_VALIDATED_ID) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + # should fail because it is out of the allowed scope + with self.assertRaises(TransactionNotInAllowedScopeError): + # XXX: avoid using validate_save because an exception could be raised for other reasons + self.tx_storage.save_transaction(self.tx) + # should succeed because a custom scope is being used + with self.tx_storage.allow_invalid_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + self.validate_save(self.tx) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + # should fail because it is out of the allowed scope + with self.assertRaises(TransactionNotInAllowedScopeError): + self.tx_storage.get_transaction(self.tx.hash) + # should return None since TransactionNotInAllowedScopeError inherits TransactionDoesNotExist + self.assertIsNone(self.tx_storage.get_metadata(self.tx.hash)) + # should succeed because a custom scope is being used + with self.tx_storage.allow_invalid_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + self.tx_storage.get_transaction(self.tx.hash) + self.assertIsNotNone(self.tx_storage.get_metadata(self.tx.hash)) + + def test_allow_scope_context_composing(self): + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_invalid_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_only_valid_context(): + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertTrue(self.tx_storage.is_invalid_allowed()) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + + def test_allow_scope_context_stacking(self): + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + with self.tx_storage.allow_partially_validated_context(): + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.assertFalse(self.tx_storage.is_only_valid_allowed()) + self.assertTrue(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) + self.assertTrue(self.tx_storage.is_only_valid_allowed()) + self.assertFalse(self.tx_storage.is_partially_validated_allowed()) + self.assertFalse(self.tx_storage.is_invalid_allowed()) def test_save_token_creation_tx(self): tx = create_tokens(self.manager, propagate=False) diff --git a/tests/tx/test_validation_states.py b/tests/tx/test_validation_states.py new file mode 100644 index 000000000..b844690f4 --- /dev/null +++ b/tests/tx/test_validation_states.py @@ -0,0 +1,102 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.transaction.transaction_metadata import ValidationState + + +def test_validation_states_list_unchanged(): + # XXX: if these change there are some code that make certain assumptions that should be reviewd, in particular: + # - hathor.transaction.storage.transaction_storage.tx_allow_scope.TxAllowScope.is_allowed + assert list(ValidationState), [ + ValidationState.INITIAL, + ValidationState.BASIC, + ValidationState.CHECKPOINT, + ValidationState.FULL, + ValidationState.CHECKPOINT_FULL, + ValidationState.INVALID, + ] + + +def test_validation_states_properties(): + # ValidationState.INITIAL + assert ValidationState.INITIAL.is_initial() is True + assert ValidationState.INITIAL.is_at_least_basic() is False + assert ValidationState.INITIAL.is_valid() is False + assert ValidationState.INITIAL.is_checkpoint() is False + assert ValidationState.INITIAL.is_fully_connected() is False + assert ValidationState.INITIAL.is_partial() is True + assert ValidationState.INITIAL.is_invalid() is False + assert ValidationState.INITIAL.is_final() is False + # ValidationState.BASIC + assert ValidationState.BASIC.is_initial() is False + assert ValidationState.BASIC.is_at_least_basic() is True + assert ValidationState.BASIC.is_valid() is False + assert ValidationState.BASIC.is_checkpoint() is False + assert ValidationState.BASIC.is_fully_connected() is False + assert ValidationState.BASIC.is_partial() is True + assert ValidationState.BASIC.is_invalid() is False + assert ValidationState.BASIC.is_final() is False + # ValidationState.CHECKPOINT + assert ValidationState.CHECKPOINT.is_initial() is False + assert ValidationState.CHECKPOINT.is_at_least_basic() is True + assert ValidationState.CHECKPOINT.is_valid() is True + assert ValidationState.CHECKPOINT.is_checkpoint() is True + assert ValidationState.CHECKPOINT.is_fully_connected() is False + assert ValidationState.CHECKPOINT.is_partial() is True + assert ValidationState.CHECKPOINT.is_invalid() is False + assert ValidationState.CHECKPOINT.is_final() is False + # ValidationState.FULL + assert ValidationState.FULL.is_initial() is False + assert ValidationState.FULL.is_at_least_basic() is True + assert ValidationState.FULL.is_valid() is True + assert ValidationState.FULL.is_checkpoint() is False + assert ValidationState.FULL.is_fully_connected() is True + assert ValidationState.FULL.is_partial() is False + assert ValidationState.FULL.is_invalid() is False + assert ValidationState.FULL.is_final() is True + # ValidationState.CHECKPOINT_FULL + assert ValidationState.CHECKPOINT_FULL.is_initial() is False + assert ValidationState.CHECKPOINT_FULL.is_at_least_basic() is True + assert ValidationState.CHECKPOINT_FULL.is_valid() is True + assert ValidationState.CHECKPOINT_FULL.is_checkpoint() is True + assert ValidationState.CHECKPOINT_FULL.is_fully_connected() is True + assert ValidationState.CHECKPOINT_FULL.is_partial() is False + assert ValidationState.CHECKPOINT_FULL.is_invalid() is False + assert ValidationState.CHECKPOINT_FULL.is_final() is True + # ValidationState.INVALID + assert ValidationState.INVALID.is_initial() is False + assert ValidationState.INVALID.is_at_least_basic() is False + assert ValidationState.INVALID.is_valid() is False + assert ValidationState.INVALID.is_checkpoint() is False + assert ValidationState.INVALID.is_fully_connected() is False + assert ValidationState.INVALID.is_partial() is False + assert ValidationState.INVALID.is_invalid() is True + assert ValidationState.INVALID.is_final() is True + + +def test_validation_states_partition_properties(): + # these set of properties must not overlap and must cover all states: + # - is_partial + # - is_fully_connected + # - is_invalid + # this means that: + # - for each state at most one of these properties must be true + # - for each state at least one of these properties must be true + properties = [ + ValidationState.is_partial, + ValidationState.is_fully_connected, + ValidationState.is_invalid, + ] + for state in ValidationState: + assert sum(int(prop(state)) for prop in properties) == 1