From fd6e9b6eb89d7a2278f332cbcb2d697630719aa3 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 3 Jun 2021 22:15:29 -0300 Subject: [PATCH 01/15] fix(metrics): protocol_state=None case not accounted for --- hathor/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hathor/metrics.py b/hathor/metrics.py index 95e11fd08b..426f2fe588 100644 --- a/hathor/metrics.py +++ b/hathor/metrics.py @@ -216,7 +216,8 @@ def handle_publish(self, key: HathorEvents, args: EventArguments) -> None: self.peers += 1 elif key == HathorEvents.NETWORK_PEER_DISCONNECTED: # Check if peer was ready before disconnecting - if data['protocol'].state.state_name == HathorProtocol.PeerState.READY.name: + protocol_state = data['protocol'].state + if protocol_state is not None and protocol_state.state_name == HathorProtocol.PeerState.READY.name: self.peers -= 1 else: raise ValueError('Invalid key') From 9d897aa6f9941b088888877cd6438eddcdb07d25 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 27 Nov 2020 21:50:01 -0300 Subject: [PATCH 02/15] feat(sync-v2): base classes and methods, shouldn't break current code --- hathor/checkpoint.py | 20 ++++ hathor/cli/run_node.py | 4 +- hathor/conf/mainnet.py | 17 ++++ hathor/conf/settings.py | 8 +- hathor/daa.py | 2 +- hathor/manager.py | 76 +++++++------- hathor/p2p/protocol.py | 2 +- hathor/p2p/states/peer_id.py | 2 +- hathor/protos/transaction.proto | 1 + hathor/transaction/base_transaction.py | 76 +++++++++++++- hathor/transaction/block.py | 75 +++++++++++++- hathor/transaction/exceptions.py | 4 + hathor/transaction/genesis.py | 10 +- hathor/transaction/resources/__init__.py | 2 - hathor/transaction/resources/tips.py | 109 --------------------- hathor/transaction/storage/traversal.py | 24 +++-- hathor/transaction/transaction.py | 33 ++++++- hathor/transaction/transaction_metadata.py | 96 +++++++++++++++++- hathor/util.py | 12 +++ tests/resources/transaction/test_mining.py | 2 + tests/resources/transaction/test_tips.py | 47 --------- tests/tx/test_tx.py | 12 +++ tests/utils.py | 2 +- 23 files changed, 410 insertions(+), 226 deletions(-) create mode 100644 hathor/checkpoint.py delete mode 100644 hathor/transaction/resources/tips.py delete mode 100644 tests/resources/transaction/test_tips.py diff --git a/hathor/checkpoint.py b/hathor/checkpoint.py new file mode 100644 index 0000000000..67b6845b7d --- /dev/null +++ b/hathor/checkpoint.py @@ -0,0 +1,20 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import NamedTuple + + +class Checkpoint(NamedTuple): + height: int + hash: bytes diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index a84a3867b2..771a8f8461 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -204,7 +204,7 @@ def create_wallet(): network = settings.NETWORK_NAME self.manager = HathorManager(reactor, peer_id=peer_id, network=network, hostname=hostname, tx_storage=self.tx_storage, wallet=self.wallet, wallet_index=args.wallet_index, - stratum_port=args.stratum, ssl=True) + stratum_port=args.stratum, ssl=True, checkpoints=settings.CHECKPOINTS) if args.allow_mining_without_peers: self.manager.allow_mining_without_peers() @@ -255,7 +255,6 @@ def register_resources(self, args: Namespace) -> None: PushTxResource, SubmitBlockResource, TipsHistogramResource, - TipsResource, TransactionAccWeightResource, TransactionResource, TxParentsResource, @@ -329,7 +328,6 @@ def register_resources(self, args: Namespace) -> None: (b'push_tx', PushTxResource(self.manager), root), (b'graphviz', graphviz, root), (b'tips-histogram', TipsHistogramResource(self.manager), root), - (b'tips', TipsResource(self.manager), root), (b'transaction', TransactionResource(self.manager), root), (b'transaction_acc_weight', TransactionAccWeightResource(self.manager), root), (b'dashboard_tx', DashboardTransactionResource(self.manager), root), diff --git a/hathor/conf/mainnet.py b/hathor/conf/mainnet.py index f00fc33e31..7f3a20431c 100644 --- a/hathor/conf/mainnet.py +++ b/hathor/conf/mainnet.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from hathor.checkpoint import Checkpoint as cp from hathor.conf.settings import HathorSettings SETTINGS = HathorSettings( @@ -31,4 +32,20 @@ GENESIS_TX1_HASH=bytes.fromhex('0002d4d2a15def7604688e1878ab681142a7b155cbe52a6b4e031250ae96db0a'), GENESIS_TX2_NONCE=3769, GENESIS_TX2_HASH=bytes.fromhex('0002ad8d1519daaddc8e1a37b14aac0b045129c01832281fb1c02d873c7abbf9'), + CHECKPOINTS=[ + cp(100_000, bytes.fromhex('0000000000001247073138556b4f60fff3ff6eec6521373ccee5a6526a7c10af')), + cp(200_000, bytes.fromhex('00000000000001bf13197340ae0807df2c16f4959da6054af822550d7b20e19e')), + cp(300_000, bytes.fromhex('00000000000000e1e8bdba2006cc34db3a1f20294cbe87bd52cceda245238290')), + cp(400_000, bytes.fromhex('000000000000002ae98f2a15db331d059eeed34d71f813f51d1ac1dbf3d94089')), + cp(500_000, bytes.fromhex('00000000000000036f2f7234f7bf83b5746ce9b8179922d2781efd82aa3d72bf')), + cp(600_000, bytes.fromhex('0000000000000001ad38d502f537ce757d7e732230d22434238ca215dd92cca1')), + cp(700_000, bytes.fromhex('000000000000000066f04be2f3a8607c1c71682e65e07150822fb03afcbf4035')), + cp(800_000, bytes.fromhex('0000000000000000958372b3ce24a26ce97a3b063c835e7d55c632f289f2cdb0')), + cp(900_000, bytes.fromhex('0000000000000000c9bac3c3c71a1324f66481be03ad0e5d5fbbed94fc6b8794')), + cp(1_000_000, bytes.fromhex('00000000000000001060adafe703b8aa28c7d2cfcbddf77d52e62ea0a1df5416')), + cp(1_100_000, bytes.fromhex('00000000000000000ecc349992158a3972e7a24af494a891a8d1ae3ab982ee4e')), + cp(1_200_000, bytes.fromhex('000000000000000091ddabd35a0c3984609e2892b72b14d38d23d58e1fa87c91')), + cp(1_300_000, bytes.fromhex('00000000000000000244794568649ac43e0abd4b53b1a583b6cc8e243e65f582')), + cp(1_400_000, bytes.fromhex('000000000000000011a65b1c3cba2b94ad05525ac2ec60f315bb7b204c8160c7')), + ] ) diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index 520c2d45f0..4883abb6ee 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -16,6 +16,8 @@ from math import log from typing import List, NamedTuple, Optional +from hathor.checkpoint import Checkpoint + DECIMAL_PLACES = 2 GENESIS_TOKEN_UNITS = 1 * (10**9) # 1B @@ -297,8 +299,9 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: # the difficulty of all blocks, we execute the validation every N blocks only VERIFY_WEIGHT_EVERY_N_BLOCKS: int = 1000 - # Name of whitelist capability + # Capabilities CAPABILITY_WHITELIST: str = 'whitelist' + CAPABILITY_SYNC_V2: str = 'node-sync-v2' # Where to download whitelist from WHITELIST_URL: Optional[str] = None @@ -311,3 +314,6 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: # Interval (in seconds) to collect metrics data METRICS_COLLECT_DATA_INTERVAL: int = 5 + + # Block checkpoints + CHECKPOINTS: List[Checkpoint] = [] diff --git a/hathor/daa.py b/hathor/daa.py index 296fae58f6..6e56725276 100644 --- a/hathor/daa.py +++ b/hathor/daa.py @@ -1,4 +1,4 @@ -# Copyright 2020 Hathor Labs +# Copyright 2021 Hathor Labs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/hathor/manager.py b/hathor/manager.py index 8d65d78d2f..c45dce349b 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -17,7 +17,7 @@ import sys import time from enum import Enum -from typing import Any, Iterator, List, NamedTuple, Optional, Union, cast +from typing import Any, Iterator, List, NamedTuple, Optional, Tuple, Union from structlog import get_logger from twisted.internet import defer @@ -27,6 +27,7 @@ import hathor.util from hathor import daa +from hathor.checkpoint import Checkpoint from hathor.conf import HathorSettings from hathor.consensus import ConsensusAlgorithm from hathor.exception import InvalidNewTransaction @@ -65,7 +66,7 @@ def __init__(self, reactor: IReactorCore, peer_id: Optional[PeerId] = None, netw wallet: Optional[BaseWallet] = None, tx_storage: Optional[TransactionStorage] = None, peer_storage: Optional[Any] = None, default_port: int = 40403, wallet_index: bool = False, stratum_port: Optional[int] = None, ssl: bool = True, - capabilities: Optional[List[str]] = None) -> None: + capabilities: Optional[List[str]] = None, checkpoints: Optional[List[Checkpoint]] = None) -> None: """ :param reactor: Twisted reactor which handles the mainloop and the events. :param peer_id: Id of this node. If not given, a new one is created. @@ -120,6 +121,15 @@ def __init__(self, reactor: IReactorCore, peer_id: Optional[PeerId] = None, netw self.cpu = cpu + # XXX: first checkpoint must be genesis (height=0) + self.checkpoints: List[Checkpoint] = checkpoints or [] + self.checkpoints_ready: List[bool] = [False] * len(self.checkpoints) + if not self.checkpoints or self.checkpoints[0].height > 0: + self.checkpoints.insert(0, Checkpoint(0, settings.GENESIS_BLOCK_HASH)) + self.checkpoints_ready.insert(0, True) + else: + self.checkpoints_ready[0] = True + # XXX Should we use a singleton or a new PeerStorage? [msbrogli 2018-08-29] self.pubsub = pubsub or PubSubManager(self.reactor) self.tx_storage = tx_storage or TransactionMemoryStorage() @@ -178,7 +188,7 @@ def __init__(self, reactor: IReactorCore, peer_id: Optional[PeerId] = None, netw if capabilities is not None: self.capabilities = capabilities else: - self.capabilities = [settings.CAPABILITY_WHITELIST] + self.capabilities = [settings.CAPABILITY_WHITELIST, settings.CAPABILITY_SYNC_V2] def start(self) -> None: """ A factory must be started only once. And it is usually automatically started. @@ -417,12 +427,14 @@ def get_new_tx_parents(self, timestamp: Optional[float] = None) -> List[bytes]: timestamp, [x.hex() for x in self.tx_storage.get_tx_tips(timestamp - 1)]) return [x.data for x in ret] - def generate_parent_txs(self, timestamp: float) -> 'ParentTxs': + def generate_parent_txs(self, timestamp: Optional[float]) -> 'ParentTxs': """Select which transactions will be confirmed by a new block. This method tries to return a stable result, such that for a given timestamp and storage state it will always return the same. """ + if timestamp is None: + timestamp = self.reactor.seconds() can_include_intervals = sorted(self.tx_storage.get_tx_tips(timestamp - 1)) assert can_include_intervals, 'tips cannot be empty' max_timestamp = max(int(i.begin) for i in can_include_intervals) @@ -605,45 +617,11 @@ def validate_new_tx(self, tx: BaseTransaction, skip_block_weight_verification: b raise InvalidNewTransaction('Ignoring transaction in the future {} (timestamp={}, now={})'.format( tx.hash_hex, tx.timestamp, now)) - # Verify transaction and raises an TxValidationError if tx is not valid. - tx.verify() - - if tx.is_block: - tx = cast(Block, tx) - assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost - - if not skip_block_weight_verification: - # Validate minimum block difficulty - block_weight = daa.calculate_block_difficulty(tx) - if tx.weight < block_weight - settings.WEIGHT_TOL: - raise InvalidNewTransaction( - 'Invalid new block {}: weight ({}) is smaller than the minimum weight ({})'.format( - tx.hash.hex(), tx.weight, block_weight - ) - ) - - parent_block = tx.get_block_parent() - tokens_issued_per_block = daa.get_tokens_issued_per_block(parent_block.get_metadata().height + 1) - if tx.sum_outputs != tokens_issued_per_block: - raise InvalidNewTransaction( - 'Invalid number of issued tokens tag=invalid_issued_tokens' - ' tx.hash={tx.hash_hex} issued={tx.sum_outputs} allowed={allowed}'.format( - tx=tx, - allowed=tokens_issued_per_block, - ) - ) - else: - assert tx.hash is not None # XXX: it appears that after casting this assert "casting" is lost - assert isinstance(tx, Transaction) + if self.state != self.NodeState.INITIALIZING and not tx.can_validate_full(): + raise InvalidNewTransaction('Cannot validate, missing dependency') - # Validate minimum tx difficulty - min_tx_weight = daa.minimum_tx_weight(tx) - if tx.weight < min_tx_weight - settings.WEIGHT_TOL: - raise InvalidNewTransaction( - 'Invalid new tx {}: weight ({}) is smaller than the minimum weight ({})'.format( - tx.hash_hex, tx.weight, min_tx_weight - ) - ) + # validate transaction, raises a TxValidationError if tx is not valid + tx.validate_full() return True @@ -776,3 +754,17 @@ class ParentTxs(NamedTuple): max_timestamp: int can_include: List[bytes] must_include: List[bytes] + + def get_random_parents(self) -> Tuple[bytes, bytes]: + """ Get parents from self.parents plus a random choice from self.parents_any to make it 3 in total. + + Using tuple as return type to make it explicit that the length is always 2. + """ + assert len(self.must_include) <= 1 + fill = [x for _, x in sorted(random.sample(list(enumerate(self.can_include)), 2 - len(self.must_include)))] + p1, p2 = self.must_include[:] + fill + return p1, p2 + + def get_all_tips(self) -> List[bytes]: + """All generated "tips", can_include + must_include.""" + return self.must_include + self.can_include diff --git a/hathor/p2p/protocol.py b/hathor/p2p/protocol.py index be154788c8..62b01da9ea 100644 --- a/hathor/p2p/protocol.py +++ b/hathor/p2p/protocol.py @@ -357,7 +357,7 @@ def connectionLost(self, reason: Failure) -> None: self.on_disconnect(reason) def lineLengthExceeded(self, line: str) -> None: - self.log.warn('lineLengthExceeded', line=line, line_len=len(line), max_line_len=self.MAX_LENGTH) + self.log.warn('line length exceeded', line=line, line_len=len(line), max_line_len=self.MAX_LENGTH) super(HathorLineReceiver, self).lineLengthExceeded(line) @cpu.profiler(key=lambda self: 'p2p!{}'.format(self.get_short_remote())) diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index f3c72c8396..b10e8c279c 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -97,7 +97,7 @@ def handle_peer_id(self, payload: str) -> Generator[Any, Any, None]: # is it on the whitelist? if settings.ENABLE_PEER_WHITELIST and peer.id not in protocol.node.peers_whitelist: - protocol.send_error_and_close_connection('Blocked. Get in touch with Hathor team.') + protocol.send_error_and_close_connection('Blocked (id={}). Get in touch with Hathor team.'.format(peer.id)) return if peer.id == protocol.my_peer.id: diff --git a/hathor/protos/transaction.proto b/hathor/protos/transaction.proto index 0edb568ad7..aa105dfecb 100644 --- a/hathor/protos/transaction.proto +++ b/hathor/protos/transaction.proto @@ -94,4 +94,5 @@ message Metadata { double score = 8; bytes first_block = 9; uint64 height = 10; + uint32 validation = 11; } diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index 22f36f1a43..03618168b0 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -19,9 +19,10 @@ import weakref from abc import ABC, abstractclassmethod, abstractmethod from enum import IntEnum +from itertools import chain from math import inf, isfinite, log from struct import error as StructError, pack -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, List, Optional, Set, Tuple, Type from structlog import get_logger @@ -424,8 +425,81 @@ def get_struct(self) -> bytes: struct_bytes += self.get_struct_nonce() return struct_bytes + def get_all_dependencies(self) -> Set[bytes]: + """Set of all tx-hashes needed to fully validate this tx, including parent blocks/txs and inputs.""" + return set(chain(self.parents, (i.tx_id for i in self.inputs))) + + def get_tx_dependencies(self) -> Set[bytes]: + """Set of all tx-hashes needed to fully validate this, except for block parent, i.e. only tx parents/inputs.""" + parents = self.parents[1:] if self.is_block else self.parents + return set(chain(parents, (i.tx_id for i in self.inputs))) + + def get_tx_parents(self) -> Set[bytes]: + """Set of parent tx hashes, typically used for syncing transactions.""" + return set(self.parents[1:] if self.is_block else self.parents) + + def can_validate_full(self) -> bool: + """ Check if this transaction is ready to be fully validated, either all deps are full-valid or one is invalid. + """ + assert self.storage is not None + deps = self.get_all_dependencies() + all_exist = True + all_valid = True + # either they all exist and are fully valid + for dep in deps: + meta = self.storage.get_metadata(dep) + if meta is None: + all_exist = False + continue + if not meta.validation.is_valid(): + all_valid = False + if meta.validation.is_invalid(): + # or any of them is invalid (which would make this one invalid too) + return True + return all_exist and all_valid + + def validate_basic(self, skip_block_weight_verification: bool = False) -> bool: + """ Run basic validations (all that are possible without dependencies) and update the validation state. + + If no exception is raised, the ValidationState will end up as `BASIC` and return `True`. + """ + from hathor.transaction.transaction_metadata import ValidationState + + meta = self.get_metadata() + + self.verify_basic(skip_block_weight_verification=skip_block_weight_verification) + + meta.validation = ValidationState.BASIC + return True + + def validate_full(self, skip_block_weight_verification: bool = False) -> bool: + """ Run full validations (these need access to all dependencies) and update the validation state. + + If no exception is raised, the ValidationState will end up as `FULL` and return `True`. + """ + from hathor.transaction.transaction_metadata import ValidationState + + meta = self.get_metadata() + if not meta.validation.is_at_least_basic(): + self.validate_basic(skip_block_weight_verification=skip_block_weight_verification) + + self.verify() + + meta.validation = ValidationState.FULL + return True + + @abstractmethod + def verify_basic(self, skip_block_weight_verification: bool = False) -> None: + """Basic verifications (the ones without access to dependencies: parents+inputs). Raises on error. + + To be implemented by tx/block, used by `self.validate_basic`. Should not modify the validation state.""" + raise NotImplementedError + @abstractmethod def verify(self) -> None: + """Run all verifications. Raises on error. + + To be implemented by tx/block, used by `self.validate_full`. Should not modify the validation state.""" raise NotImplementedError def verify_parents(self) -> None: diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index b5a19466e7..e59653ef64 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -16,11 +16,17 @@ from struct import pack from typing import TYPE_CHECKING, Any, Dict, List, Optional -from hathor import protos +from hathor import daa, protos from hathor.conf import HathorSettings from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, TxOutput, TxVersion -from hathor.transaction.exceptions import BlockWithInputs, BlockWithTokensError, TransactionDataError +from hathor.transaction.exceptions import ( + BlockWithInputs, + BlockWithTokensError, + InvalidBlockReward, + TransactionDataError, + WeightError, +) from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len if TYPE_CHECKING: @@ -130,6 +136,38 @@ def calculate_height(self) -> int: parent_block = self.get_block_parent() return parent_block.get_metadata().height + 1 + def get_next_block_best_chain_hash(self) -> Optional[bytes]: + """Return the hash of the next (child/left-to-right) block in the best blockchain. + """ + assert self.storage is not None + meta = self.get_metadata() + assert not meta.voided_by + + candidates = [] + for h in meta.children: + blk = self.storage.get_transaction(h) + assert blk.is_block + blk_meta = blk.get_metadata() + if blk_meta.voided_by: + continue + candidates.append(h) + + if len(candidates) == 0: + return None + assert len(candidates) == 1 + return candidates[0] + + def get_next_block_best_chain(self) -> Optional['Block']: + """Return the next (child/left-to-right)block in the best blockchain. + """ + assert self.storage is not None + h = self.get_next_block_best_chain_hash() + if h is None: + return None + tx = self.storage.get_transaction(h) + assert isinstance(tx, Block) + return tx + def get_block_parent_hash(self) -> bytes: """ Return the hash of the parent block. """ @@ -235,6 +273,39 @@ def to_json_extended(self) -> Dict[str, Any]: return json + def has_basic_block_parent(self) -> bool: + """Whether all block parent is in storage and is at least basic-valid.""" + assert self.storage is not None + parent_block_hash = self.parents[0] + if not self.storage.transaction_exists(parent_block_hash): + return False + metadata = self.storage.get_metadata(parent_block_hash) + assert metadata is not None + return metadata.validation.is_at_least_basic() + + def verify_basic(self, skip_block_weight_verification: bool = False) -> None: + """Partially run validations, the ones that need parents/inputs are skipped.""" + if not skip_block_weight_verification: + self.verify_weight() + self.verify_reward() + + def verify_weight(self) -> None: + """Validate minimum block difficulty.""" + block_weight = daa.calculate_block_difficulty(self) + if self.weight < block_weight - settings.WEIGHT_TOL: + raise WeightError(f'Invalid new block {self.hash_hex}: weight ({self.weight}) is ' + f'smaller than the minimum weight ({block_weight})') + + def verify_reward(self) -> None: + """Validate reward amount.""" + parent_block = self.get_block_parent() + tokens_issued_per_block = daa.get_tokens_issued_per_block(parent_block.get_metadata().height + 1) + if self.sum_outputs != tokens_issued_per_block: + raise InvalidBlockReward( + f'Invalid number of issued tokens tag=invalid_issued_tokens tx.hash={self.hash_hex} ' + f'issued={self.sum_outputs} allowed={tokens_issued_per_block}' + ) + def verify_no_inputs(self) -> None: inputs = getattr(self, 'inputs', None) if inputs: diff --git a/hathor/transaction/exceptions.py b/hathor/transaction/exceptions.py index b428dc3419..1717ec3521 100644 --- a/hathor/transaction/exceptions.py +++ b/hathor/transaction/exceptions.py @@ -98,6 +98,10 @@ class WeightError(TxValidationError): """Transaction not using correct weight""" +class InvalidBlockReward(TxValidationError): + """Wrong amount of issued tokens""" + + class DuplicatedParents(TxValidationError): """Transaction has duplicated parents""" diff --git a/hathor/transaction/genesis.py b/hathor/transaction/genesis.py index b6d4817ce1..851e4db913 100644 --- a/hathor/transaction/genesis.py +++ b/hathor/transaction/genesis.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from hathor.conf import HathorSettings from hathor.transaction import BaseTransaction, Block, Transaction, TxOutput -from hathor.transaction.storage import TransactionStorage + +if TYPE_CHECKING: + from hathor.transaction.storage import TransactionStorage # noqa: F401 settings = HathorSettings() @@ -46,6 +48,8 @@ GENESIS = [BLOCK_GENESIS, TX_GENESIS1, TX_GENESIS2] +GENESIS_HASHES = [settings.GENESIS_BLOCK_HASH, settings.GENESIS_TX1_HASH, settings.GENESIS_TX2_HASH] + def _get_genesis_hash() -> bytes: import hashlib @@ -60,7 +64,7 @@ def _get_genesis_hash() -> bytes: GENESIS_HASH = _get_genesis_hash() -def _get_genesis_transactions_unsafe(tx_storage: Optional[TransactionStorage]) -> List[BaseTransaction]: +def _get_genesis_transactions_unsafe(tx_storage: Optional['TransactionStorage']) -> List[BaseTransaction]: """You shouldn't get genesis directly. Please, get it from your storage instead.""" genesis = [] for tx in GENESIS: diff --git a/hathor/transaction/resources/__init__.py b/hathor/transaction/resources/__init__.py index a9e70f3fb1..3c5897ad2f 100644 --- a/hathor/transaction/resources/__init__.py +++ b/hathor/transaction/resources/__init__.py @@ -18,7 +18,6 @@ from hathor.transaction.resources.graphviz import GraphvizFullResource, GraphvizNeighboursResource from hathor.transaction.resources.mining import GetBlockTemplateResource, SubmitBlockResource from hathor.transaction.resources.push_tx import PushTxResource -from hathor.transaction.resources.tips import TipsResource from hathor.transaction.resources.tips_histogram import TipsHistogramResource from hathor.transaction.resources.transaction import TransactionResource from hathor.transaction.resources.transaction_confirmation import TransactionAccWeightResource @@ -37,7 +36,6 @@ 'TransactionResource', 'DashboardTransactionResource', 'TipsHistogramResource', - 'TipsResource', 'TxParentsResource', 'ValidateAddressResource', ] diff --git a/hathor/transaction/resources/tips.py b/hathor/transaction/resources/tips.py deleted file mode 100644 index f06c8ae760..0000000000 --- a/hathor/transaction/resources/tips.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from twisted.web import resource - -from hathor.api_util import set_cors -from hathor.cli.openapi_files.register import register_resource - - -@register_resource -class TipsResource(resource.Resource): - """ Implements a web server API to return the tips - Returns a list of tips hashes - - You must run with option `--status `. - """ - isLeaf = True - - def __init__(self, manager): - self.manager = manager - - def render_GET(self, request): - """ Get request to /tips/ that return a list of tips hashes - - 'timestamp' is an optional parameter to be used in the get_tx_tips method - - :rtype: string (json) - """ - request.setHeader(b'content-type', b'application/json; charset=utf-8') - set_cors(request, 'GET') - - timestamp = None - if b'timestamp' in request.args: - try: - timestamp = int(request.args[b'timestamp'][0]) - except ValueError: - return json.dumps({ - 'success': False, - 'message': 'Invalid timestamp parameter, expecting an integer' - }).encode('utf-8') - - tx_tips = self.manager.tx_storage.get_tx_tips(timestamp) - ret = {'success': True, 'tips': [tip.data.hex() for tip in tx_tips]} - return json.dumps(ret).encode('utf-8') - - -TipsResource.openapi = { - '/tips': { - 'x-visibility': 'private', - 'get': { - 'tags': ['transaction'], - 'operationId': 'tips', - 'summary': 'Tips', - 'description': 'Returns a list of tips hashes in hexadecimal', - 'parameters': [ - { - 'name': 'timestamp', - 'in': 'query', - 'description': 'Timestamp to search for the tips', - 'required': False, - 'schema': { - 'type': 'int' - } - } - ], - 'responses': { - '200': { - 'description': 'Success', - 'content': { - 'application/json': { - 'examples': { - 'success': { - 'summary': 'Success', - 'value': { - 'success': True, - 'tips': [ - '00002b3be4e3876e67b5e090d76dcd71cde1a30ca1e54e38d65717ba131cd22f', - '0002bb171de3490828028ec5eef3325956acb6bcffa6a50466bb9a81d38363c2' - ] - } - }, - 'error': { - 'summary': 'Invalid timestamp parameter', - 'value': { - 'success': False, - 'message': 'Invalid timestamp parameter, expecting an integer' - } - } - } - } - } - } - } - } - } -} diff --git a/hathor/transaction/storage/traversal.py b/hathor/transaction/storage/traversal.py index 098da880fc..4f541599e1 100644 --- a/hathor/transaction/storage/traversal.py +++ b/hathor/transaction/storage/traversal.py @@ -16,7 +16,7 @@ import heapq from abc import ABC, abstractmethod from itertools import chain -from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set +from typing import TYPE_CHECKING, Any, Iterable, Iterator, List, Optional, Set, Union if TYPE_CHECKING: from hathor.transaction import BaseTransaction # noqa: F401 @@ -111,17 +111,25 @@ def skip_neighbors(self, tx: 'BaseTransaction') -> None: """ self._ignore_neighbors = tx - def run(self, root: 'BaseTransaction', *, skip_root: bool = False) -> Iterator['BaseTransaction']: + def run(self, root: Union['BaseTransaction', Iterable['BaseTransaction']], *, + skip_root: bool = False) -> Iterator['BaseTransaction']: """ Run the walk. + XXX: when using multiple roots the behavior of skip_root=True is undefined. We don't have a need for any + particular behavior when one of the roots is a parent/child of another (still visit it, or skip it). + :param skip_root: Indicate whether we should include the `root` or not in the walk """ - assert root.hash is not None - self.seen.add(root.hash) - if not skip_root: - self._push_visit(root) - else: - self.add_neighbors(root) + + roots = root if isinstance(root, Iterable) else [root] + + for root in roots: + assert root.hash is not None + self.seen.add(root.hash) + if not skip_root: + self._push_visit(root) + else: + self.add_neighbors(root) while self.to_visit: tx = self._pop_visit() diff --git a/hathor/transaction/transaction.py b/hathor/transaction/transaction.py index 64842fe985..02dc4e8ae1 100644 --- a/hathor/transaction/transaction.py +++ b/hathor/transaction/transaction.py @@ -17,13 +17,16 @@ from struct import pack from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple -from hathor import protos +from hathor import daa, protos from hathor.conf import HathorSettings +from hathor.exception import InvalidNewTransaction from hathor.profiler import get_cpu_profiler from hathor.transaction import MAX_NUM_INPUTS, BaseTransaction, Block, TxInput, TxOutput, TxVersion from hathor.transaction.base_transaction import TX_HASH_SIZE from hathor.transaction.exceptions import ( ConflictingInputs, + DuplicatedParents, + IncorrectParents, InexistentInput, InputOutputMismatch, InvalidInputData, @@ -260,6 +263,34 @@ def to_json(self, decode_script: bool = False, include_metadata: bool = False) - json['tokens'] = [h.hex() for h in self.tokens] return json + def verify_basic(self, skip_block_weight_verification: bool = False) -> None: + """Partially run validations, the ones that need parents/inputs are skipped.""" + if self.is_genesis: + # TODO do genesis validation? + return + self.verify_parents_basic() + self.verify_weight() + self.verify_without_storage() + + def verify_parents_basic(self) -> None: + """Verify number and non-duplicity of parents.""" + assert self.storage is not None + + # check if parents are duplicated + parents_set = set(self.parents) + if len(self.parents) > len(parents_set): + raise DuplicatedParents('Tx has duplicated parents: {}', [tx_hash.hex() for tx_hash in self.parents]) + + if len(self.parents) != 2: + raise IncorrectParents(f'wrong number of parents (tx type): {len(self.parents)}, expecting 2') + + def verify_weight(self) -> None: + """Validate minimum tx difficulty.""" + min_tx_weight = daa.minimum_tx_weight(self) + if self.weight < min_tx_weight - settings.WEIGHT_TOL: + raise InvalidNewTransaction(f'Invalid new tx {self.hash_hex}: weight ({self.weight}) is ' + f'smaller than the minimum weight ({min_tx_weight})') + @cpu.profiler(key=lambda self: 'tx-verify!{}'.format(self.hash.hex())) def verify(self) -> None: """ Common verification for all transactions: diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 45ff683e4f..3b56e637bf 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -13,6 +13,7 @@ # limitations under the License. from collections import defaultdict +from enum import IntEnum, unique from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set from hathor import protos @@ -21,7 +22,73 @@ if TYPE_CHECKING: from weakref import ReferenceType # noqa: F401 - from hathor.transaction import BaseTransaction # noqa: F401 + from hathor.transaction import BaseTransaction + + +@unique +class ValidationState(IntEnum): + """ + + Possible transitions: + + - Initial + -> Basic: parents exist, graph information checks-out + -> Invalid: all information to reach `Basic` was available, but something doesn't check out + -> Checkpoint: is a block which hash matches a known checkpoint is a parent of a Checkpoint-valid tx + - Basic + -> Full: all parents reached `Full`, and validation+consensus ran successfully + -> Invalid: all information to reach `Full` was available, but something doesn't check out + - Checkpoint + -> Checkpoint-Full: when all the chain of parents and inputs up to the genesis exist in the database + - Full: final + - Checkpoint-Full: final + - Invalid: final + + `BASIC` means only the validations that can run without access to the dependencies (parents+inputs, except for + blocks the block parent has to exist and be at least BASIC) have been run. For example, if it's `BASIC` the weight + of a tx has been validated and is correct, but it may be spending a tx that has already been spent, we will not run + this validation until _all_ the dependencies have reached `FULL` or any of them `INVALID` (which should + automatically invalidate this tx). In theory it should be possible to have even more granular validation (if one of + the inputs exists, validate that we can spend it), but the complexity for that is too high. + + """ + INITIAL = 0 # aka, not validated + BASIC = 1 # only graph info has been validated + CHECKPOINT = 2 # validation can be safely assumed because it traces up to a known checkpoint + FULL = 3 # fully validated + CHECKPOINT_FULL = 4 # besides being checkpoint valid, it is fully connected + INVALID = -1 # not valid, this does not mean not best chain, orphan chains can be valid + + def is_at_least_basic(self) -> bool: + """Until a validation is final, it is possible to change its state when more information is available.""" + return self >= ValidationState.BASIC + + def is_valid(self) -> bool: + """Short-hand property.""" + return self in {ValidationState.FULL, ValidationState.CHECKPOINT} + + def is_checkpoint(self) -> bool: + """Short-hand property.""" + return self in {ValidationState.CHECKPOINT, ValidationState.CHECKPOINT_FULL} + + def is_fully_connected(self) -> bool: + """Short-hand property.""" + return self in {ValidationState.FULL, ValidationState.CHECKPOINT_FULL} + + def is_invalid(self) -> bool: + """Short-hand property.""" + return self is ValidationState.INVALID + + def is_final(self) -> bool: + """Until a validation is final, it is possible to change its state when more information is available.""" + return self in {ValidationState.FULL, ValidationState.CHECKPOINT_FULL, ValidationState.INVALID} + + @classmethod + def from_name(cls, name: str) -> 'ValidationState': + value = getattr(cls, name.upper(), None) + if value is None: + raise ValueError('invalid name') + return value class TransactionMetadata: @@ -37,6 +104,7 @@ class TransactionMetadata: score: float first_block: Optional[bytes] height: int + validation: ValidationState # It must be a weakref. _tx_ref: Optional['ReferenceType[BaseTransaction]'] @@ -47,6 +115,7 @@ class TransactionMetadata: def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: Optional[bytes] = None, accumulated_weight: float = 0, score: float = 0, height: int = 0) -> None: + from hathor.transaction.genesis import is_genesis # Hash of the transaction. self.hash = hash @@ -95,6 +164,13 @@ def __init__(self, spent_outputs: Optional[Dict[int, List[bytes]]] = None, hash: # Height self.height = height + # Validation + self.validation = ValidationState.INITIAL + + # Genesis specific: + if hash is not None and is_genesis(hash): + self.validation = ValidationState.FULL + def get_tx(self) -> 'BaseTransaction': assert self._tx_ref is not None tx = self._tx_ref() @@ -152,7 +228,7 @@ def __eq__(self, other: Any) -> bool: return False for field in ['hash', 'conflict_with', 'voided_by', 'received_by', 'children', 'accumulated_weight', 'twins', 'score', - 'first_block']: + 'first_block', 'validation']: if (getattr(self, field) or None) != (getattr(other, field) or None): return False @@ -190,10 +266,13 @@ def to_json(self) -> Dict[str, Any]: data['first_block'] = self.first_block.hex() else: data['first_block'] = None + data['validation'] = self.validation.name.lower() return data @classmethod def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata': + from hathor.transaction.genesis import is_genesis + meta = cls() meta.hash = bytes.fromhex(data['hash']) if data['hash'] else None for idx, hashes in data['spent_outputs']: @@ -225,6 +304,12 @@ def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata': if first_block_raw: meta.first_block = bytes.fromhex(first_block_raw) + _val_name = data.get('validation', None) + meta.validation = ValidationState.from_name(_val_name) if _val_name is not None else ValidationState.INITIAL + + if meta.hash is not None and is_genesis(meta.hash): + meta.validation = ValidationState.FULL + return meta # XXX(jansegre): I did not put the transaction hash in the protobuf object to keep it less redundant. Is this OK? @@ -241,6 +326,8 @@ def create_from_proto(cls, hash_bytes: bytes, metadata_proto: protos.Metadata) - :return: A transaction metadata :rtype: TransactionMetadata """ + from hathor.transaction.genesis import is_genesis + metadata = cls(hash=hash_bytes) for i, hashes in metadata_proto.spent_outputs.items(): metadata.spent_outputs[i] = list(hashes.hashes) @@ -253,6 +340,9 @@ def create_from_proto(cls, hash_bytes: bytes, metadata_proto: protos.Metadata) - metadata.score = metadata_proto.score metadata.first_block = metadata_proto.first_block or None metadata.height = metadata_proto.height + metadata.validation = ValidationState(metadata_proto.validation) + if is_genesis(hash_bytes): + metadata.validation = ValidationState.FULL return metadata def to_proto(self) -> protos.Metadata: @@ -261,7 +351,6 @@ def to_proto(self) -> protos.Metadata: :return: Protobuf object :rtype: :py:class:`hathor.protos.Metadata` """ - from hathor import protos return protos.Metadata( spent_outputs={k: protos.Metadata.Hashes(hashes=v) for k, v in self.spent_outputs.items()}, @@ -274,6 +363,7 @@ def to_proto(self) -> protos.Metadata: score=self.score, first_block=self.first_block, height=self.height, + validation=self.validation, ) def clone(self) -> 'TransactionMetadata': diff --git a/hathor/util.py b/hathor/util.py index 8c7e8b624b..b4be9ccc94 100644 --- a/hathor/util.py +++ b/hathor/util.py @@ -262,3 +262,15 @@ def __str__(x): else: return '<1ms' __repr__ = __str__ + + +_T = TypeVar("_T") + + +# borrowed from: https://github.com/facebook/pyre-check/blob/master/pyre_extensions/__init__.py +def not_none(optional: Optional[_T], message: str = 'Unexpected `None`') -> _T: + """Convert an optional to its value. Raises an `AssertionError` if the + value is `None`""" + if optional is None: + raise AssertionError(message) + return optional diff --git a/tests/resources/transaction/test_mining.py b/tests/resources/transaction/test_mining.py index d6d4c64edf..b233a365f9 100644 --- a/tests/resources/transaction/test_mining.py +++ b/tests/resources/transaction/test_mining.py @@ -30,6 +30,7 @@ def test_get_block_template_with_address(self): 'conflict_with': [], 'voided_by': [], 'twins': [], + 'validation': 'initial', 'accumulated_weight': 1.0, 'score': 0, 'height': 1, @@ -59,6 +60,7 @@ def test_get_block_template_without_address(self): 'conflict_with': [], 'voided_by': [], 'twins': [], + 'validation': 'initial', # FIXME: change to 'full' when validations are enabled 'accumulated_weight': 1.0, 'score': 0, 'height': 1, diff --git a/tests/resources/transaction/test_tips.py b/tests/resources/transaction/test_tips.py deleted file mode 100644 index 25994ce005..0000000000 --- a/tests/resources/transaction/test_tips.py +++ /dev/null @@ -1,47 +0,0 @@ -from twisted.internet.defer import inlineCallbacks - -from hathor.transaction.resources import TipsResource -from tests.resources.base_resource import StubSite, _BaseResourceTest -from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_transactions - - -class TipsTest(_BaseResourceTest._ResourceTest): - def setUp(self): - super().setUp() - self.web = StubSite(TipsResource(self.manager)) - - @inlineCallbacks - def test_get_tips(self): - genesis_txs = [tx for tx in self.manager.tx_storage.get_all_genesis() if not tx.is_block] - - # Tips are only the genesis - response1 = yield self.web.get("tips") - data1 = response1.json_value() - self.assertTrue(data1['success']) - self.assertEqual(len(data1['tips']), len(genesis_txs)) - - self.manager.wallet.unlock(b'MYPASS') - - # Add blocks to have funds - add_new_blocks(self.manager, 2, advance_clock=1) - add_blocks_unlock_reward(self.manager) - - # Add one tx, now you have only one tip - tx = add_new_transactions(self.manager, 1)[0] - - response2 = yield self.web.get("tips") - data2 = response2.json_value() - self.assertTrue(data2['success']) - self.assertEqual(len(data2['tips']), 1) - - # Getting tips sending timestamp as parameter - response3 = yield self.web.get("tips", {b'timestamp': tx.timestamp - 1}) - data3 = response3.json_value() - self.assertEqual(len(data3), 2) - - @inlineCallbacks - def test_invalid_data(self): - # invalid timestamp parameter - response = yield self.web.get("tips", {b'timestamp': 'a'}) - data = response.json_value() - self.assertFalse(data['success']) diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 407699b04a..d84e54d88c 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -80,6 +80,18 @@ def test_input_output_match(self): with self.assertRaises(InputOutputMismatch): tx.verify_sum() + def test_validation(self): + # add 100 blocks and check that walking through get_next_block_best_chain yields the same blocks + blocks = add_new_blocks(self.manager, 100, advance_clock=1) + iblocks = iter(blocks) + block_from_chain = self.last_block + for _ in range(100): + block_from_list = next(iblocks) + block_from_chain = block_from_chain.get_next_block_best_chain() + self.assertEqual(block_from_chain, block_from_list) + self.assertTrue(block_from_chain.has_basic_block_parent()) + self.assertEqual(block_from_chain.get_next_block_best_chain(), None) + def test_script(self): genesis_block = self.genesis_blocks[0] diff --git a/tests/utils.py b/tests/utils.py index 058bd11658..5fd91a8b61 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -160,7 +160,7 @@ def add_new_block(manager, advance_clock=None, *, parent_block_hash=None, data=b if weight is not None: block.weight = weight block.resolve() - block.verify() + block.validate_full() manager.propagate_tx(block, fails_silently=False) if advance_clock: manager.reactor.advance(advance_clock) From e3766ebf28f980f5cc5529408324d442d0840c95 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Mon, 14 Jun 2021 22:29:11 -0300 Subject: [PATCH 03/15] feat(cli): option to log in plain JSON --- hathor/cli/main.py | 6 +++++- hathor/cli/util.py | 29 +++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/hathor/cli/main.py b/hathor/cli/main.py index 32cb80d458..6e0f482913 100644 --- a/hathor/cli/main.py +++ b/hathor/cli/main.py @@ -135,7 +135,11 @@ def execute_from_command_line(self): pudb.set_trace(paused=False) capture_stdout = False - setup_logging(debug, capture_stdout) + json_logs = '--json-logs' in sys.argv + if json_logs: + sys.argv.remove('--json-logs') + + setup_logging(debug, capture_stdout, json_logs) module.main() diff --git a/hathor/cli/util.py b/hathor/cli/util.py index d2f3596e49..590868d077 100644 --- a/hathor/cli/util.py +++ b/hathor/cli/util.py @@ -23,7 +23,13 @@ def create_parser() -> ArgumentParser: return configargparse.ArgumentParser(auto_env_var_prefix='hathor_') -def setup_logging(debug: bool = False, capture_stdout: bool = False, *, _test_logging: bool = False) -> None: +def setup_logging( + debug: bool = False, + capture_stdout: bool = False, + json_logging: bool = False, + *, + _test_logging: bool = False, + ) -> None: import logging import logging.config @@ -142,6 +148,11 @@ def _repr(self, val): else: return super()._repr(val) + if json_logging: + handlers = ['json'] + else: + handlers = ['pretty'] + # See: https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema logging.config.dictConfig({ 'version': 1, @@ -157,13 +168,23 @@ def _repr(self, val): 'processor': ConsoleRenderer(colors=True), 'foreign_pre_chain': pre_chain, }, + 'json': { + '()': structlog.stdlib.ProcessorFormatter, + 'processor': structlog.processors.JSONRenderer(), + 'foreign_pre_chain': pre_chain, + }, }, 'handlers': { - 'default': { + 'pretty': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', 'formatter': 'colored', }, + 'json': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'json', + }, # 'file': { # 'level': 'DEBUG', # 'class': 'logging.handlers.WatchedFileHandler', @@ -174,12 +195,12 @@ def _repr(self, val): 'loggers': { # set twisted verbosity one level lower than hathor's 'twisted': { - 'handlers': ['default'], + 'handlers': handlers, 'level': 'INFO' if debug else 'WARN', 'propagate': False, }, '': { - 'handlers': ['default'], + 'handlers': handlers, 'level': 'DEBUG' if debug else 'INFO', }, } From 2cdf41b5137902a97d120cbd25f6c40eb75d53c2 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Wed, 16 Jun 2021 16:03:36 -0300 Subject: [PATCH 04/15] chore(ci): use codecov for minimum coverage check --- codecov.yml => .codecov.yml | 12 ++++++++++-- Makefile | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) rename codecov.yml => .codecov.yml (55%) diff --git a/codecov.yml b/.codecov.yml similarity index 55% rename from codecov.yml rename to .codecov.yml index 696b99cadd..dc5e22a6a5 100644 --- a/codecov.yml +++ b/.codecov.yml @@ -1,11 +1,19 @@ codecov: + branch: dev + +coverage: + # https://docs.codecov.com/docs/coverage-configuration + range: "80...90" # https://docs.codecov.io/docs/commit-status status: + # TODO: re-enable patch in the future + patch: off project: default: # minimum coverage ratio that the commit must meet to be considered a success - target: 82% + target: 83% if_ci_failed: error only_pulls: true + github_checks: - annotations: false + annotations: true diff --git a/Makefile b/Makefile index fc9c86de19..65c2b20327 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ tests-doctests: .PHONY: tests-lib tests-lib: - pytest --durations=10 $(pytest_flags) --doctest-modules hathor --cov-fail-under=83 $(tests_lib) + pytest --durations=10 $(pytest_flags) --doctest-modules hathor $(tests_lib) .PHONY: tests-genesis tests-genesis: @@ -46,7 +46,7 @@ tests: tests-cli tests-lib tests-genesis .PHONY: tests-full tests-full: - pytest $(pytest_flags) --durations=10 --cov-fail-under=90 --cov-config=.coveragerc_full ./tests + pytest $(pytest_flags) --durations=10 --cov-config=.coveragerc_full ./tests # checking: From 2bc43f217e9f215d48aadbb6435af9d5093dffa0 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 11 Jun 2021 04:37:14 -0300 Subject: [PATCH 05/15] feat(cli): param to enable non-standard transaction and higher max-size --- hathor/cli/run_node.py | 8 +++++++- hathor/transaction/resources/push_tx.py | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 771a8f8461..41c911f16b 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -69,6 +69,10 @@ def create_parser(self) -> ArgumentParser: 'This is still a beta feature as it may cause issues when restarting the full node ' 'after a crash.') parser.add_argument('--procname-prefix', help='Add a prefix to the process name', default='') + parser.add_argument('--allow-non-standard-script', action='store_true', help='Accept non-standard scripts on ' + '/push-tx API') + parser.add_argument('--max-output-script-size', type=int, default=None, help='Custom max accepted script size ' + 'on /push-tx API') return parser def prepare(self, args: Namespace) -> None: @@ -325,7 +329,9 @@ def register_resources(self, args: Namespace) -> None: (b'create_tx', CreateTxResource(self.manager), root), (b'decode_tx', DecodeTxResource(self.manager), root), (b'validate_address', ValidateAddressResource(self.manager), root), - (b'push_tx', PushTxResource(self.manager), root), + (b'push_tx', + PushTxResource(self.manager, args.max_output_script_size, args.allow_non_standard_script), + root), (b'graphviz', graphviz, root), (b'tips-histogram', TipsHistogramResource(self.manager), root), (b'transaction', TransactionResource(self.manager), root), diff --git a/hathor/transaction/resources/push_tx.py b/hathor/transaction/resources/push_tx.py index 179d10f8a7..fb596575ea 100644 --- a/hathor/transaction/resources/push_tx.py +++ b/hathor/transaction/resources/push_tx.py @@ -47,7 +47,11 @@ def __init__(self, manager: 'HathorManager', max_output_script_size: Optional[in allow_non_standard_script: bool = False) -> None: # Important to have the manager so we can know the tx_storage self.manager = manager - self.max_output_script_size: int = settings.PUSHTX_MAX_OUTPUT_SCRIPT_SIZE + self.max_output_script_size: int = ( + settings.PUSHTX_MAX_OUTPUT_SCRIPT_SIZE + if max_output_script_size is None else + max_output_script_size + ) self.allow_non_standard_script = allow_non_standard_script def handle_push_tx(self, params: Dict[str, Any]) -> Dict[str, Any]: From dd5836590bfc5458941e8e2cbffe53cb3c32ec02 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 11 Jun 2021 03:39:08 -0300 Subject: [PATCH 06/15] refactor(storage): remove unused async/remote/subprocess methods/storages --- Makefile | 8 +- hathor/protos/__init__.py | 94 -- hathor/protos/transaction_storage.proto | 180 ---- hathor/transaction/storage/__init__.py | 9 - hathor/transaction/storage/binary_storage.py | 4 +- hathor/transaction/storage/cache_storage.py | 53 +- hathor/transaction/storage/compact_storage.py | 4 +- hathor/transaction/storage/memory_storage.py | 4 +- hathor/transaction/storage/remote_storage.py | 891 ------------------ hathor/transaction/storage/rocksdb_storage.py | 4 +- .../transaction/storage/subprocess_storage.py | 78 -- .../storage/transaction_storage.py | 112 +-- tests/p2p/test_double_spending.py | 14 +- tests/p2p/test_sync.py | 22 +- tests/resources/transaction/test_create_tx.py | 13 +- tests/resources/transaction/test_tx.py | 13 +- tests/tx/test_cache_storage.py | 63 +- tests/tx/test_remote_storage.py | 72 -- tests/tx/test_tx_serialization.py | 1 - tests/tx/test_tx_storage.py | 46 - tests/utils.py | 30 - 21 files changed, 22 insertions(+), 1693 deletions(-) delete mode 100644 hathor/protos/transaction_storage.proto delete mode 100644 hathor/transaction/storage/remote_storage.py delete mode 100644 hathor/transaction/storage/subprocess_storage.py delete mode 100644 tests/tx/test_remote_storage.py diff --git a/Makefile b/Makefile index 65c2b20327..ea5ba19635 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,9 @@ isort: proto_dir = ./hathor/protos proto_srcs = $(wildcard $(proto_dir)/*.proto) -proto_outputs = $(patsubst %.proto,%_pb2.py,$(proto_srcs)) $(patsubst %.proto,%_pb2_grpc.py,$(proto_srcs)) $(patsubst %.proto,%_pb2.pyi,$(proto_srcs)) +proto_srcs_old = $(proto_dir)/transaction_storage.proto +proto_outputs = $(patsubst %.proto,%_pb2.py,$(proto_srcs)) $(patsubst %.proto,%_pb2.pyi,$(proto_srcs)) +proto_outputs_old = $(patsubst %.proto,%_pb2_grpc.py,$(proto_srcs) $(proto_srcs_old)) GRPC_TOOLS_VERSION = "$(shell python -m grpc_tools.protoc --version 2>/dev/null || true)" #ifdef GRPC_TOOLS_VERSION ifneq ($(GRPC_TOOLS_VERSION),"") @@ -94,8 +96,6 @@ endif # all proto_srcs are added as deps so when a change on any of them triggers all to be rebuilt %_pb2.pyi %_pb2.py: %.proto $(proto_srcs) $(protoc) -I. --python_out=. --mypy_out=. $< -%_pb2_grpc.py: %.proto $(proto_srcs) - $(protoc) -I. --grpc_python_out=. $< || true .PHONY: protos protos: $(proto_outputs) @@ -104,7 +104,7 @@ protos: $(proto_outputs) .PHONY: clean-protos clean-protos: - rm -f $(proto_outputs) + rm -f $(proto_outputs) $(proto_outputs_old) .PHONY: clean-pyc clean-pyc: diff --git a/hathor/protos/__init__.py b/hathor/protos/__init__.py index b6f58c3430..b0a453ad14 100644 --- a/hathor/protos/__init__.py +++ b/hathor/protos/__init__.py @@ -22,56 +22,6 @@ TxInput, TxOutput, ) -from hathor.protos.transaction_storage_pb2 import ( - ANY_ORDER, - ANY_TYPE, - ASC_ORDER, - BLOCK_TYPE, - FOR_CACHING, - LEFT_RIGHT_ORDER_CHILDREN, - LEFT_RIGHT_ORDER_SPENT, - NO_FILTER, - ONLY_NEWER, - ONLY_OLDER, - TOPOLOGICAL_ORDER, - TRANSACTION_TYPE, - AddValueRequest, - CountRequest, - CountResponse, - Empty, - ExistsRequest, - ExistsResponse, - FirstTimestampRequest, - FirstTimestampResponse, - GetRequest, - GetResponse, - GetValueRequest, - GetValueResponse, - Interval, - LatestTimestampRequest, - LatestTimestampResponse, - ListItemResponse, - ListNewestRequest, - ListRequest, - ListTipsRequest, - MarkAsRequest, - MarkAsResponse, - RemoveRequest, - RemoveResponse, - RemoveValueRequest, - SaveRequest, - SaveResponse, - SortedTxsRequest, -) - -try: - from hathor.protos.transaction_storage_pb2_grpc import ( - TransactionStorageServicer, - TransactionStorageStub, - add_TransactionStorageServicer_to_server, - ) -except ImportError: - pass __all__ = [ 'BaseTransaction', @@ -81,50 +31,6 @@ 'TxOutput', 'BitcoinAuxPow', 'Metadata', - 'ExistsRequest', - 'ExistsResponse', - 'GetRequest', - 'GetResponse', - 'SaveRequest', - 'SaveResponse', - 'RemoveRequest', - 'RemoveResponse', - 'CountRequest', - 'CountResponse', - 'LatestTimestampRequest', - 'LatestTimestampResponse', - 'AddValueRequest', - 'GetValueRequest', - 'GetValueResponse', - 'RemoveValueRequest', - 'Empty', - 'FirstTimestampRequest', - 'FirstTimestampResponse', - 'MarkAsRequest', - 'MarkAsResponse', - 'ListRequest', - 'ListTipsRequest', - 'ListNewestRequest', - 'ListItemResponse', - 'Interval', - 'SortedTxsRequest', 'TokenCreationTransaction', - 'TransactionStorageStub', - 'TransactionStorageServicer', - 'ANY_TYPE', - 'TRANSACTION_TYPE', - 'BLOCK_TYPE', - 'NO_FILTER', - 'ONLY_NEWER', - 'ONLY_OLDER', - 'ANY_ORDER', - 'ASC_ORDER', - 'TOPOLOGICAL_ORDER', - 'ONLY_NEWER', - 'ONLY_OLDER', - 'FOR_CACHING', - 'LEFT_RIGHT_ORDER_CHILDREN', - 'LEFT_RIGHT_ORDER_SPENT', 'VOIDED', - 'add_TransactionStorageServicer_to_server', ] diff --git a/hathor/protos/transaction_storage.proto b/hathor/protos/transaction_storage.proto deleted file mode 100644 index b22f447681..0000000000 --- a/hathor/protos/transaction_storage.proto +++ /dev/null @@ -1,180 +0,0 @@ -syntax = "proto3"; - -package hathor; - -import "hathor/protos/transaction.proto"; - -service TransactionStorage { - rpc Exists(ExistsRequest) returns (ExistsResponse) {} - rpc Get(GetRequest) returns (GetResponse) {} - rpc Save(SaveRequest) returns (SaveResponse) {} - rpc Remove(RemoveRequest) returns (RemoveResponse) {} - rpc Count(CountRequest) returns (CountResponse) {} - rpc LatestTimestamp(LatestTimestampRequest) returns (LatestTimestampResponse) {} - rpc FirstTimestamp(FirstTimestampRequest) returns (FirstTimestampResponse) {} - rpc MarkAs(MarkAsRequest) returns (MarkAsResponse) {} - rpc List(ListRequest) returns (stream ListItemResponse) {} - rpc ListTips(ListTipsRequest) returns (stream Interval) {} - rpc ListNewest(ListNewestRequest) returns (stream ListItemResponse) {} - rpc SortedTxs(SortedTxsRequest) returns (stream Transaction) {} - rpc AddValue(AddValueRequest) returns (Empty) {} - rpc RemoveValue(RemoveValueRequest) returns (Empty) {} - rpc GetValue(GetValueRequest) returns (GetValueResponse) {} -} - -message ExistsRequest { - bytes hash = 1; -} - -message ExistsResponse { - bool exists = 1; -} - -message GetRequest { - bytes hash = 1; - bool exclude_metadata = 2; -} - -message GetResponse { - BaseTransaction transaction = 1; -} - -message SaveRequest { - BaseTransaction transaction = 1; - bool only_metadata = 2; -} - -message SaveResponse { - bool saved = 1; -} - -message RemoveRequest { - BaseTransaction transaction = 1; -} - -message RemoveResponse { - bool removed = 1; -} - -enum TxType { - ANY_TYPE = 0; - TRANSACTION_TYPE = 1; - BLOCK_TYPE = 2; -} - -message CountRequest { - TxType tx_type = 1; -} - -message CountResponse { - uint64 count = 1; -} - -message LatestTimestampRequest { -} - -message LatestTimestampResponse { - uint32 timestamp = 1; -} - -message FirstTimestampRequest { -} - -message FirstTimestampResponse { - uint32 timestamp = 1; -} - -enum MarkType { - FOR_CACHING = 0; -} - -message MarkAsRequest { - BaseTransaction transaction = 1; - MarkType mark_type = 2; - bool remove_mark = 3; - bool relax_assert = 4; -} - -message MarkAsResponse { - bool marked = 1; -} - -enum TimeFilterType { - NO_FILTER = 0; - ONLY_NEWER = 1; - ONLY_OLDER = 2; -} - -enum OrderBy { - ANY_ORDER = 0; - ASC_ORDER = 1; - TOPOLOGICAL_ORDER = 2; - LEFT_RIGHT_ORDER_CHILDREN = 3; - LEFT_RIGHT_ORDER_SPENT = 4; -} - -message ListRequest { - bool exclude_metadata = 1; - TxType tx_type = 2; - TimeFilterType time_filter = 3; - uint32 timestamp = 4; - OrderBy order_by = 5; - bool filter_before = 6; - oneof tx_oneof { - bytes hash = 7; - BaseTransaction tx = 9; - } - uint64 max_count = 8; -} - -message ListTipsRequest { - // optional timestamp, `oneof` used to differentiate unset (None) from 0 - oneof timestamp_oneof { - double timestamp = 1; - } - TxType tx_type = 2; -} - -message Interval { - double begin = 1; - double end = 2; - bytes data = 3; -} - -message ListNewestRequest { - uint64 count = 1; - TxType tx_type = 2; -} - -message ListItemResponse { - oneof list_item_oneof { - BaseTransaction transaction = 1; - bool has_more = 2; - } -} - -message SortedTxsRequest { - uint32 timestamp = 1; - uint32 count = 2; - uint32 offset = 3; -} - -message AddValueRequest { - string key = 1; - string value = 2; -} - -message RemoveValueRequest { - string key = 1; -} - -message GetValueRequest { - string key = 1; -} - -message GetValueResponse { - string value = 1; -} - -message Empty { -} diff --git a/hathor/transaction/storage/__init__.py b/hathor/transaction/storage/__init__.py index 255453592e..3be3ee06b7 100644 --- a/hathor/transaction/storage/__init__.py +++ b/hathor/transaction/storage/__init__.py @@ -18,12 +18,6 @@ from hathor.transaction.storage.memory_storage import TransactionMemoryStorage from hathor.transaction.storage.transaction_storage import TransactionStorage -try: - from hathor.transaction.storage.remote_storage import TransactionRemoteStorage, create_transaction_storage_server - from hathor.transaction.storage.subprocess_storage import TransactionSubprocessStorage -except ImportError: - pass - try: from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage except ImportError: @@ -35,8 +29,5 @@ 'TransactionCompactStorage', 'TransactionCacheStorage', 'TransactionBinaryStorage', - 'TransactionSubprocessStorage', - 'TransactionRemoteStorage', 'TransactionRocksDBStorage', - 'create_transaction_storage_server', ] diff --git a/hathor/transaction/storage/binary_storage.py b/hathor/transaction/storage/binary_storage.py index 5a63d7f219..c8a801e800 100644 --- a/hathor/transaction/storage/binary_storage.py +++ b/hathor/transaction/storage/binary_storage.py @@ -23,14 +23,14 @@ TransactionDoesNotExist, TransactionMetadataDoesNotExist, ) -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage, TransactionStorageAsyncFromSync +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.transaction.transaction_metadata import TransactionMetadata if TYPE_CHECKING: from hathor.transaction import BaseTransaction -class TransactionBinaryStorage(BaseTransactionStorage, TransactionStorageAsyncFromSync): +class TransactionBinaryStorage(BaseTransactionStorage): def __init__(self, path='./', with_index=True): self.tx_path = os.path.join(path, 'tx') os.makedirs(self.tx_path, exist_ok=True) diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index a5067e7393..852c8061a4 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -13,10 +13,9 @@ # limitations under the License. from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Generator, Iterator, Optional, Set +from typing import TYPE_CHECKING, Any, Optional, Set from twisted.internet import threads -from twisted.internet.defer import Deferred, inlineCallbacks, succeed from hathor.transaction import BaseTransaction from hathor.transaction.storage.transaction_storage import BaseTransactionStorage @@ -191,56 +190,6 @@ def get_count_tx_blocks(self) -> int: self._flush_to_storage(self.dirty_txs.copy()) return self.store.get_count_tx_blocks() - @inlineCallbacks - def save_transaction_deferred(self, tx: BaseTransaction, *, only_metadata: bool = False) -> Iterator[Deferred]: - # TODO: yield self._save_transaction_deferred - self._save_transaction(tx) - - # call super which adds to index if needed - yield super().save_transaction_deferred(tx) - - @inlineCallbacks - def remove_transaction_deferred(self, tx: BaseTransaction) -> Iterator[Deferred]: - yield super().remove_transaction_deferred(tx) - - def transaction_exists_deferred(self, hash_bytes: bytes) -> Deferred: - if hash_bytes in self.cache: - return succeed(True) - return self.store.transaction_exists_deferred(hash_bytes) - - @inlineCallbacks - def get_transaction_deferred(self, hash_bytes: bytes) -> Generator[Deferred, Any, BaseTransaction]: - if hash_bytes in self.cache: - tx = self._clone(self.cache[hash_bytes]) - self.cache.move_to_end(hash_bytes, last=True) - self.stats['hit'] += 1 - return tx - else: - tx = yield self.store.get_transaction_deferred(hash_bytes) - # TODO: yield self._update_cache_deferred(tx) - self._update_cache(tx) - self.stats['miss'] += 1 - return tx - - @inlineCallbacks - def get_all_transactions_deferred(self): - # TODO: yield self._flush_to_storage_deferred(self.dirty_txs.copy()) - self._flush_to_storage(self.dirty_txs.copy()) - all_transactions = yield self.store.get_all_transactions_deferred() - - def _mygenerator(): - for tx in all_transactions: - tx.storage = self - yield tx - return _mygenerator() - - @inlineCallbacks - def get_count_tx_blocks_deferred(self): - # TODO: yield self._flush_to_storage_deferred(self.dirty_txs.copy()) - self._flush_to_storage(self.dirty_txs.copy()) - res = yield self.store.get_count_tx_blocks_deferred() - return res - def add_value(self, key: str, value: str) -> None: self.store.add_value(key, value) diff --git a/hathor/transaction/storage/compact_storage.py b/hathor/transaction/storage/compact_storage.py index 63254f3966..e97dfc495f 100644 --- a/hathor/transaction/storage/compact_storage.py +++ b/hathor/transaction/storage/compact_storage.py @@ -21,7 +21,7 @@ from hathor.conf import HathorSettings from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage, TransactionStorageAsyncFromSync +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.transaction.transaction_metadata import TransactionMetadata if TYPE_CHECKING: @@ -30,7 +30,7 @@ settings = HathorSettings() -class TransactionCompactStorage(BaseTransactionStorage, TransactionStorageAsyncFromSync): +class TransactionCompactStorage(BaseTransactionStorage): """This storage saves tx and metadata in the same file. It also uses JSON format. Saved file is of format {'tx': {...}, 'meta': {...}} diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index 13c1477b2d..5a4f350be8 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -15,14 +15,14 @@ from typing import Any, Dict, Iterator, Optional, TypeVar from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage, TransactionStorageAsyncFromSync +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage from hathor.transaction.transaction import BaseTransaction from hathor.transaction.transaction_metadata import TransactionMetadata _Clonable = TypeVar('_Clonable', BaseTransaction, TransactionMetadata) -class TransactionMemoryStorage(BaseTransactionStorage, TransactionStorageAsyncFromSync): +class TransactionMemoryStorage(BaseTransactionStorage): def __init__(self, with_index: bool = True, *, _clone_if_needed: bool = False) -> None: """ :param _clone_if_needed: *private parameter*, defaults to True, controls whether to clone diff --git a/hathor/transaction/storage/remote_storage.py b/hathor/transaction/storage/remote_storage.py deleted file mode 100644 index 780ce44862..0000000000 --- a/hathor/transaction/storage/remote_storage.py +++ /dev/null @@ -1,891 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from math import inf -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Iterator, List, Optional, Set, Tuple, Union - -import grpc -from grpc._server import _Context -from intervaltree import Interval -from structlog import get_logger -from twisted.internet.defer import Deferred, inlineCallbacks - -from hathor import protos -from hathor.exception import HathorError -from hathor.indexes import TransactionIndexElement, TransactionsIndex -from hathor.transaction import Block -from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.transaction_storage import AllTipsCache, TransactionStorage - -if TYPE_CHECKING: - from hathor.transaction import BaseTransaction # noqa: F401 - -logger = get_logger() - - -class RemoteCommunicationError(HathorError): - pass - - -def convert_grpc_exceptions(func: Callable) -> Callable: - """Decorator to catch and conver grpc exceptions for hathor expections. - """ - from functools import wraps - - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except grpc.RpcError as e: - if e.code() == grpc.StatusCode.NOT_FOUND: - raise TransactionDoesNotExist - else: - raise RemoteCommunicationError from e - - return wrapper - - -def convert_grpc_exceptions_generator(func: Callable) -> Callable: - """Decorator to catch and conver grpc excpetions for hathor expections. (for generators) - """ - from functools import wraps - - @wraps(func) - def wrapper(*args, **kwargs): - try: - yield from func(*args, **kwargs) - except grpc.RpcError as e: - if e.code() == grpc.StatusCode.NOT_FOUND: - raise TransactionDoesNotExist - else: - raise RemoteCommunicationError from e - - return wrapper - - -def convert_hathor_exceptions(func: Callable) -> Callable: - """Decorator to annotate better details and codes on the grpc context for known exceptions. - """ - from functools import wraps - - @wraps(func) - def wrapper(self: Any, request: Any, context: _Context) -> Any: - try: - return func(self, request, context) - except TransactionDoesNotExist: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details('Transaction does not exist.') - raise - - return wrapper - - -def convert_hathor_exceptions_generator(func: Callable) -> Callable: - """Decorator to annotate better details and codes on the grpc context for known exceptions. (for generators) - """ - from functools import wraps - - @wraps(func) - def wrapper(self: Any, request: Any, context: _Context) -> Iterator: - try: - yield from func(self, request, context) - except TransactionDoesNotExist: - context.set_code(grpc.StatusCode.NOT_FOUND) - context.set_details('Transaction does not exist.') - raise - - return wrapper - - -class TransactionRemoteStorage(TransactionStorage): - """Connects to a Storage API Server at given port and exposes standard storage interface. - """ - - def __init__(self, with_index=None): - super().__init__() - self._channel = None - self.with_index = with_index - # Set initial value for _best_block_tips cache. - self._best_block_tips = [] - - def connect_to(self, port: int) -> None: - if self._channel: - self._channel.close() - self._channel = grpc.insecure_channel('127.0.0.1:{}'.format(port)) - self._stub = protos.TransactionStorageStub(self._channel) - - # Initialize genesis. - self._save_or_verify_genesis() - - # Set initial value for _best_block_tips cache. - self._best_block_tips = [x.hash for x in self.get_all_genesis() if x.is_block] - - def _check_connection(self) -> None: - """raise error if not connected""" - from .subprocess_storage import SubprocessNotAliveError - if not self._channel: - raise SubprocessNotAliveError('subprocess not started') - - # TransactionStorageSync interface implementation: - - @convert_grpc_exceptions - def remove_transaction(self, tx: 'BaseTransaction') -> None: - self._check_connection() - - tx_proto = tx.to_proto() - request = protos.RemoveRequest(transaction=tx_proto) - result = self._stub.Remove(request) # noqa: F841 - assert result.removed - self._remove_from_weakref(tx) - - @convert_grpc_exceptions - def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - self._check_connection() - - tx_proto = tx.to_proto() - request = protos.SaveRequest(transaction=tx_proto, only_metadata=only_metadata) - result = self._stub.Save(request) # noqa: F841 - assert result.saved - self._save_to_weakref(tx) - - @convert_grpc_exceptions - def transaction_exists(self, hash_bytes: bytes) -> bool: - self._check_connection() - request = protos.ExistsRequest(hash=hash_bytes) - result = self._stub.Exists(request) - return result.exists - - @convert_grpc_exceptions - def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': - tx = self.get_transaction_from_weakref(hash_bytes) - if tx is not None: - return tx - - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - request = protos.GetRequest(hash=hash_bytes) - result = self._stub.Get(request) - - tx = tx_or_block_from_proto(result.transaction, storage=self) - self._save_to_weakref(tx) - return tx - - @convert_grpc_exceptions_generator - def get_all_transactions(self) -> Iterator['BaseTransaction']: - yield from self._call_list_request_generators() - - @convert_grpc_exceptions - def get_count_tx_blocks(self) -> int: - self._check_connection() - request = protos.CountRequest(tx_type=protos.ANY_TYPE) - result = self._stub.Count(request) - return result.count - - # TransactionStorageAsync interface implementation: - - @convert_grpc_exceptions - def save_transaction_deferred(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - # self._check_connection() - raise NotImplementedError - - @convert_grpc_exceptions - def remove_transaction_deferred(self, tx: 'BaseTransaction') -> None: - # self._check_connection() - raise NotImplementedError - - @inlineCallbacks - @convert_grpc_exceptions_generator - def transaction_exists_deferred(self, hash_bytes: bytes) -> Generator[None, protos.ExistsResponse, bool]: - self._check_connection() - request = protos.ExistsRequest(hash=hash_bytes) - result = yield Deferred.fromFuture(self._stub.Exists.future(request)) - return result.exists - - @convert_grpc_exceptions - def get_transaction_deferred(self, hash_bytes: bytes) -> Deferred: - # self._check_connection() - raise NotImplementedError - - @convert_grpc_exceptions - def get_all_transactions_deferred(self) -> Deferred: - # self._check_connection() - raise NotImplementedError - - @convert_grpc_exceptions - def get_count_tx_blocks_deferred(self) -> Deferred: - # self._check_connection() - raise NotImplementedError - - # TransactionStorage interface implementation: - - @property - def latest_timestamp(self) -> int: - return self._latest_timestamp() - - @convert_grpc_exceptions - def _latest_timestamp(self) -> int: - self._check_connection() - request = protos.LatestTimestampRequest() - result = self._stub.LatestTimestamp(request) - return result.timestamp - - @property - def first_timestamp(self) -> int: - if not hasattr(self, '_cached_first_timestamp'): - timestamp = self._first_timestamp() - if timestamp: - setattr(self, '_cached_first_timestamp', timestamp) - return getattr(self, '_cached_first_timestamp', None) - - @convert_grpc_exceptions - def _first_timestamp(self) -> int: - self._check_connection() - request = protos.FirstTimestampRequest() - result = self._stub.FirstTimestamp(request) - return result.timestamp - - def get_best_block_tips(self, timestamp: Optional[float] = None, *, skip_cache: bool = False) -> List[bytes]: - return super().get_best_block_tips(timestamp, skip_cache=skip_cache) - - @convert_grpc_exceptions - def get_all_tips(self, timestamp: Optional[Union[int, float]] = None) -> Set[Interval]: - self._check_connection() - if isinstance(timestamp, float) and timestamp != inf: - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - - if self._all_tips_cache is not None and timestamp is not None and timestamp >= self._all_tips_cache.timestamp: - return self._all_tips_cache.tips - - request = protos.ListTipsRequest(tx_type=protos.ANY_TYPE, timestamp=timestamp) - result = self._stub.ListTips(request) - tips = set() - for interval_proto in result: - tips.add(Interval(interval_proto.begin, interval_proto.end, interval_proto.data)) - - if timestamp is not None and timestamp >= self.latest_timestamp: - merkle_tree, hashes = self.calculate_merkle_tree(tips) - self._all_tips_cache = AllTipsCache(self.latest_timestamp, tips, merkle_tree, hashes) - - return tips - - @convert_grpc_exceptions - def get_block_tips(self, timestamp: Optional[float] = None) -> Set[Interval]: - self._check_connection() - if isinstance(timestamp, float) and timestamp != inf: - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListTipsRequest(tx_type=protos.BLOCK_TYPE, timestamp=timestamp) - result = self._stub.ListTips(request) - tips = set() - for interval_proto in result: - tips.add(Interval(interval_proto.begin, interval_proto.end, interval_proto.data)) - return tips - - @convert_grpc_exceptions - def get_tx_tips(self, timestamp: Optional[float] = None) -> Set[Interval]: - self._check_connection() - if isinstance(timestamp, float) and timestamp != inf: - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListTipsRequest(tx_type=protos.TRANSACTION_TYPE, timestamp=timestamp) - result = self._stub.ListTips(request) - tips = set() - for interval_proto in result: - tips.add(Interval(interval_proto.begin, interval_proto.end, interval_proto.data)) - return tips - - @convert_grpc_exceptions - def get_newest_blocks(self, count: int) -> Tuple[List['Block'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - request = protos.ListNewestRequest(tx_type=protos.BLOCK_TYPE, count=count) - result = self._stub.ListNewest(request) - tx_list: List['Block'] = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - blk = tx_or_block_from_proto(tx_proto, storage=self) - assert isinstance(blk, Block) - tx_list.append(blk) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert isinstance(has_more, bool) - return tx_list, has_more - - @convert_grpc_exceptions - def get_newest_txs(self, count: int) -> Tuple[List['BaseTransaction'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - request = protos.ListNewestRequest(tx_type=protos.TRANSACTION_TYPE, count=count) - result = self._stub.ListNewest(request) - tx_list = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert has_more is not None - return tx_list, has_more - - @convert_grpc_exceptions - def get_older_blocks_after(self, timestamp: int, hash_bytes: bytes, - count: int) -> Tuple[List['BaseTransaction'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - if isinstance(timestamp, float): - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListRequest( - tx_type=protos.BLOCK_TYPE, - time_filter=protos.ONLY_OLDER, - timestamp=timestamp, - hash=hash_bytes, - max_count=count, - ) - result = self._stub.List(request) - tx_list = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert has_more is not None - return tx_list, has_more - - @convert_grpc_exceptions - def get_newer_blocks_after(self, timestamp: int, hash_bytes: bytes, - count: int) -> Tuple[List['BaseTransaction'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - if isinstance(timestamp, float): - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListRequest( - tx_type=protos.BLOCK_TYPE, - time_filter=protos.ONLY_NEWER, - timestamp=timestamp, - hash=hash_bytes, - max_count=count, - ) - result = self._stub.List(request) - tx_list = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert has_more is not None - return tx_list, has_more - - @convert_grpc_exceptions - def get_older_txs_after(self, timestamp: int, hash_bytes: bytes, - count: int) -> Tuple[List['BaseTransaction'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - if isinstance(timestamp, float): - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListRequest( - tx_type=protos.TRANSACTION_TYPE, - time_filter=protos.ONLY_OLDER, - timestamp=timestamp, - hash=hash_bytes, - max_count=count, - ) - result = self._stub.List(request) - tx_list = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert has_more is not None - return tx_list, has_more - - @convert_grpc_exceptions - def get_newer_txs_after(self, timestamp: int, hash_bytes: bytes, - count: int) -> Tuple[List['BaseTransaction'], bool]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - if isinstance(timestamp, float): - self.log.warn('timestamp given in float will be truncated, use int instead') - timestamp = int(timestamp) - request = protos.ListRequest( - tx_type=protos.TRANSACTION_TYPE, - time_filter=protos.ONLY_NEWER, - timestamp=timestamp, - hash=hash_bytes, - max_count=count, - ) - result = self._stub.List(request) - tx_list = [] - has_more = None - for list_item in result: - if list_item.HasField('transaction'): - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - elif list_item.HasField('has_more'): - has_more = list_item.has_more - # assuming there are no more items after `has_more`, break soon - break - else: - raise ValueError('unexpected list_item_oneof') - assert has_more is not None - return tx_list, has_more - - def _manually_initialize(self) -> None: - pass - - @convert_grpc_exceptions_generator - def _call_list_request_generators(self, kwargs: Optional[Dict[str, Any]] = None) -> Iterator['BaseTransaction']: - """ Execute a call for the ListRequest and yield the blocks or txs - - :param kwargs: Parameters to be sent to ListRequest - :type kwargs: Dict[str,] - """ - from hathor.transaction import tx_or_block_from_proto - - def get_tx(tx): - tx2 = self.get_transaction_from_weakref(tx.hash) - if tx2: - tx = tx2 - else: - self._save_to_weakref(tx) - return tx - - self._check_connection() - if kwargs: - request = protos.ListRequest(**kwargs) - else: - request = protos.ListRequest() - - result = self._stub.List(request) - for list_item in result: - if not list_item.HasField('transaction'): - break - tx_proto = list_item.transaction - tx = tx_or_block_from_proto(tx_proto, storage=self) - assert tx.hash is not None - lock = self._get_lock(tx.hash) - - if lock: - with lock: - tx = get_tx(tx) - else: - tx = get_tx(tx) - yield tx - - @convert_grpc_exceptions_generator - def _topological_sort(self): - yield from self._call_list_request_generators({'order_by': protos.TOPOLOGICAL_ORDER}) - - @convert_grpc_exceptions - def _add_to_cache(self, tx): - self._check_connection() - tx_proto = tx.to_proto() - request = protos.MarkAsRequest(transaction=tx_proto, mark_type=protos.FOR_CACHING, relax_assert=False) - result = self._stub.MarkAs(request) # noqa: F841 - - @convert_grpc_exceptions - def _del_from_cache(self, tx: 'BaseTransaction', *, relax_assert: bool = False) -> None: - self._check_connection() - tx_proto = tx.to_proto() - request = protos.MarkAsRequest(transaction=tx_proto, mark_type=protos.FOR_CACHING, remove_mark=True, - relax_assert=relax_assert) - result = self._stub.MarkAs(request) # noqa: F841 - - @convert_grpc_exceptions - def get_block_count(self) -> int: - self._check_connection() - request = protos.CountRequest(tx_type=protos.BLOCK_TYPE) - result = self._stub.Count(request) - return result.count - - @convert_grpc_exceptions - def get_tx_count(self) -> int: - self._check_connection() - request = protos.CountRequest(tx_type=protos.TRANSACTION_TYPE) - result = self._stub.Count(request) - return result.count - - def get_genesis(self, hash_bytes: bytes) -> Optional['BaseTransaction']: - assert self._genesis_cache is not None - return self._genesis_cache.get(hash_bytes, None) - - def get_all_genesis(self) -> Set['BaseTransaction']: - assert self._genesis_cache is not None - return set(self._genesis_cache.values()) - - @convert_grpc_exceptions - def get_transactions_before(self, hash_bytes, num_blocks=100): # pragma: no cover - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - request = protos.ListRequest( - tx_type=protos.TRANSACTION_TYPE, - hash=hash_bytes, - max_count=num_blocks, - filter_before=True, - ) - result = self._stub.List(request) - tx_list = [] - for list_item in result: - if not list_item.HasField('transaction'): - break - tx_proto = list_item.transaction - tx_list.append(tx_or_block_from_proto(tx_proto, storage=self)) - return tx_list - - @convert_grpc_exceptions - def get_blocks_before(self, hash_bytes: bytes, num_blocks: int = 100) -> List[Block]: - from hathor.transaction import tx_or_block_from_proto - self._check_connection() - request = protos.ListRequest( - tx_type=protos.BLOCK_TYPE, - hash=hash_bytes, - max_count=num_blocks, - filter_before=True, - ) - result = self._stub.List(request) - tx_list: List[Block] = [] - for list_item in result: - if not list_item.HasField('transaction'): - break - tx_proto = list_item.transaction - block = tx_or_block_from_proto(tx_proto, storage=self) - assert isinstance(block, Block) - tx_list.append(block) - return tx_list - - @convert_grpc_exceptions - def get_all_sorted_txs(self, timestamp: int, count: int, offset: int) -> TransactionsIndex: - self._check_connection() - request = protos.SortedTxsRequest( - timestamp=timestamp, - count=count, - offset=offset - ) - result = self._stub.SortedTxs(request) - tx_list = [] - for tx_proto in result: - tx_list.append(TransactionIndexElement(tx_proto.timestamp, tx_proto.hash)) - - all_sorted = TransactionsIndex() - all_sorted.update(tx_list) - return all_sorted - - @convert_grpc_exceptions - def add_value(self, key: str, value: str) -> None: - self._check_connection() - request = protos.AddValueRequest( - key=key, - value=value - ) - result = self._stub.AddValue(request) # noqa: F841 - - @convert_grpc_exceptions - def remove_value(self, key: str) -> None: - self._check_connection() - request = protos.RemoveValueRequest( - key=key, - ) - result = self._stub.RemoveValue(request) # noqa: F841 - - @convert_grpc_exceptions - def get_value(self, key: str) -> Optional[str]: - self._check_connection() - request = protos.GetValueRequest( - key=key - ) - result = self._stub.GetValue(request) - if not result.value: - return None - - return result.value - - -class TransactionStorageServicer(protos.TransactionStorageServicer): - - def __init__(self, tx_storage): - self.log = logger.new() - self.storage = tx_storage - # We must always disable weakref because it will run remotely, which means - # each call will create a new instance of the block/transaction during the - # deserialization process. - self.storage._disable_weakref() - - @convert_hathor_exceptions - def Exists(self, request: protos.ExistsRequest, context: _Context) -> protos.ExistsResponse: - hash_bytes = request.hash - exists = self.storage.transaction_exists(hash_bytes) - return protos.ExistsResponse(exists=exists) - - @convert_hathor_exceptions - def Get(self, request: protos.GetRequest, context: _Context) -> protos.GetResponse: - hash_bytes = request.hash - exclude_metadata = request.exclude_metadata - - tx = self.storage.get_transaction(hash_bytes) - - if exclude_metadata: - del tx._metadata - else: - tx.get_metadata() - - return protos.GetResponse(transaction=tx.to_proto()) - - @convert_hathor_exceptions - def Save(self, request: protos.SaveRequest, context: _Context) -> protos.SaveResponse: - from hathor.transaction import tx_or_block_from_proto - - tx_proto = request.transaction - only_metadata = request.only_metadata - - result = protos.SaveResponse(saved=False) - - tx = tx_or_block_from_proto(tx_proto, storage=self.storage) - self.storage.save_transaction(tx, only_metadata=only_metadata) - result.saved = True - - return result - - @convert_hathor_exceptions - def Remove(self, request: protos.RemoveRequest, context: _Context) -> protos.RemoveResponse: - from hathor.transaction import tx_or_block_from_proto - - tx_proto = request.transaction - - result = protos.RemoveResponse(removed=False) - - tx = tx_or_block_from_proto(tx_proto, storage=self.storage) - self.storage.remove_transaction(tx) - result.removed = True - - return result - - @convert_hathor_exceptions - def Count(self, request: protos.CountRequest, context: _Context) -> protos.CountResponse: - tx_type = request.tx_type - if tx_type == protos.ANY_TYPE: - count = self.storage.get_count_tx_blocks() - elif tx_type == protos.TRANSACTION_TYPE: - count = self.storage.get_tx_count() - elif tx_type == protos.BLOCK_TYPE: - count = self.storage.get_block_count() - else: - raise ValueError('invalid tx_type %s' % (tx_type,)) - return protos.CountResponse(count=count) - - @convert_hathor_exceptions - def LatestTimestamp(self, request: protos.LatestTimestampRequest, - context: _Context) -> protos.LatestTimestampResponse: - return protos.LatestTimestampResponse(timestamp=self.storage.latest_timestamp) - - @convert_hathor_exceptions - def FirstTimestamp(self, request: protos.FirstTimestampRequest, - context: _Context) -> protos.FirstTimestampResponse: - return protos.FirstTimestampResponse(timestamp=self.storage.first_timestamp) - - @convert_hathor_exceptions - def MarkAs(self, request, context): - from hathor.transaction import tx_or_block_from_proto - - tx = tx_or_block_from_proto(request.transaction, storage=self.storage) - - if request.mark_type == protos.FOR_CACHING: - if request.remove_mark: - self.storage._del_from_cache(tx, relax_assert=request.relax_assert) - else: - self.storage._add_to_cache(tx) - else: - raise ValueError('invalid mark_type') - - # TODO: correct value for `marked` - return protos.MarkAsResponse(marked=True) - - @convert_hathor_exceptions_generator - def List(self, request: protos.ListRequest, context: _Context) -> Iterator[protos.ListItemResponse]: - exclude_metadata = request.exclude_metadata - has_more = None - - hash_bytes = request.hash - count = request.max_count - timestamp = request.timestamp - - # TODO: more exceptions for unsupported cases - if request.filter_before: - if request.tx_type == protos.ANY_TYPE: - raise NotImplementedError - elif request.tx_type == protos.TRANSACTION_TYPE: - tx_iter = self.storage.get_transactions_before(hash_bytes, count) - elif request.tx_type == protos.BLOCK_TYPE: - tx_iter = self.storage.get_blocks_before(hash_bytes, count) - else: - raise ValueError('invalid tx_type %s' % (request.tx_type,)) - elif request.time_filter == protos.ONLY_NEWER: - if request.tx_type == protos.ANY_TYPE: - raise NotImplementedError - elif request.tx_type == protos.TRANSACTION_TYPE: - tx_iter, has_more = self.storage.get_newer_txs_after(timestamp, hash_bytes, count) - elif request.tx_type == protos.BLOCK_TYPE: - tx_iter, has_more = self.storage.get_newer_blocks_after(timestamp, hash_bytes, count) - else: - raise ValueError('invalid tx_type %s' % (request.tx_type,)) - elif request.time_filter == protos.ONLY_OLDER: - if request.tx_type == protos.ANY_TYPE: - raise NotImplementedError - elif request.tx_type == protos.TRANSACTION_TYPE: - tx_iter, has_more = self.storage.get_older_txs_after(timestamp, hash_bytes, count) - elif request.tx_type == protos.BLOCK_TYPE: - tx_iter, has_more = self.storage.get_older_blocks_after(timestamp, hash_bytes, count) - else: - raise ValueError('invalid tx_type %s' % (request.tx_type,)) - elif request.time_filter == protos.NO_FILTER: - if request.order_by == protos.ANY_ORDER: - tx_iter = self.storage.get_all_transactions() - elif request.order_by == protos.TOPOLOGICAL_ORDER: - tx_iter = self.storage._topological_sort() - else: - raise ValueError('invalid order_by') - else: - raise ValueError('invalid request') - - for tx in tx_iter: - if exclude_metadata: - del tx._metadata - else: - tx.get_metadata() - yield protos.ListItemResponse(transaction=tx.to_proto()) - if has_more is not None: - yield protos.ListItemResponse(has_more=has_more) - - @convert_hathor_exceptions_generator - def ListTips(self, request: protos.ListTipsRequest, context: _Context) -> Iterator[protos.Interval]: - # XXX: using HasField (and oneof) to differentiate None from 0, which is very important in this context - timestamp = None - if request.HasField('timestamp'): - timestamp = request.timestamp - - if request.tx_type == protos.ANY_TYPE: - tx_intervals = self.storage.get_all_tips(timestamp) - elif request.tx_type == protos.TRANSACTION_TYPE: - tx_intervals = self.storage.get_tx_tips(timestamp) - elif request.tx_type == protos.BLOCK_TYPE: - tx_intervals = self.storage.get_block_tips(timestamp) - else: - raise ValueError('invalid tx_type %s' % (request.tx_type,)) - - for interval in tx_intervals: - yield protos.Interval(begin=interval.begin, end=interval.end, data=interval.data) - - @convert_hathor_exceptions_generator - def ListNewest(self, request: protos.ListNewestRequest, context: _Context) -> Iterator[protos.ListItemResponse]: - has_more = False - if request.tx_type == protos.ANY_TYPE: - raise NotImplementedError - elif request.tx_type == protos.TRANSACTION_TYPE: - tx_list, has_more = self.storage.get_newest_txs(request.count) - elif request.tx_type == protos.BLOCK_TYPE: - tx_list, has_more = self.storage.get_newest_blocks(request.count) - else: - raise ValueError('invalid tx_type %s' % (request.tx_type,)) - - for tx in tx_list: - yield protos.ListItemResponse(transaction=tx.to_proto()) - yield protos.ListItemResponse(has_more=has_more) - - @convert_hathor_exceptions_generator - def SortedTxs(self, request: protos.SortedTxsRequest, context: _Context) -> Iterator[protos.Transaction]: - timestamp = request.timestamp - offset = request.offset - count = request.count - - txs_index = self.storage.get_all_sorted_txs(timestamp, count, offset) - for tx_element in txs_index[:]: - yield protos.Transaction(timestamp=tx_element.timestamp, hash=tx_element.hash) - - @convert_hathor_exceptions - def AddValue(self, request: protos.AddValueRequest, context: _Context) -> protos.Empty: - key = request.key - value = request.value - - self.storage.add_value(key, value) - return protos.Empty() - - @convert_hathor_exceptions - def RemoveValue(self, request: protos.RemoveValueRequest, context: _Context) -> protos.Empty: - key = request.key - self.storage.remove_value(key) - return protos.Empty() - - @convert_hathor_exceptions - def GetValue(self, request: protos.GetValueRequest, context: _Context) -> protos.GetValueResponse: - key = request.key - value = self.storage.get_value(key) - - if value: - return protos.GetValueResponse(value=value) - else: - return protos.GetValueResponse() - - -def create_transaction_storage_server(server: grpc.Server, tx_storage: TransactionStorage, - port: Optional[int] = None) -> Tuple[protos.TransactionStorageServicer, int]: - """Create a GRPC servicer for the given storage, returns a (servicer, port) tuple. - - :param server: a GRPC server - :type server: :py:class:`grpc.Server` - - :param tx_storage: an instance of TransactionStorage - :type tx_storage: :py:class:`hathor.transaction.storage.TransactionStorage` - - :param port: optional listen port, if None a random port will be chosen (and returned) - :type server: :py:class:`typing.Optional[int]` - - :rtype :py:class:`typing.Tuple[hathor.protos.TransactionStorageServicer, int]` - """ - servicer = TransactionStorageServicer(tx_storage) - protos.add_TransactionStorageServicer_to_server(servicer, server) - port = server.add_insecure_port('127.0.0.1:0') - assert port is not None - return servicer, port diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index 37a8d664fa..aacb0a6774 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -16,13 +16,13 @@ from typing import TYPE_CHECKING, Iterator, Optional from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage, TransactionStorageAsyncFromSync +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage if TYPE_CHECKING: from hathor.transaction import BaseTransaction -class TransactionRocksDBStorage(BaseTransactionStorage, TransactionStorageAsyncFromSync): +class TransactionRocksDBStorage(BaseTransactionStorage): """This storage saves tx and metadata to the same key on RocksDB It uses Protobuf serialization internally. diff --git a/hathor/transaction/storage/subprocess_storage.py b/hathor/transaction/storage/subprocess_storage.py deleted file mode 100644 index c4e9bd2448..0000000000 --- a/hathor/transaction/storage/subprocess_storage.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from concurrent import futures -from multiprocessing import Process, Queue - -import grpc - -from hathor.exception import HathorError -from hathor.transaction.storage.remote_storage import TransactionRemoteStorage, create_transaction_storage_server - - -class SubprocessNotAliveError(HathorError): - pass - - -class TransactionSubprocessStorage(TransactionRemoteStorage, Process): - """Subprocess storage to be used 'on top' of other storages. - - Wraps a given store constructor and spawns it on a subprocess. - """ - - def __init__(self, store_constructor, with_index=None): - """ - :param store_constructor: a callable that returns an instance of TransactionStorage - :type store_constructor: :py:class:`typing.Callable[..., hathor.transaction.storage.TransactionStorage]` - """ - Process.__init__(self) - TransactionRemoteStorage.__init__(self, with_index=with_index) - self._store_constructor = store_constructor - # this queue is used by the subprocess to inform which port was selected - self._port_q: 'Queue[int]' = Queue(1) - # this queue is used to inform the subprocess it can end - self._exit_q: 'Queue[int]' = Queue(1) - - def _check_connection(self): - """raise error if subprocess is not alive""" - super()._check_connection() - if not self.is_alive(): - raise SubprocessNotAliveError('subprocess is dead') - - def stop(self): - self._exit_q.put(None) - if self._channel: - self._channel.close() - - def start(self): - super().start() - port = self._port_q.get() - self.connect_to(port) - - def terminate(self): - self.close() - super().terminate() - - def run(self): - """internal method for Process interface, do not run directly""" - # TODO: some tuning with benchmarks - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - tx_storage = self._store_constructor() - tx_storage._manually_initialize() - _servicer, port = create_transaction_storage_server(server, tx_storage) - self._port_q.put(port) - server.start() - self._exit_q.get() - # the above all blocks until _exit_q.put(None) or _exit_q closes - server.stop(0) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index d20cd43e8a..03449f49d4 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -16,12 +16,11 @@ from abc import ABC, abstractmethod, abstractproperty from collections import deque from threading import Lock -from typing import Any, Dict, Generator, Iterator, List, NamedTuple, Optional, Set, Tuple, cast +from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Set, Tuple, cast from weakref import WeakValueDictionary from intervaltree.interval import Interval from structlog import get_logger -from twisted.internet.defer import Deferred, inlineCallbacks, succeed from hathor.conf import HathorSettings from hathor.indexes import IndexesManager, TokensIndex, TransactionsIndex, WalletIndex @@ -30,7 +29,6 @@ from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionIsNotABlock from hathor.transaction.transaction import BaseTransaction from hathor.transaction.transaction_metadata import TransactionMetadata -from hathor.util import skip_warning settings = HathorSettings() @@ -271,92 +269,6 @@ def get_count_tx_blocks(self) -> int: """ raise NotImplementedError - """Async interface, all methods mirrorred from TransactionStorageSync, but suffixed with `_deferred`.""" - - @abstractmethod - def save_transaction_deferred(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: - """Saves the tx. - - :param tx: Transaction to save - :type tx: :py:class:`hathor.transaction.BaseTransaction` - - :param only_metadata: Don't save the transaction, only the metadata of this transaction - :type only_metadata: bool - - :rtype :py:class:`twisted.internet.defer.Deferred[None]` - """ - if self.with_index: - self._add_to_cache(tx) - return succeed(None) - - @abstractmethod - def remove_transaction_deferred(self, tx: BaseTransaction) -> None: - """Remove the tx. - - :param tx: Transaction to be removed - - :rtype :py:class:`twisted.internet.defer.Deferred[None]` - """ - if self.with_index: - self._del_from_cache(tx) - return succeed(None) - - @abstractmethod - def transaction_exists_deferred(self, hash_bytes: bytes) -> bool: - """Returns `True` if transaction with hash `hash_bytes` exists. - - :param hash_bytes: Hash in bytes that will be checked. - :type hash_bytes: bytes - - :rtype :py:class:`twisted.internet.defer.Deferred[bool]` - """ - raise NotImplementedError - - @abstractmethod - def get_transaction_deferred(self, hash_bytes: bytes) -> BaseTransaction: - """Returns the transaction with hash `hash_bytes`. - - :param hash_bytes: Hash in bytes that will be checked. - :type hash_bytes: bytes - - :rtype :py:class:`twisted.internet.defer.Deferred[hathor.transaction.BaseTransaction]` - """ - raise NotImplementedError - - @inlineCallbacks - def get_metadata_deferred(self, hash_bytes: bytes) -> Generator[Any, Any, Optional[TransactionMetadata]]: - """Returns the transaction metadata with hash `hash_bytes`. - - :param hash_bytes: Hash in bytes that will be checked. - :type hash_bytes: bytes - - :rtype :py:class:`twisted.internet.defer.Deferred[hathor.transaction.TransactionMetadata]` - """ - try: - tx = yield self.get_transaction_deferred(hash_bytes) - return tx.get_metadata(use_storage=False) - except TransactionDoesNotExist: - return None - - @abstractmethod - def get_all_transactions_deferred(self) -> Iterator[BaseTransaction]: - # TODO: find an `async generator` type - # TODO: verify the following claim: - """Return all transactions that are not blocks. - - :rtype :py:class:`twisted.internet.defer.Deferred[typing.Iterable[hathor.transaction.BaseTransaction]]` - """ - raise NotImplementedError - - @abstractmethod - def get_count_tx_blocks_deferred(self) -> int: - # TODO: verify the following claim: - """Return the number of transactions/blocks stored. - - :rtype :py:class:`twisted.internet.defer.Deferred[int]` - """ - raise NotImplementedError - @abstractproperty def latest_timestamp(self) -> int: raise NotImplementedError @@ -644,28 +556,6 @@ def is_db_clean(self) -> bool: return self.get_value(self._clean_db_attribute) == '1' -class TransactionStorageAsyncFromSync(TransactionStorage): - """Implement async interface from sync interface, for legacy implementations.""" - - def save_transaction_deferred(self, tx: BaseTransaction, *, only_metadata: bool = False) -> Deferred: - return succeed(skip_warning(self.save_transaction)(tx, only_metadata=only_metadata)) - - def remove_transaction_deferred(self, tx: BaseTransaction) -> Deferred: - return succeed(skip_warning(self.remove_transaction)(tx)) - - def transaction_exists_deferred(self, hash_bytes: bytes) -> Deferred: - return succeed(skip_warning(self.transaction_exists)(hash_bytes)) - - def get_transaction_deferred(self, hash_bytes: bytes) -> Deferred: - return succeed(skip_warning(self.get_transaction)(hash_bytes)) - - def get_all_transactions_deferred(self) -> Deferred: - return succeed(skip_warning(self.get_all_transactions)()) - - def get_count_tx_blocks_deferred(self) -> Deferred: - return succeed(skip_warning(self.get_count_tx_blocks)()) - - class BaseTransactionStorage(TransactionStorage): def __init__(self, with_index: bool = True, pubsub: Optional[Any] = None) -> None: super().__init__() diff --git a/tests/p2p/test_double_spending.py b/tests/p2p/test_double_spending.py index 4238b8b756..fa073bd43d 100644 --- a/tests/p2p/test_double_spending.py +++ b/tests/p2p/test_double_spending.py @@ -2,7 +2,7 @@ from hathor.crypto.util import decode_address from tests import unittest -from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_tx, start_remote_storage +from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_tx class HathorSyncMethodsTestCase(unittest.TestCase): @@ -311,15 +311,3 @@ def test_double_spending_propagation(self): # dot2 = self.manager1.tx_storage.graphviz_funds(format='pdf', acc_weight=True) # dot2.render('dot2') - - -class RemoteStorageSyncTest(HathorSyncMethodsTestCase): - def setUp(self): - super().setUp() - tx_storage, self._server = start_remote_storage() - - self.manager1.tx_storage = tx_storage - - def tearDown(self): - self._server.stop(0).wait() - super().tearDown() diff --git a/tests/p2p/test_sync.py b/tests/p2p/test_sync.py index 5ecda8a5e3..1c9edd552d 100644 --- a/tests/p2p/test_sync.py +++ b/tests/p2p/test_sync.py @@ -5,9 +5,7 @@ from hathor.p2p.protocol import PeerIdState from hathor.simulator import FakeConnection from hathor.transaction.storage.exceptions import TransactionIsNotABlock -from hathor.transaction.storage.remote_storage import RemoteCommunicationError, TransactionRemoteStorage from tests import unittest -from tests.utils import start_remote_storage class HathorSyncMethodsTestCase(unittest.TestCase): @@ -72,12 +70,8 @@ def test_get_blocks_before(self): self.assertEqual(0, len(result)) genesis_tx = [tx for tx in self.genesis if not tx.is_block][0] - if isinstance(self.manager1.tx_storage, TransactionRemoteStorage): - with self.assertRaises(RemoteCommunicationError): - self.manager1.tx_storage.get_blocks_before(genesis_tx.hash) - else: - with self.assertRaises(TransactionIsNotABlock): - self.manager1.tx_storage.get_blocks_before(genesis_tx.hash) + with self.assertRaises(TransactionIsNotABlock): + self.manager1.tx_storage.get_blocks_before(genesis_tx.hash) blocks = self._add_new_blocks(20) num_blocks = 5 @@ -301,15 +295,3 @@ def test_downloader(self): # And try again downloader.check_downloading_queue() self.assertEqual(len(downloader.downloading_deque), 0) - - -class RemoteStorageSyncTest(HathorSyncMethodsTestCase): - def setUp(self): - super().setUp() - tx_storage, self._server = start_remote_storage() - - self.manager1.tx_storage = tx_storage - - def tearDown(self): - self._server.stop(0).wait() - super().tearDown() diff --git a/tests/resources/transaction/test_create_tx.py b/tests/resources/transaction/test_create_tx.py index df34b00743..f660ad0c73 100644 --- a/tests/resources/transaction/test_create_tx.py +++ b/tests/resources/transaction/test_create_tx.py @@ -7,7 +7,7 @@ from hathor.transaction.resources import CreateTxResource from hathor.transaction.scripts import P2PKH, create_base_script from tests.resources.base_resource import StubSite, _BaseResourceTest -from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_tx, start_remote_storage +from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_tx class TransactionTest(_BaseResourceTest._ResourceTest): @@ -357,14 +357,3 @@ def test_invalid_address(self): }) # TODO: tests that use the tokens field (i.e. not only HTR) - - -class RemoteStorageTransactionTest(TransactionTest): - def setUp(self): - self.tx_storage, self._server = start_remote_storage() - self.tx_storage.with_index = True - super().setUp() - - def tearDown(self): - super().tearDown() - self._server.stop(0).wait() diff --git a/tests/resources/transaction/test_tx.py b/tests/resources/transaction/test_tx.py index 6af3e1f8f9..b7138ec855 100644 --- a/tests/resources/transaction/test_tx.py +++ b/tests/resources/transaction/test_tx.py @@ -3,7 +3,7 @@ from hathor.transaction import Transaction from hathor.transaction.resources import TransactionResource from tests.resources.base_resource import StubSite, _BaseResourceTest -from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_transactions, start_remote_storage +from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_transactions class TransactionTest(_BaseResourceTest._ResourceTest): @@ -231,14 +231,3 @@ def test_invalid_params(self): }) data = response.json_value() self.assertFalse(data['success']) - - -class RemoteStorageTransactionTest(TransactionTest): - def setUp(self): - self.tx_storage, self._server = start_remote_storage() - self.tx_storage.with_index = True - super().setUp() - - def tearDown(self): - super().tearDown() - self._server.stop(0).wait() diff --git a/tests/tx/test_cache_storage.py b/tests/tx/test_cache_storage.py index 9edcb847ca..423bab0eca 100644 --- a/tests/tx/test_cache_storage.py +++ b/tests/tx/test_cache_storage.py @@ -1,13 +1,8 @@ -import collections - -from twisted.internet.defer import inlineCallbacks - from hathor.daa import TestMode, _set_test_mode -from hathor.transaction import Block, Transaction, TransactionMetadata, TxOutput -from hathor.transaction.scripts import P2PKH +from hathor.transaction import Transaction, TransactionMetadata from hathor.transaction.storage import TransactionCacheStorage, TransactionMemoryStorage from tests import unittest -from tests.utils import BURN_ADDRESS, MIN_TIMESTAMP, add_new_blocks, add_new_transactions +from tests.utils import add_new_blocks, add_new_transactions CACHE_SIZE = 5 @@ -26,7 +21,7 @@ def setUp(self): self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] # Save genesis metadata - self.cache_storage.save_transaction_deferred(self.genesis_txs[0], only_metadata=True) + self.cache_storage.save_transaction(self.genesis_txs[0], only_metadata=True) self.manager = self.create_peer('testnet', tx_storage=self.cache_storage, unlock_wallet=True) @@ -143,58 +138,6 @@ def test_flush_thread(self): del self.cache_storage.cache[next(iter(self.cache_storage.dirty_txs))] self.cache_storage._flush_to_storage(self.cache_storage.dirty_txs.copy()) - def test_deferred_methods(self): - for _ in self._test_deferred_methods(): - pass - - @inlineCallbacks - def _test_deferred_methods(self): - # Testing without cloning - self.cache_storage._clone_if_needed = False - - block_parents = [block.hash for block in self.genesis_blocks] + [tx.hash for tx in self.genesis_txs] - output = TxOutput(200, P2PKH.create_output_script(BURN_ADDRESS)) - obj = Block(timestamp=MIN_TIMESTAMP, weight=12, outputs=[output], parents=block_parents, nonce=100781, - storage=self.cache_storage) - obj.resolve() - obj.verify() - - self.cache_storage.save_transaction_deferred(obj) - - loaded_obj1 = yield self.cache_storage.get_transaction_deferred(obj.hash) - - metadata_obj1_def = yield self.cache_storage.get_metadata_deferred(obj.hash) - metadata_obj1 = obj.get_metadata() - self.assertEqual(metadata_obj1_def, metadata_obj1) - metadata_error = yield self.cache_storage.get_metadata_deferred( - bytes.fromhex('0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e')) - self.assertIsNone(metadata_error) - - self.cache_storage._flush_to_storage(self.cache_storage.dirty_txs.copy()) - self.cache_storage.cache = collections.OrderedDict() - loaded_obj2 = yield self.cache_storage.get_transaction_deferred(obj.hash) - - self.assertEqual(loaded_obj1, loaded_obj2) - - self.assertTrue((yield self.cache_storage.transaction_exists_deferred(obj.hash))) - self.assertFalse((yield self.cache_storage.transaction_exists_deferred( - '0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e'))) - - self.assertFalse( - self.cache_storage.transaction_exists('0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e')) - - self.assertEqual(obj, loaded_obj1) - self.assertEqual(obj.is_block, loaded_obj1.is_block) - - count = yield self.cache_storage.get_count_tx_blocks_deferred() - self.assertEqual(count, 4) - - all_transactions = yield self.cache_storage.get_all_transactions_deferred() - total = 0 - for tx in all_transactions: - total += 1 - self.assertEqual(total, 4) - def test_topological_sort_dfs(self): _set_test_mode(TestMode.TEST_ALL_WEIGHT) add_new_blocks(self.manager, 11, advance_clock=1) diff --git a/tests/tx/test_remote_storage.py b/tests/tx/test_remote_storage.py deleted file mode 100644 index 1639eb8557..0000000000 --- a/tests/tx/test_remote_storage.py +++ /dev/null @@ -1,72 +0,0 @@ -import datetime - -from hathor.transaction import Block, Transaction -from hathor.transaction.base_transaction import tx_or_block_from_proto -from hathor.transaction.storage.remote_storage import RemoteCommunicationError -from tests import unittest -from tests.utils import add_new_blocks, add_new_transactions, start_remote_storage - - -class RemoteStorageTest(unittest.TestCase): - def setUp(self): - super().setUp() - tx_storage, self._server = start_remote_storage() - - self.network = 'testnet' - self.manager = self.create_peer(self.network, unlock_wallet=True) - self.manager.tx_storage = tx_storage - - self.genesis = tx_storage.get_all_genesis() - self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] - self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] - - def tearDown(self): - self._server.stop(0).wait() - super().tearDown() - - def test_exceptions(self): - self._server.stop(0).wait() - - with self.assertRaises(RemoteCommunicationError): - self.manager.tx_storage.get_block_count() - - def test_get_txs(self): - first_block = add_new_blocks(self.manager, 30, advance_clock=1)[0] - first_tx = add_new_transactions(self.manager, 3, advance_clock=1)[0] - - # Using timestamp as float to test code - txs, _ = self.manager.tx_storage.get_older_txs_after(float(first_tx.timestamp), first_tx.hash, 3) - self.assertEqual(len(txs), 2) - - txs, _ = self.manager.tx_storage.get_newer_txs_after(float(first_tx.timestamp), first_tx.hash, 3) - self.assertEqual(len(txs), 2) - - blocks, _ = self.manager.tx_storage.get_older_blocks_after(float(first_block.timestamp), first_block.hash, 3) - self.assertEqual(len(blocks), 1) - - blocks, _ = self.manager.tx_storage.get_newer_blocks_after(float(first_block.timestamp), first_block.hash, 3) - self.assertEqual(len(blocks), 3) - - tx = txs[0] - proto = tx.to_proto() - tx2 = Transaction.create_from_proto(proto) - self.assertEqual(tx, tx2) - - block = blocks[0] - proto2 = block.to_proto() - block2 = Block.create_from_proto(proto2) - self.assertEqual(block, block2) - - tx3 = tx_or_block_from_proto(proto) - self.assertEqual(tx, tx3) - - proto.ClearField('transaction') - - with self.assertRaises(ValueError): - tx_or_block_from_proto(proto) - - t = datetime.datetime.now() - datetime.timedelta(seconds=1) - t_tx = tx.get_time_from_now() - t2_tx = tx.get_time_from_now(now=t) - - self.assertNotEqual(t_tx, t2_tx) diff --git a/tests/tx/test_tx_serialization.py b/tests/tx/test_tx_serialization.py index 30a30506db..5e7077f03a 100644 --- a/tests/tx/test_tx_serialization.py +++ b/tests/tx/test_tx_serialization.py @@ -1,4 +1,3 @@ -import hathor.protos.transaction_pb2_grpc # noqa this file has nothing to test, only import from hathor.crypto.util import decode_address from hathor.transaction import Transaction from hathor.wallet.base_wallet import WalletOutputInfo diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index f1ef9e5eb2..8a37c3a098 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -21,7 +21,6 @@ TransactionCompactStorage, TransactionMemoryStorage, TransactionRocksDBStorage, - TransactionSubprocessStorage, ) from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.wallet import Wallet @@ -32,7 +31,6 @@ add_new_blocks, add_new_transactions, create_tokens, - start_remote_storage, ) try: @@ -406,25 +404,6 @@ def test_key_value_attribute(self): # Key should not exist again self.assertIsNone(self.tx_storage.get_value(attr)) - class _RemoteStorageTest(_TransactionStorageTest): - def setUp(self, tx_storage, reactor=None): - tx_storage, self._server = start_remote_storage(tx_storage=tx_storage) - super().setUp(tx_storage, reactor=reactor) - - def tearDown(self): - self._server.stop(0) - super().tearDown() - - class _SubprocessStorageTest(_TransactionStorageTest): - def setUp(self, tx_storage_constructor, reactor=None): - tx_storage = TransactionSubprocessStorage(tx_storage_constructor) - tx_storage.start() - super().setUp(tx_storage, reactor=reactor) - - def tearDown(self): - self.tx_storage.stop() - super().tearDown() - class TransactionBinaryStorageTest(_BaseTransactionStorageTest._TransactionStorageTest): def setUp(self): @@ -492,31 +471,6 @@ def setUp(self): super().setUp(TransactionCacheStorage(store, reactor, capacity=5)) -# class SubprocessMemoryStorageTest(_BaseTransactionStorageTest._SubprocessStorageTest): -# def setUp(self): -# super().setUp(TransactionMemoryStorage) - -# class SubprocessCacheMemoryStorageTest(_BaseTransactionStorageTest._SubprocessStorageTest): -# def setUp(self): -# def storage_constructor(): -# store = TransactionMemoryStorage() -# reactor = Clock() -# return TransactionCacheStorage(store, reactor, capacity=5) -# super().setUp(storage_constructor) - - -class RemoteMemoryStorageTest(_BaseTransactionStorageTest._RemoteStorageTest): - def setUp(self): - super().setUp(TransactionMemoryStorage()) - - -class RemoteCacheMemoryStorageTest(_BaseTransactionStorageTest._RemoteStorageTest): - def setUp(self): - store = TransactionMemoryStorage() - reactor = Clock() - super().setUp(TransactionCacheStorage(store, reactor, capacity=5)) - - @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') class TransactionRocksDBStorageTest(_BaseTransactionStorageTest._TransactionStorageTest): def setUp(self): diff --git a/tests/utils.py b/tests/utils.py index 5fd91a8b61..fa299adcd6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,10 +3,8 @@ import subprocess import time import urllib.parse -from concurrent import futures from typing import List, Optional, cast -import grpc import requests from hathor.conf import HathorSettings @@ -14,11 +12,6 @@ from hathor.manager import HathorManager from hathor.transaction import Transaction, TxInput, TxOutput, genesis from hathor.transaction.scripts import P2PKH, HathorScript, Opcode -from hathor.transaction.storage import ( - TransactionMemoryStorage, - TransactionRemoteStorage, - create_transaction_storage_server, -) from hathor.transaction.token_creation_tx import TokenCreationTransaction from hathor.transaction.util import get_deposit_amount @@ -428,29 +421,6 @@ def create_tokens(manager: 'HathorManager', address_b58: Optional[str] = None, m return tx -def start_remote_storage(tx_storage=None): - """ Starts a remote storage - - :param tx_storage: storage to run in the remote storage - :type tx_storage: :py:class:`hathor.transaction.storage.TransactionStorage` - - :return: Remote tx storage and the remote server - :rtype: Tuple[:py:class:`hathor.transaction.storage.TransactionRemoteStorage`, grpc server] - """ - if not tx_storage: - tx_storage = TransactionMemoryStorage() - - _server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - tx_storage._manually_initialize() - _servicer, port = create_transaction_storage_server(_server, tx_storage) - _server.start() - - tx_storage = TransactionRemoteStorage() - tx_storage.connect_to(port) - - return tx_storage, _server - - def create_script_with_sigops(nops: int) -> bytes: """ Generate a script with multiple OP_CHECKMULTISIG that amounts to `nops` sigops """ From b6ad701f3e02de0f8d85b969f22999a23c11a8c5 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 11 Jun 2021 04:48:42 -0300 Subject: [PATCH 07/15] feat(p2p): only say the peer is "blocked" when explicitly configured --- hathor/conf/settings.py | 3 +++ hathor/p2p/states/peer_id.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index 4883abb6ee..d27aaaffe5 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -154,6 +154,9 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: # that the peer is synced (in seconds). P2P_SYNC_THRESHOLD: int = 60 + # Whether to warn the other peer of the reason for closing the connection + WHITELIST_WARN_BLOCKED_PEERS: bool = False + # Maximum number of opened threads that are solving POW for send tokens MAX_POW_THREADS: int = 5 diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index b10e8c279c..6400f3ac16 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -97,7 +97,10 @@ def handle_peer_id(self, payload: str) -> Generator[Any, Any, None]: # is it on the whitelist? if settings.ENABLE_PEER_WHITELIST and peer.id not in protocol.node.peers_whitelist: - protocol.send_error_and_close_connection('Blocked (id={}). Get in touch with Hathor team.'.format(peer.id)) + if settings.WHITELIST_WARN_BLOCKED_PEERS: + protocol.send_error_and_close_connection(f'Blocked (by {peer.id}). Get in touch with Hathor team.') + else: + protocol.send_error_and_close_connection('Connection rejected.') return if peer.id == protocol.my_peer.id: From 7b9835e6cf85ff1157b76d5080767434a0e8a246 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Wed, 3 Mar 2021 04:41:06 -0300 Subject: [PATCH 08/15] feat(storage): new rocksdb storage with improved performance --- hathor/cli/run_node.py | 8 +- hathor/transaction/storage/__init__.py | 2 + .../storage/old_rocksdb_storage.py | 141 ++++++++++++++++++ hathor/transaction/storage/rocksdb_storage.py | 128 ++++++++++------ tests/tx/test_tx_storage.py | 12 ++ 5 files changed, 247 insertions(+), 44 deletions(-) create mode 100644 hathor/transaction/storage/old_rocksdb_storage.py diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 41c911f16b..5cbbf4a19b 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -48,6 +48,7 @@ def create_parser(self) -> ArgumentParser: parser.add_argument('--stratum', type=int, help='Port to run stratum server') parser.add_argument('--data', help='Data directory') parser.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend') + # parser.add_argument('--rocksdb-cache', type=int, help='Size (bytes) of RocksDB block-table cache', default=0) parser.add_argument('--wallet', help='Set wallet type. Options are hd (Hierarchical Deterministic) or keypair', default=None) parser.add_argument('--wallet-enable-api', action='store_true', @@ -88,7 +89,7 @@ def prepare(self, args: Namespace) -> None: TransactionCacheStorage, TransactionCompactStorage, TransactionMemoryStorage, - TransactionRocksDBStorage, + TransactionOldRocksDBStorage, TransactionStorage, ) from hathor.wallet import HDWallet, Wallet @@ -166,7 +167,10 @@ def create_wallet(): tx_storage: TransactionStorage if args.data: if args.rocksdb_storage: - tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache)) + # cache_capacity = args.rocksdb_cache or None + # tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache), + # cache_capacity=cache_capacity) + tx_storage = TransactionOldRocksDBStorage(path=args.data, with_index=(not args.cache)) else: tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache)) self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data) diff --git a/hathor/transaction/storage/__init__.py b/hathor/transaction/storage/__init__.py index 3be3ee06b7..648484019d 100644 --- a/hathor/transaction/storage/__init__.py +++ b/hathor/transaction/storage/__init__.py @@ -19,6 +19,7 @@ from hathor.transaction.storage.transaction_storage import TransactionStorage try: + from hathor.transaction.storage.old_rocksdb_storage import TransactionOldRocksDBStorage from hathor.transaction.storage.rocksdb_storage import TransactionRocksDBStorage except ImportError: pass @@ -30,4 +31,5 @@ 'TransactionCacheStorage', 'TransactionBinaryStorage', 'TransactionRocksDBStorage', + 'TransactionOldRocksDBStorage', ] diff --git a/hathor/transaction/storage/old_rocksdb_storage.py b/hathor/transaction/storage/old_rocksdb_storage.py new file mode 100644 index 0000000000..7035672f4b --- /dev/null +++ b/hathor/transaction/storage/old_rocksdb_storage.py @@ -0,0 +1,141 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import TYPE_CHECKING, Iterator, Optional + +from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage + +if TYPE_CHECKING: + from hathor.transaction import BaseTransaction + + +class TransactionOldRocksDBStorage(BaseTransactionStorage): + """This storage saves tx and metadata to the same key on RocksDB + + It uses Protobuf serialization internally. + """ + + def __init__(self, path='./', with_index=True): + import rocksdb + tx_dir = os.path.join(path, 'tx.db') + self._db = rocksdb.DB(tx_dir, rocksdb.Options(create_if_missing=True)) + + attributes_dir = os.path.join(path, 'attributes.db') + self.attributes_db = rocksdb.DB(attributes_dir, rocksdb.Options(create_if_missing=True)) + super().__init__(with_index=with_index) + + def _load_from_bytes(self, data: bytes) -> 'BaseTransaction': + from hathor import protos + from hathor.transaction.base_transaction import tx_or_block_from_proto + + tx_proto = protos.BaseTransaction() + tx_proto.ParseFromString(data) + return tx_or_block_from_proto(tx_proto, storage=self) + + def _tx_to_bytes(self, tx: 'BaseTransaction') -> bytes: + tx_proto = tx.to_proto() + return tx_proto.SerializeToString() + + def remove_transaction(self, tx: 'BaseTransaction') -> None: + super().remove_transaction(tx) + self._db.delete(tx.hash) + self._remove_from_weakref(tx) + + def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: + super().save_transaction(tx, only_metadata=only_metadata) + self._save_transaction(tx, only_metadata=only_metadata) + self._save_to_weakref(tx) + + def _save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: + data = self._tx_to_bytes(tx) + key = tx.hash + self._db.put(key, data) + + def transaction_exists(self, hash_bytes: bytes) -> bool: + may_exist, _ = self._db.key_may_exist(hash_bytes) + if not may_exist: + return False + tx_exists = self._get_transaction_from_db(hash_bytes) is not None + return tx_exists + + def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': + tx = self.get_transaction_from_weakref(hash_bytes) + if tx is not None: + return tx + + tx = self._get_transaction_from_db(hash_bytes) + if not tx: + raise TransactionDoesNotExist(hash_bytes.hex()) + + assert tx.hash == hash_bytes + + self._save_to_weakref(tx) + return tx + + def _get_transaction_from_db(self, hash_bytes: bytes) -> Optional['BaseTransaction']: + key = hash_bytes + data = self._db.get(key) + if data is None: + return None + tx = self._load_from_bytes(data) + return tx + + def get_all_transactions(self) -> Iterator['BaseTransaction']: + tx: Optional['BaseTransaction'] + + items = self._db.iteritems() + items.seek_to_first() + + def get_tx(hash_bytes, data): + tx = self.get_transaction_from_weakref(hash_bytes) + if tx is None: + tx = self._load_from_bytes(data) + assert tx.hash == hash_bytes + self._save_to_weakref(tx) + return tx + + for key, data in items: + hash_bytes = key + + lock = self._get_lock(hash_bytes) + if lock: + with lock: + tx = get_tx(hash_bytes, data) + else: + tx = get_tx(hash_bytes, data) + + assert tx is not None + yield tx + + def get_count_tx_blocks(self) -> int: + # XXX: there may be a more efficient way, see: https://stackoverflow.com/a/25775882 + keys = self._db.iterkeys() + keys.seek_to_first() + keys_count = sum(1 for _ in keys) + return keys_count + + def add_value(self, key: str, value: str) -> None: + self.attributes_db.put(key.encode('utf-8'), value.encode('utf-8')) + + def remove_value(self, key: str) -> None: + self.attributes_db.delete(key.encode('utf-8')) + + def get_value(self, key: str) -> Optional[str]: + data = self.attributes_db.get(key.encode('utf-8')) + if data is None: + return None + else: + return data.decode() diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index aacb0a6774..84bd3ea4cd 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -17,9 +17,16 @@ from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.transaction_storage import BaseTransactionStorage +from hathor.util import json_dumpb, json_loadb if TYPE_CHECKING: - from hathor.transaction import BaseTransaction + from hathor.transaction import BaseTransaction, TransactionMetadata + + +_DB_NAME = 'data_v2.db' +_CF_NAME_TX = b'tx' +_CF_NAME_META = b'meta' +_CF_NAME_ATTR = b'attr' class TransactionRocksDBStorage(BaseTransactionStorage): @@ -28,30 +35,62 @@ class TransactionRocksDBStorage(BaseTransactionStorage): It uses Protobuf serialization internally. """ - def __init__(self, path='./', with_index=True): + def __init__(self, path: str = './', with_index: bool = True, cache_capacity: Optional[int] = None): import rocksdb - tx_dir = os.path.join(path, 'tx.db') - self._db = rocksdb.DB(tx_dir, rocksdb.Options(create_if_missing=True)) - attributes_dir = os.path.join(path, 'attributes.db') - self.attributes_db = rocksdb.DB(attributes_dir, rocksdb.Options(create_if_missing=True)) + tx_dir = os.path.join(path, _DB_NAME) + lru_cache = cache_capacity and rocksdb.LRUCache(cache_capacity) + table_factory = rocksdb.BlockBasedTableFactory(block_cache=lru_cache) + opts = dict( + table_factory=table_factory, + write_buffer_size=83886080, # 80MB (default is 4MB) + compression=rocksdb.CompressionType.no_compression, + allow_mmap_writes=True, # default is False + allow_mmap_reads=True, # default is already True + ) + + try: + self.log.debug('create new db') + new_db = rocksdb.DB(tx_dir, rocksdb.Options(error_if_exists=True, create_if_missing=True, **opts)) + new_db.create_column_family(_CF_NAME_TX, rocksdb.ColumnFamilyOptions()) + new_db.create_column_family(_CF_NAME_META, rocksdb.ColumnFamilyOptions()) + new_db.create_column_family(_CF_NAME_ATTR, rocksdb.ColumnFamilyOptions()) + del new_db # XXX: this forces deallocation allowing us to reopen the db right after + except rocksdb.errors.InvalidArgument: + self.log.debug('db already exists') + + options = rocksdb.Options(**opts) + self._db = rocksdb.DB(tx_dir, options, column_families={ + _CF_NAME_TX: rocksdb.ColumnFamilyOptions(), + _CF_NAME_META: rocksdb.ColumnFamilyOptions(), + _CF_NAME_ATTR: rocksdb.ColumnFamilyOptions(), + }) + + self._cf_tx = self._db.get_column_family(_CF_NAME_TX) + self._cf_meta = self._db.get_column_family(_CF_NAME_META) + self._cf_attr = self._db.get_column_family(_CF_NAME_ATTR) + super().__init__(with_index=with_index) - def _load_from_bytes(self, data: bytes) -> 'BaseTransaction': - from hathor import protos - from hathor.transaction.base_transaction import tx_or_block_from_proto + def _load_from_bytes(self, tx_data: bytes, meta_data: bytes) -> 'BaseTransaction': + from hathor.transaction.base_transaction import tx_or_block_from_bytes + from hathor.transaction.transaction_metadata import TransactionMetadata - tx_proto = protos.BaseTransaction() - tx_proto.ParseFromString(data) - return tx_or_block_from_proto(tx_proto, storage=self) + tx = tx_or_block_from_bytes(tx_data) + tx._metadata = TransactionMetadata.create_from_json(json_loadb(meta_data)) + tx.storage = self + return tx def _tx_to_bytes(self, tx: 'BaseTransaction') -> bytes: - tx_proto = tx.to_proto() - return tx_proto.SerializeToString() + return bytes(tx) + + def _meta_to_bytes(self, meta: 'TransactionMetadata') -> bytes: + return json_dumpb(meta.to_json()) def remove_transaction(self, tx: 'BaseTransaction') -> None: super().remove_transaction(tx) - self._db.delete(tx.hash) + self._db.delete((self._cf_tx, tx.hash)) + self._db.delete((self._cf_meta, tx.hash)) self._remove_from_weakref(tx) def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: @@ -60,15 +99,18 @@ def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False self._save_to_weakref(tx) def _save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - data = self._tx_to_bytes(tx) key = tx.hash - self._db.put(key, data) + if not only_metadata: + tx_data = self._tx_to_bytes(tx) + self._db.put((self._cf_tx, key), tx_data) + meta_data = self._meta_to_bytes(tx.get_metadata(use_storage=False)) + self._db.put((self._cf_meta, key), meta_data) def transaction_exists(self, hash_bytes: bytes) -> bool: - may_exist, _ = self._db.key_may_exist(hash_bytes) + may_exist, _ = self._db.key_may_exist((self._cf_tx, hash_bytes)) if not may_exist: return False - tx_exists = self._get_transaction_from_db(hash_bytes) is not None + tx_exists = self._db.get((self._cf_tx, hash_bytes)) is not None return tx_exists def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': @@ -87,54 +129,56 @@ def _get_transaction(self, hash_bytes: bytes) -> 'BaseTransaction': def _get_transaction_from_db(self, hash_bytes: bytes) -> Optional['BaseTransaction']: key = hash_bytes - data = self._db.get(key) - if data is None: + tx_data = self._db.get((self._cf_tx, key)) + meta_data = self._db.get((self._cf_meta, key)) + if tx_data is None: return None - tx = self._load_from_bytes(data) + assert meta_data is not None, 'expected metadata to exist when tx exists' + tx = self._load_from_bytes(tx_data, meta_data) + return tx + + def _get_tx(self, hash_bytes: bytes, tx_data: bytes) -> 'BaseTransaction': + tx = self.get_transaction_from_weakref(hash_bytes) + if tx is None: + meta_data = self._db.get((self._cf_meta, hash_bytes)) + tx = self._load_from_bytes(tx_data, meta_data) + assert tx.hash == hash_bytes + self._save_to_weakref(tx) return tx def get_all_transactions(self) -> Iterator['BaseTransaction']: tx: Optional['BaseTransaction'] - items = self._db.iteritems() + items = self._db.iteritems(self._cf_tx) items.seek_to_first() - def get_tx(hash_bytes, data): - tx = self.get_transaction_from_weakref(hash_bytes) - if tx is None: - tx = self._load_from_bytes(data) - assert tx.hash == hash_bytes - self._save_to_weakref(tx) - return tx - - for key, data in items: - hash_bytes = key + for key, tx_data in items: + _, hash_bytes = key lock = self._get_lock(hash_bytes) if lock: with lock: - tx = get_tx(hash_bytes, data) + tx = self._get_tx(hash_bytes, tx_data) else: - tx = get_tx(hash_bytes, data) + tx = self._get_tx(hash_bytes, tx_data) assert tx is not None yield tx def get_count_tx_blocks(self) -> int: - # XXX: there may be a more efficient way, see: https://stackoverflow.com/a/25775882 - keys = self._db.iterkeys() - keys.seek_to_first() - keys_count = sum(1 for _ in keys) + keys_bcount = self._db.get_property(b'rocksdb.estimate-num-keys', self._cf_tx) + keys_count = int(keys_bcount) return keys_count def add_value(self, key: str, value: str) -> None: - self.attributes_db.put(key.encode('utf-8'), value.encode('utf-8')) + self._db.put((self._cf_attr, key.encode('utf-8')), value.encode('utf-8')) def remove_value(self, key: str) -> None: - self.attributes_db.delete(key.encode('utf-8')) + self._db.delete((self._cf_attr, key.encode('utf-8'))) def get_value(self, key: str) -> Optional[str]: - data = self.attributes_db.get(key.encode('utf-8')) + data = self._db.get((self._cf_attr, key.encode('utf-8'))) + if data is None: return None else: diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index 8a37c3a098..ed403a6539 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -20,6 +20,7 @@ TransactionCacheStorage, TransactionCompactStorage, TransactionMemoryStorage, + TransactionOldRocksDBStorage, TransactionRocksDBStorage, ) from hathor.transaction.storage.exceptions import TransactionDoesNotExist @@ -471,6 +472,17 @@ def setUp(self): super().setUp(TransactionCacheStorage(store, reactor, capacity=5)) +@pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') +class TransactionOldRocksDBStorageTest(_BaseTransactionStorageTest._TransactionStorageTest): + def setUp(self): + self.directory = tempfile.mkdtemp() + super().setUp(TransactionOldRocksDBStorage(self.directory)) + + def tearDown(self): + shutil.rmtree(self.directory) + super().tearDown() + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') class TransactionRocksDBStorageTest(_BaseTransactionStorageTest._TransactionStorageTest): def setUp(self): From 913a62ef67791159e7e7de2629cd3749ed41e52e Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Thu, 4 Feb 2021 16:36:15 -0300 Subject: [PATCH 09/15] feat(sync-v2): automatic database migration for backwards compatibility --- hathor/cli/run_node.py | 54 ++++++++++++------- hathor/cli/util.py | 8 +++ hathor/manager.py | 13 +---- hathor/transaction/storage/cache_storage.py | 3 +- hathor/transaction/storage/rocksdb_storage.py | 37 ++++++++++++- .../storage/transaction_storage.py | 8 +++ hathor/transaction/transaction_metadata.py | 4 ++ tests/cli/test_shell.py | 14 +++-- tests/tx/test_cache_storage.py | 2 +- tests/utils.py | 1 + 10 files changed, 106 insertions(+), 38 deletions(-) diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 5cbbf4a19b..cb13f24bdb 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -47,8 +47,13 @@ def create_parser(self) -> ArgumentParser: parser.add_argument('--status', type=int, help='Port to run status server') parser.add_argument('--stratum', type=int, help='Port to run stratum server') parser.add_argument('--data', help='Data directory') - parser.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend') - # parser.add_argument('--rocksdb-cache', type=int, help='Size (bytes) of RocksDB block-table cache', default=0) + storage = parser.add_mutually_exclusive_group() + storage.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend (default)') + storage.add_argument('--old-rocksdb-storage', action='store_true', + help='Use old RocksDB storage backend (deprecated)') + storage.add_argument('--memory-storage', action='store_true', help='Do not use any storage') + storage.add_argument('--json-storage', action='store_true', help='Use legacy JSON storage (not recommended)') + parser.add_argument('--rocksdb-cache', type=int, help='RocksDB block-table cache size (bytes)', default=None) parser.add_argument('--wallet', help='Set wallet type. Options are hd (Hierarchical Deterministic) or keypair', default=None) parser.add_argument('--wallet-enable-api', action='store_true', @@ -78,6 +83,7 @@ def create_parser(self) -> ArgumentParser: def prepare(self, args: Namespace) -> None: import hathor + from hathor.cli.util import check_or_exit from hathor.conf import HathorSettings from hathor.daa import TestMode, _set_test_mode from hathor.manager import HathorManager @@ -90,6 +96,7 @@ def prepare(self, args: Namespace) -> None: TransactionCompactStorage, TransactionMemoryStorage, TransactionOldRocksDBStorage, + TransactionRocksDBStorage, TransactionStorage, ) from hathor.wallet import HDWallet, Wallet @@ -165,27 +172,34 @@ def create_wallet(): raise ValueError('Invalid type for wallet') tx_storage: TransactionStorage - if args.data: - if args.rocksdb_storage: - # cache_capacity = args.rocksdb_cache or None - # tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache), - # cache_capacity=cache_capacity) - tx_storage = TransactionOldRocksDBStorage(path=args.data, with_index=(not args.cache)) - else: - tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache)) - self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data) - if args.cache: - tx_storage = TransactionCacheStorage(tx_storage, reactor) - if args.cache_size: - tx_storage.capacity = args.cache_size - if args.cache_interval: - tx_storage.interval = args.cache_interval - self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval) - tx_storage.start() - else: + if args.memory_storage: + check_or_exit(not args.data, '--data should not be used with --memory-storage') # if using MemoryStorage, no need to have cache tx_storage = TransactionMemoryStorage() self.log.info('with storage', storage_class=type(tx_storage).__name__) + elif args.json_storage: + check_or_exit(args.data, '--data is expected') + tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache)) + elif args.old_rocksdb_storage: + check_or_exit(args.data, '--data is expected') + self.log.warn('the old rocksdb storage is deprecated and support will be removed') + tx_storage = TransactionOldRocksDBStorage(path=args.data) + else: + check_or_exit(args.data, '--data is expected') + if args.rocksdb_storage: + self.log.warn('--rocksdb-storage is now implied, no need to specify it') + cache_capacity = args.rocksdb_cache + tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache), + cache_capacity=cache_capacity) + self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data) + if args.cache: + check_or_exit(not args.memory_storage, '--cache should not be used with --memory-storage') + tx_storage = TransactionCacheStorage(tx_storage, reactor) + if args.cache_size: + tx_storage.capacity = args.cache_size + if args.cache_interval: + tx_storage.interval = args.cache_interval + self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval) self.tx_storage = tx_storage if args.wallet: diff --git a/hathor/cli/util.py b/hathor/cli/util.py index 590868d077..52de8a0dcf 100644 --- a/hathor/cli/util.py +++ b/hathor/cli/util.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys from argparse import ArgumentParser from collections import OrderedDict from datetime import datetime @@ -259,3 +260,10 @@ def twisted_structlog_observer(event): logger.warning('Test: warning.') logger.error('Test error.') logger.critical('Test: critical.') + + +def check_or_exit(condition: bool, message: str) -> None: + """Will exit printing `message` if `condition` is False.""" + if not condition: + print(message) + sys.exit(2) diff --git a/hathor/manager.py b/hathor/manager.py index c45dce349b..0973df2867 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -334,18 +334,7 @@ def _initialize_components(self) -> None: block_count = 0 tx_count = 0 - if self.tx_storage.get_count_tx_blocks() > 3 and not self.tx_storage.is_db_clean(): - # If has more than 3 txs on storage (the genesis txs that are always on storage by default) - # and the db is not clean (the db has old data before we cleaned the voided txs/blocks) - # then we can't move forward and ask the user to remove the old db - self.log.error( - 'Error initializing the node. You can\'t use an old database right now. ' - 'Please remove your database or start your full node again with an empty data folder.' - ) - sys.exit() - - # If has reached this line, the db is clean, so we add this attribute to it - self.tx_storage.set_db_clean() + self.tx_storage.pre_init() # self.start_profiler() diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index 852c8061a4..cb808da01e 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -73,7 +73,8 @@ def _clone(self, x: BaseTransaction) -> BaseTransaction: else: return x - def start(self) -> None: + def pre_init(self) -> None: + self.store.pre_init() self.reactor.callLater(self.interval, self._start_flush_thread) def _enable_weakref(self) -> None: diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index 84bd3ea4cd..0706af9518 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Iterator, Optional from hathor.transaction.storage.exceptions import TransactionDoesNotExist -from hathor.transaction.storage.transaction_storage import BaseTransactionStorage +from hathor.transaction.storage.transaction_storage import BaseTransactionStorage, TransactionStorage from hathor.util import json_dumpb, json_loadb if TYPE_CHECKING: @@ -38,6 +38,8 @@ class TransactionRocksDBStorage(BaseTransactionStorage): def __init__(self, path: str = './', with_index: bool = True, cache_capacity: Optional[int] = None): import rocksdb + self._path = path + tx_dir = os.path.join(path, _DB_NAME) lru_cache = cache_capacity and rocksdb.LRUCache(cache_capacity) table_factory = rocksdb.BlockBasedTableFactory(block_cache=lru_cache) @@ -87,6 +89,39 @@ def _tx_to_bytes(self, tx: 'BaseTransaction') -> bytes: def _meta_to_bytes(self, meta: 'TransactionMetadata') -> bytes: return json_dumpb(meta.to_json()) + def _import_from_other(self, other_storage: TransactionStorage) -> None: + """Internal method, do not run without understanding how to use it. + + Effectively migrates the database to be compatible with the stateful validation that sync-v2 expects. + """ + + from hathor.transaction.transaction_metadata import ValidationState + + self.log.info('importing from old database, be patient, might take a while') + # XXX: ordering is not important since we only load and save transactions without looking at their content + for tx in other_storage.get_all_transactions(): + tx_meta = tx.get_metadata() + assert tx_meta.validation.is_fully_connected() or tx_meta.validation.is_initial(), \ + 'Expected either INITIAL or FULL validation.' + tx_meta.validation = ValidationState.FULL + # XXX: using _save_transaction because we don't want to trigger any cache/index mechanism, this will be + # handled by the manager after pre_init + self._save_transaction(tx) + + def pre_init(self) -> None: + from rocksdb.errors import RocksIOError + if self.is_empty(): + # try to load the old rocksdb storage + from hathor.transaction.storage.old_rocksdb_storage import TransactionOldRocksDBStorage + try: + old_storage = TransactionOldRocksDBStorage(self._path) + except RocksIOError: + # old storage does not exist, no need to continue + return + self._import_from_other(old_storage) + else: + self.log.debug('new db not empty, skipping migration') + def remove_transaction(self, tx: 'BaseTransaction') -> None: super().remove_transaction(tx) self._db.delete((self._cf_tx, tx.hash)) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 03449f49d4..e49302331c 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -94,6 +94,14 @@ def __init__(self): # Key storage attribute to save if the node has clean db self._clean_db_attribute: str = 'clean_db' + def is_empty(self) -> bool: + """True when only genesis is present, useful for checking for a fresh database.""" + return self.get_count_tx_blocks() <= 3 + + def pre_init(self) -> None: + """Storages can implement this to run code before transaction loading starts""" + pass + def _save_or_verify_genesis(self) -> None: """Save all genesis in the storage.""" for tx in self._get_genesis_from_settings(): diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 3b56e637bf..9c47f7fb4a 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -59,6 +59,10 @@ class ValidationState(IntEnum): CHECKPOINT_FULL = 4 # besides being checkpoint valid, it is fully connected INVALID = -1 # not valid, this does not mean not best chain, orphan chains can be valid + def is_initial(self) -> bool: + """Short-hand property""" + return self is ValidationState.INITIAL + def is_at_least_basic(self) -> bool: """Until a validation is final, it is possible to change its state when more information is available.""" return self >= ValidationState.BASIC diff --git a/tests/cli/test_shell.py b/tests/cli/test_shell.py index 6994bb71f6..bda156d982 100644 --- a/tests/cli/test_shell.py +++ b/tests/cli/test_shell.py @@ -1,9 +1,17 @@ +import tempfile + from hathor.cli.shell import Shell from tests import unittest class ShellTest(unittest.TestCase): - def test_shell_execution(self): - # In this case we just want to go through the code to see if it's okay - shell = Shell(argv=[]) + # In this case we just want to go through the code to see if it's okay + + def test_shell_execution_memory_storage(self): + shell = Shell(argv=['--memory-storage']) + self.assertTrue(shell is not None) + + def test_shell_execution_default_storage(self): + temp_data = tempfile.TemporaryDirectory() + shell = Shell(argv=['--data', temp_data.name]) self.assertTrue(shell is not None) diff --git a/tests/tx/test_cache_storage.py b/tests/tx/test_cache_storage.py index 423bab0eca..78be670795 100644 --- a/tests/tx/test_cache_storage.py +++ b/tests/tx/test_cache_storage.py @@ -14,7 +14,7 @@ def setUp(self): store = TransactionMemoryStorage() self.cache_storage = TransactionCacheStorage(store, self.clock, capacity=5) self.cache_storage._manually_initialize() - self.cache_storage.start() + self.cache_storage.pre_init() self.genesis = self.cache_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] diff --git a/tests/utils.py b/tests/utils.py index fa299adcd6..64a133d28d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -215,6 +215,7 @@ def run_server(hostname='localhost', listen=8005, status=8085, bootstrap=None, t """ command = ' '.join([ 'python -m hathor run_node', + '--memory-storage', '--wallet hd', '--wallet-enable-api', '--hostname {}'.format(hostname), From 2b179d6433718247f1bebab12d2339c8c51ba3d8 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Fri, 18 Jun 2021 03:23:11 -0300 Subject: [PATCH 10/15] fix(p2p): trying to parse None on timeouts/ctrl-c's updating whitelist --- hathor/p2p/manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index 60f5e161b2..37233405e8 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -274,8 +274,12 @@ def update_whitelist(self) -> Deferred: def _update_whitelist_err(self, *args: Any, **kwargs: Any) -> None: self.log.error('update whitelist failed', args=args, kwargs=kwargs) - def _update_whitelist_cb(self, body: bytes) -> None: - self.log.info('update whitelist got response') + def _update_whitelist_cb(self, body: Optional[bytes]) -> None: + if body is None: + self.log.warn('update whitelist got no response') + return + else: + self.log.info('update whitelist got response') try: text = body.decode() new_whitelist = parse_whitelist(text) From d6db83295f2c317e6ec26530789c5e335d75f9f5 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Wed, 26 May 2021 02:04:02 -0700 Subject: [PATCH 11/15] feat(sentry): Add support to publish exception events in Sentry --- hathor/cli/run_node.py | 27 +++++++++++++-- hathor/cli/shell.py | 2 +- hathor/cli/util.py | 35 +++++++++++++------ poetry.lock | 78 ++++++++++++++++++++++++++++++++++-------- pyproject.toml | 3 ++ 5 files changed, 116 insertions(+), 29 deletions(-) diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index cb13f24bdb..806b4c9b60 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -79,6 +79,7 @@ def create_parser(self) -> ArgumentParser: '/push-tx API') parser.add_argument('--max-output-script-size', type=int, default=None, help='Custom max accepted script size ' 'on /push-tx API') + parser.add_argument('--sentry-dsn', help='Sentry DSN') return parser def prepare(self, args: Namespace) -> None: @@ -254,10 +255,32 @@ def create_wallet(): for description in args.listen: self.manager.add_listen_address(description) - self.start_manager() + self.start_manager(args) self.register_resources(args) - def start_manager(self) -> None: + def start_sentry_if_possible(self, args: Namespace) -> None: + """Start Sentry integration if possible.""" + if not args.sentry_dsn: + return + self.log.info('Starting Sentry', dsn=args.sentry_dsn) + try: + import sentry_sdk + from structlog_sentry import SentryProcessor # noqa: F401 + except ModuleNotFoundError: + self.log.error('Please use `poetry install -E sentry` for enabling Sentry.') + sys.exit(-3) + + import hathor + from hathor.conf import HathorSettings + settings = HathorSettings() + sentry_sdk.init( + dsn=args.sentry_dsn, + release=hathor.__version__, + environment=settings.NETWORK_NAME, + ) + + def start_manager(self, args: Namespace) -> None: + self.start_sentry_if_possible(args) self.manager.start() def register_resources(self, args: Namespace) -> None: diff --git a/hathor/cli/shell.py b/hathor/cli/shell.py index eccf5f3e3e..2360b75d12 100644 --- a/hathor/cli/shell.py +++ b/hathor/cli/shell.py @@ -28,7 +28,7 @@ def run_ipython(): class Shell(RunNode): - def start_manager(self) -> None: + def start_manager(self, args: Namespace) -> None: pass def register_signal_handlers(self, args: Namespace) -> None: diff --git a/hathor/cli/util.py b/hathor/cli/util.py index 52de8a0dcf..12c1c7e704 100644 --- a/hathor/cli/util.py +++ b/hathor/cli/util.py @@ -16,6 +16,7 @@ from argparse import ArgumentParser from collections import OrderedDict from datetime import datetime +from typing import Any, List import configargparse @@ -212,18 +213,30 @@ def kwargs_formatter(_, __, event_dict): event_dict['event'] = event_dict['event'].format(**event_dict) return event_dict + processors: List[Any] = [ + structlog.stdlib.filter_by_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + ] + + try: + from structlog_sentry import SentryProcessor + except ModuleNotFoundError: + pass + else: + processors.append(SentryProcessor(level=logging.ERROR)) + + processors.extend([ + structlog.stdlib.PositionalArgumentsFormatter(), + kwargs_formatter, + timestamper, + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ]) + structlog.configure( - processors=[ - structlog.stdlib.filter_by_level, - structlog.stdlib.add_logger_name, - structlog.stdlib.add_log_level, - structlog.stdlib.PositionalArgumentsFormatter(), - kwargs_formatter, - timestamper, - structlog.processors.StackInfoRenderer(), - structlog.processors.format_exc_info, - structlog.stdlib.ProcessorFormatter.wrap_for_formatter, - ], + processors=processors, context_class=OrderedDict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, diff --git a/poetry.lock b/poetry.lock index a044005bf6..3c5f25ccc7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -193,11 +193,11 @@ cffi = ">=1.12" six = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] [[package]] name = "cython" @@ -320,7 +320,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "incremental" @@ -419,7 +419,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" parso = ">=0.7.0,<0.8.0" [package.extras] -qa = ["flake8 (==3.7.9)"] +qa = ["flake8 (3.7.9)"] testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] @@ -719,7 +719,7 @@ coverage = ">=5.2.1" pytest = ">=4.6" [package.extras] -testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] name = "python-rocksdb" @@ -757,7 +757,35 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + +[[package]] +name = "sentry-sdk" +version = "1.1.0" +description = "Python client for Sentry (https://sentry.io)" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +flask = ["flask (>=0.11)", "blinker (>=1.1)"] +pure_eval = ["pure-eval", "executing", "asttokens"] +pyspark = ["pyspark (>=2.4.4)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +tornado = ["tornado (>=5)"] [[package]] name = "service-identity" @@ -822,6 +850,17 @@ dev = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest-asyncio", "pytest-r docs = ["furo", "sphinx", "sphinx-toolbox", "twisted"] tests = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest-asyncio", "pytest-randomly", "pytest (>=6.0)", "simplejson"] +[[package]] +name = "structlog-sentry" +version = "1.4.0" +description = "Sentry integration for structlog" +category = "main" +optional = true +python-versions = ">=3.6,<4.0" + +[package.dependencies] +sentry-sdk = "*" + [[package]] name = "toml" version = "0.10.2" @@ -864,16 +903,16 @@ PyHamcrest = ">=1.9.0,<1.10.0 || >1.10.0" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,!=2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] +all_non_platform = ["pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,<2.3 || >2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] conch = ["pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] dev = ["pyflakes (>=1.0.0)", "twisted-dev-tools (>=0.0.2)", "python-subunit", "sphinx (>=1.3.1)", "towncrier (>=17.4.0)"] http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,!=2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,!=2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,<2.3 || >2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,<2.3 || >2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] soap = ["soappy"] -tls = ["pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,!=2.3)"] -windows_platform = ["pywin32 (!=226)", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,!=2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] +tls = ["pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,<2.3 || >2.3)"] +windows_platform = ["pywin32 (!=226)", "pyopenssl (>=16.0.0)", "service_identity (>=18.1.0)", "idna (>=0.6,<2.3 || >2.3)", "pyasn1", "cryptography (>=2.5)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "soappy", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)"] [[package]] name = "txaio" @@ -885,7 +924,7 @@ python-versions = ">=3.6" [package.extras] all = ["zope.interface (>=3.6)", "twisted (>=20.3.0)"] -dev = ["wheel", "pytest (>=2.6.4)", "pytest-cov (>=1.8.1)", "pep8 (>=1.6.2)", "sphinx (>=1.2.3)", "pyenchant (>=1.6.6)", "sphinxcontrib-spelling (>=2.1.2)", "sphinx-rtd-theme (>=0.1.9)", "tox (>=2.1.1)", "mock (==1.3.0)", "twine (>=1.6.5)", "tox-gh-actions (>=2.2.0)"] +dev = ["wheel", "pytest (>=2.6.4)", "pytest-cov (>=1.8.1)", "pep8 (>=1.6.2)", "sphinx (>=1.2.3)", "pyenchant (>=1.6.6)", "sphinxcontrib-spelling (>=2.1.2)", "sphinx-rtd-theme (>=0.1.9)", "tox (>=2.1.1)", "mock (1.3.0)", "twine (>=1.6.5)", "tox-gh-actions (>=2.2.0)"] twisted = ["zope.interface (>=3.6)", "twisted (>=20.3.0)"] [[package]] @@ -915,7 +954,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] name = "wcwidth" @@ -956,7 +995,7 @@ python-versions = ">=3.6" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [[package]] name = "zope.interface" @@ -974,11 +1013,12 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [extras] grpc = ["grpcio"] rocksdb = ["cython", "python-rocksdb"] +sentry = ["sentry-sdk", "structlog-sentry"] [metadata] lock-version = "1.1" python-versions = ">=3.6,<4" -content-hash = "0fc7d252856db69e01e6d35b3cb0b58346ac439f9736d1153d4f85209b11cfee" +content-hash = "e710cc5cc3f55d2e1ad540e6b52c9e285ac197ee0bb7744e87361a78dc2acf32" [metadata.files] aiohttp = [ @@ -1619,6 +1659,10 @@ requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] +sentry-sdk = [ + {file = "sentry-sdk-1.1.0.tar.gz", hash = "sha256:c1227d38dca315ba35182373f129c3e2722e8ed999e52584e6aca7d287870739"}, + {file = "sentry_sdk-1.1.0-py2.py3-none-any.whl", hash = "sha256:c7d380a21281e15be3d9f67a3c4fbb4f800c481d88ff8d8931f39486dd7b4ada"}, +] service-identity = [ {file = "service_identity-18.1.0-py2.py3-none-any.whl", hash = "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36"}, {file = "service_identity-18.1.0.tar.gz", hash = "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"}, @@ -1658,6 +1702,10 @@ structlog = [ {file = "structlog-20.2.0-py2.py3-none-any.whl", hash = "sha256:33dd6bd5f49355e52c1c61bb6a4f20d0b48ce0328cc4a45fe872d38b97a05ccd"}, {file = "structlog-20.2.0.tar.gz", hash = "sha256:af79dfa547d104af8d60f86eac12fb54825f54a46bc998e4504ef66177103174"}, ] +structlog-sentry = [ + {file = "structlog-sentry-1.4.0.tar.gz", hash = "sha256:5fc6cfab71b858d71433e68cc5af79a396e72015003931507e340b3687ebb0a8"}, + {file = "structlog_sentry-1.4.0-py3-none-any.whl", hash = "sha256:04627538e13bb0719a8806353279d40c1d1afb3eb2053817820754b9a08814a7"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index a4825d4a33..482e854b22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,10 +77,13 @@ idna = "~2.10" cython = {version = "<0.30", optional = true} typing-extensions = "^3.7.4" # indirect dependency, should not be needed on Python >3.7 but some packages import it anyway setproctitle = "^1.2.2" +sentry-sdk = {version = "^1.1.0", optional = true} +structlog-sentry = {version = "^1.4.0", optional = true} [tool.poetry.extras] rocksdb = ["cython", "python-rocksdb"] grpc = ["grpcio", "grpcio-tools"] +sentry = ["sentry-sdk", "structlog-sentry"] [tool.isort] combine_as_imports = true From b146d3bf7e2c431de6e560f88fdf102875d050ad Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Mon, 30 Nov 2020 20:11:58 -0300 Subject: [PATCH 12/15] feat(sync-v2): add needed storage indexes --- hathor/cli/run_node.py | 2 - hathor/consensus.py | 18 +- hathor/daa.py | 4 + hathor/manager.py | 309 +++++++++---- hathor/p2p/downloader.py | 4 +- hathor/p2p/manager.py | 6 +- hathor/p2p/node_sync.py | 6 +- hathor/transaction/base_transaction.py | 20 +- hathor/transaction/resources/__init__.py | 2 - .../transaction/resources/tips_histogram.py | 149 ------- hathor/transaction/storage/binary_storage.py | 4 +- .../transaction/storage/block_height_index.py | 62 +++ hathor/transaction/storage/cache_storage.py | 5 +- hathor/transaction/storage/compact_storage.py | 5 +- hathor/transaction/storage/memory_storage.py | 5 +- .../storage/old_rocksdb_storage.py | 5 +- hathor/transaction/storage/rocksdb_storage.py | 5 +- .../storage/transaction_storage.py | 415 ++++++++++++++++-- hathor/wallet/base_wallet.py | 1 + .../transaction/test_tips_histogram.py | 72 --- tests/tx/test_indexes.py | 134 ++++++ tests/tx/test_tx_storage.py | 13 +- 22 files changed, 880 insertions(+), 366 deletions(-) delete mode 100644 hathor/transaction/resources/tips_histogram.py create mode 100644 hathor/transaction/storage/block_height_index.py delete mode 100644 tests/resources/transaction/test_tips_histogram.py create mode 100644 tests/tx/test_indexes.py diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 806b4c9b60..0b5e2df614 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -299,7 +299,6 @@ def register_resources(self, args: Namespace) -> None: GraphvizNeighboursResource, PushTxResource, SubmitBlockResource, - TipsHistogramResource, TransactionAccWeightResource, TransactionResource, TxParentsResource, @@ -374,7 +373,6 @@ def register_resources(self, args: Namespace) -> None: PushTxResource(self.manager, args.max_output_script_size, args.allow_non_standard_script), root), (b'graphviz', graphviz, root), - (b'tips-histogram', TipsHistogramResource(self.manager), root), (b'transaction', TransactionResource(self.manager), root), (b'transaction_acc_weight', TransactionAccWeightResource(self.manager), root), (b'dashboard_tx', DashboardTransactionResource(self.manager), root), diff --git a/hathor/consensus.py b/hathor/consensus.py index 289f7e659a..80e6550376 100644 --- a/hathor/consensus.py +++ b/hathor/consensus.py @@ -184,6 +184,7 @@ def update_voided_info(self, block: Block) -> None: meta = block.get_metadata() if not meta.voided_by: storage._best_block_tips = [block.hash] + storage.add_new_to_block_height_index(meta.height, block.hash) # The following assert must be true, but it is commented out for performance reasons. # assert len(storage.get_best_block_tips(skip_cache=True)) == 1 else: @@ -242,8 +243,17 @@ def update_voided_info(self, block: Block) -> None: meta = block.get_metadata() if not meta.voided_by: storage._best_block_tips = [block.hash] + self.log.debug('index new winner block', height=meta.height, block=block.hash_hex) + # We update the height cache index with the new winner chain + storage.update_block_height_cache_new_chain(meta.height, block) else: storage._best_block_tips = [blk.hash for blk in heads] + # XXX Is it safe to select one of the heads? + best_block = heads[0] + assert best_block.hash is not None + best_meta = best_block.get_metadata() + self.log.debug('index previous best block', height=best_meta.height, block=best_block.hash_hex) + storage.add_new_to_block_height_index(best_meta.height, best_block.hash) # Uncomment the following lines to check that the cache update is working properly. # You shouldn't run this test in production because it dampens performance. @@ -286,7 +296,7 @@ def update_voided_by_from_parents(self, block: Block) -> bool: else: meta.voided_by = voided_by.copy() block.storage.save_transaction(block, only_metadata=True) - block.storage._del_from_cache(block, relax_assert=True) # XXX: accessing private method + block.storage.del_from_indexes(block, relax_assert=True) return True return False @@ -735,7 +745,7 @@ def update_voided_info(self, tx: Transaction) -> None: if voided_by: meta.voided_by = voided_by.copy() tx.storage.save_transaction(tx, only_metadata=True) - tx.storage._del_from_cache(tx) # XXX: accessing private method + tx.storage.del_from_indexes(tx) # Check conflicts of the transactions voiding us. for h in voided_by: @@ -853,7 +863,7 @@ def remove_voided_by(self, tx: Transaction, voided_hash: bytes) -> bool: tx2.storage.save_transaction(tx2, only_metadata=True) if not meta.voided_by: meta.voided_by = None - tx.storage._add_to_cache(tx2) # XXX: accessing private method + tx.storage.add_to_indexes(tx2) from hathor.transaction import Transaction for tx2 in check_list: @@ -912,7 +922,7 @@ def add_voided_by(self, tx: Transaction, voided_hash: bytes) -> bool: # All voided transactions with conflicts must have their accumulated weight calculated. tx2.update_accumulated_weight(save_file=False) tx2.storage.save_transaction(tx2, only_metadata=True) - tx2.storage._del_from_cache(tx2, relax_assert=True) # XXX: accessing private method + tx2.storage.del_from_indexes(tx2, relax_assert=True) for tx2 in check_list: self.check_conflicts(tx2) diff --git a/hathor/daa.py b/hathor/daa.py index 6e56725276..7fd79afc3c 100644 --- a/hathor/daa.py +++ b/hathor/daa.py @@ -23,6 +23,8 @@ from math import log from typing import TYPE_CHECKING, List +from structlog import get_logger + from hathor.conf import HathorSettings from hathor.profiler import get_cpu_profiler from hathor.util import iwindows @@ -30,6 +32,7 @@ if TYPE_CHECKING: from hathor.transaction import Block, Transaction +logger = get_logger() settings = HathorSettings() cpu = get_cpu_profiler() @@ -49,6 +52,7 @@ class TestMode(IntFlag): def _set_test_mode(mode: TestMode) -> None: global TEST_MODE + logger.debug('change DAA test mode', from_mode=TEST_MODE.name, to_mode=mode.name) TEST_MODE = mode diff --git a/hathor/manager.py b/hathor/manager.py index 0973df2867..3a2fd0a19d 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -17,7 +17,7 @@ import sys import time from enum import Enum -from typing import Any, Iterator, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Iterable, Iterator, List, NamedTuple, Optional, Tuple, Union from structlog import get_logger from twisted.internet import defer @@ -30,7 +30,7 @@ from hathor.checkpoint import Checkpoint from hathor.conf import HathorSettings from hathor.consensus import ConsensusAlgorithm -from hathor.exception import InvalidNewTransaction +from hathor.exception import HathorError, InvalidNewTransaction from hathor.indexes import TokensIndex, WalletIndex from hathor.mining import BlockTemplate, BlockTemplates from hathor.p2p.peer_discovery import PeerDiscovery @@ -41,6 +41,7 @@ from hathor.transaction import BaseTransaction, Block, MergeMinedBlock, Transaction, TxVersion, sum_weights from hathor.transaction.exceptions import TxValidationError from hathor.transaction.storage import TransactionStorage +from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.wallet import BaseWallet settings = HathorSettings() @@ -213,7 +214,7 @@ def start(self) -> None: 'that was stopped in the middle. The storage is not reliable anymore and, because of that, ' 'you must initialize with a full verification again or remove your storage and do a full sync.' ) - sys.exit() + sys.exit(-1) # If self.tx_storage.is_running_manager() is True, the last time the node was running it had a sudden crash # because of that, we must run a full verification because some storage data might be wrong. @@ -224,7 +225,7 @@ def start(self) -> None: 'The storage is not reliable anymore and, because of that, so you must run a full verification ' 'or remove your storage and do a full sync.' ) - sys.exit() + sys.exit(-1) self.state = self.NodeState.INITIALIZING self.pubsub.publish(HathorEvents.MANAGER_ON_START) @@ -336,10 +337,22 @@ def _initialize_components(self) -> None: self.tx_storage.pre_init() + # Checkpoints as {height: hash} + checkpoint_heights = {} + for cp in self.checkpoints: + checkpoint_heights[cp.height] = cp.hash + # self.start_profiler() + if self._full_verification: + self.log.debug('reset all metadata') + for tx in self.tx_storage.get_all_transactions(): + tx.reset_metadata() self.log.debug('load blocks and transactions') for tx in self.tx_storage._topological_sort(): + if self._full_verification: + tx.update_initial_metadata() + assert tx.hash is not None tx_meta = tx.get_metadata() @@ -367,12 +380,34 @@ def _initialize_components(self) -> None: skip_block_weight_verification = False try: - assert self.on_new_tx( - tx, - quiet=True, - fails_silently=False, - skip_block_weight_verification=skip_block_weight_verification - ) + if self._full_verification: + # TODO: deal with invalid tx + if self.tx_storage.is_tx_needed(tx.hash): + assert isinstance(tx, Transaction) + tx._height_cache = self.tx_storage.needed_index_height(tx.hash) + if tx.can_validate_full(): + self.tx_storage.add_to_indexes(tx) + assert tx.validate_full(skip_block_weight_verification=skip_block_weight_verification) + self.consensus_algorithm.update(tx) + self.tx_storage.update_tx_tips(tx) + self.step_validations([tx]) + if tx.is_block: + self.tx_storage.add_to_parent_blocks_index(tx.hash) + else: + assert tx.validate_basic(skip_block_weight_verification=skip_block_weight_verification) + self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) + self.tx_storage.add_needed_deps(tx) + self.tx_storage.save_transaction(tx, only_metadata=True) + else: + # TODO: deal with invalid tx + if not tx_meta.validation.is_final(): + assert tx_meta.validation.is_at_least_basic(), f'invalid: {tx.hash_hex}' + self.tx_storage.add_needed_deps(tx) + elif tx.is_transaction and tx_meta.first_block is None and not tx_meta.voided_by: + self.tx_storage.update_tx_tips(tx) + elif tx.is_block: + self.tx_storage.add_to_parent_blocks_index(tx.hash) + self.tx_storage.add_to_indexes(tx) except (InvalidNewTransaction, TxValidationError): self.log.error('unexpected error when initializing', tx=tx, exc_info=True) raise @@ -380,12 +415,63 @@ def _initialize_components(self) -> None: if tx.is_block: block_count += 1 + # this works because blocks on the best chain are iterated from lower to higher height + assert tx.hash is not None + assert tx_meta.validation.is_at_least_basic() + if not tx_meta.voided_by: + # XXX: this might not be needed when making a full init because the consensus should already have + self.tx_storage.add_new_to_block_height_index(tx_meta.height, tx.hash) + + # Check if it's a checkpoint block + if tx_meta.height in checkpoint_heights: + if tx.hash == checkpoint_heights[tx_meta.height]: + del checkpoint_heights[tx_meta.height] + else: + # If the hash is different from checkpoint hash, we stop the node + self.log.error('Error initializing the node. Checkpoint validation error.') + sys.exit() + else: + tx_count += 1 + if time.time() - t2 > 1: dt = hathor.util.LogDuration(time.time() - t2) self.log.warn('tx took too long to load', tx=tx.hash_hex, dt=dt) + # we have to have a best_block by now + # assert best_block is not None + self.log.debug('done loading transactions') + # Check if all checkpoints in database are ok + my_best_height = self.tx_storage.get_height_best_block() + if checkpoint_heights: + # If I have checkpoints that were not validated I must check if they are all in a height I still don't have + first = min(list(checkpoint_heights.keys())) + if first <= my_best_height: + # If the height of the first checkpoint not validated is lower than the height of the best block + # Then it's missing this block + self.log.error('Error initializing the node. Checkpoint validation error.') + sys.exit() + + # restart all validations possible + deps_size = self.tx_storage.count_deps_index() + if deps_size > 0: + self.log.debug('run pending validations', deps_size=deps_size) + depended_final_txs: List[BaseTransaction] = [] + for tx_hash in self.tx_storage.iter_deps_index(): + if not self.tx_storage.transaction_exists(tx_hash): + continue + tx = self.tx_storage.get_transaction(tx_hash) + if tx.get_metadata().validation.is_final(): + depended_final_txs.append(tx) + self.step_validations(depended_final_txs) + new_deps_size = self.tx_storage.count_deps_index() + self.log.debug('pending validations finished', changes=deps_size - new_deps_size) + + best_height = self.tx_storage.get_height_best_block() + if best_height != h: + self.log.warn('best height doesn\'t match', best_height=best_height, max_height=h) + # self.stop_profiler(save_to='profiles/initializing.prof') self.state = self.NodeState.READY tdt = hathor.util.LogDuration(t2 - t0) @@ -587,33 +673,6 @@ def get_tokens_issued_per_block(self, height: int) -> int: """Return the number of tokens issued (aka reward) per block of a given height.""" return daa.get_tokens_issued_per_block(height) - def validate_new_tx(self, tx: BaseTransaction, skip_block_weight_verification: bool = False) -> bool: - """ Process incoming transaction during initialization. - These transactions came only from storage. - """ - assert tx.hash is not None - - if self.state == self.NodeState.INITIALIZING: - if tx.is_genesis: - return True - - else: - if tx.is_genesis: - raise InvalidNewTransaction('Genesis? {}'.format(tx.hash_hex)) - - now = self.reactor.seconds() - if tx.timestamp - now > settings.MAX_FUTURE_TIMESTAMP_ALLOWED: - raise InvalidNewTransaction('Ignoring transaction in the future {} (timestamp={}, now={})'.format( - tx.hash_hex, tx.timestamp, now)) - - if self.state != self.NodeState.INITIALIZING and not tx.can_validate_full(): - raise InvalidNewTransaction('Cannot validate, missing dependency') - - # validate transaction, raises a TxValidationError if tx is not valid - tx.validate_full() - - return True - def submit_block(self, blk: Block, fails_silently: bool = True) -> bool: """Used by submit block from all mining APIs. """ @@ -633,59 +692,106 @@ def propagate_tx(self, tx: BaseTransaction, fails_silently: bool = True) -> bool assert tx.storage == self.tx_storage, 'Invalid tx storage' else: tx.storage = self.tx_storage - return self.on_new_tx(tx, fails_silently=fails_silently) + + return self.on_new_tx(tx, fails_silently=fails_silently, propagate_to_peers=True) @cpu.profiler('on_new_tx') def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = None, quiet: bool = False, fails_silently: bool = True, propagate_to_peers: bool = True, - skip_block_weight_verification: bool = False) -> bool: - """This method is called when any transaction arrive. - - If `fails_silently` is False, it may raise either InvalidNewTransaction or TxValidationError. - - :return: True if the transaction was accepted - :rtype: bool + skip_block_weight_verification: bool = False, partial: bool = False) -> bool: + """ New method for adding transactions or blocks that steps the validation state machine. """ assert tx.hash is not None - if self.state != self.NodeState.INITIALIZING: - if self.tx_storage.transaction_exists(tx.hash): - if not fails_silently: - raise InvalidNewTransaction('Transaction already exists {}'.format(tx.hash_hex)) - self.log.debug('on_new_tx(): Transaction already exists', tx=tx.hash_hex) - return False + if self.tx_storage.transaction_exists(tx.hash): + if not fails_silently: + raise InvalidNewTransaction('Transaction already exists {}'.format(tx.hash_hex)) + self.log.warn('on_new_tx(): Transaction already exists', tx=tx.hash_hex) + return False + + if tx.timestamp - self.reactor.seconds() > settings.MAX_FUTURE_TIMESTAMP_ALLOWED: + if not fails_silently: + raise InvalidNewTransaction('Ignoring transaction in the future {} (timestamp={})'.format( + tx.hash_hex, tx.timestamp)) + self.log.warn('on_new_tx(): Ignoring transaction in the future', tx=tx.hash_hex, + future_timestamp=tx.timestamp) + return False + + tx.storage = self.tx_storage + + try: + metadata = tx.get_metadata() + except TransactionDoesNotExist: + if not fails_silently: + raise InvalidNewTransaction('missing parent') + self.log.warn('on_new_tx(): missing parent', tx=tx.hash_hex) + return False - if self.state != self.NodeState.INITIALIZING or self._full_verification: + if metadata.validation.is_invalid(): + if not fails_silently: + raise InvalidNewTransaction('previously marked as invalid') + self.log.warn('on_new_tx(): previously marked as invalid', tx=tx.hash_hex) + return False + + # if partial=False (the default) we don't even try to partially validate transactions + if not partial or (metadata.validation.is_fully_connected() or tx.can_validate_full()): + if isinstance(tx, Transaction) and self.tx_storage.is_tx_needed(tx.hash): + tx._height_cache = self.tx_storage.needed_index_height(tx.hash) + + if not metadata.validation.is_valid(): + try: + tx.validate_full() + except HathorError as e: + if not fails_silently: + raise InvalidNewTransaction('full validation failed') from e + self.log.warn('on_new_tx(): full validation failed', tx=tx.hash_hex, exc_info=True) + return False + + # The method below adds the tx as a child of the parents + # This needs to be called right before the save because we were adding the children + # in the tx parents even if the tx was invalid (failing the verifications above) + # then I would have a children that was not in the storage + tx.update_initial_metadata() + self.tx_storage.save_transaction(tx, add_to_indexes=True) try: - assert self.validate_new_tx(tx, skip_block_weight_verification=skip_block_weight_verification) is True - except (InvalidNewTransaction, TxValidationError): - # Discard invalid Transaction/block. - self.log.debug('tx/block discarded', tx=tx, exc_info=True) + self.consensus_algorithm.update(tx) + except HathorError as e: + if not fails_silently: + raise InvalidNewTransaction('consensus update failed') from e + self.log.warn('on_new_tx(): consensus update failed', tx=tx.hash_hex) + return False + else: + self.tx_fully_validated(tx) + else: + if isinstance(tx, Block) and not tx.has_basic_block_parent(): + if not fails_silently: + raise InvalidNewTransaction('block parent needs to be at least basic-valid') + self.log.warn('on_new_tx(): block parent needs to be at least basic-valid', tx=tx.hash_hex) + return False + if not tx.validate_basic(): if not fails_silently: - raise + raise InvalidNewTransaction('basic validation failed') + self.log.warn('on_new_tx(): basic validation failed', tx=tx.hash_hex) return False - if self.state != self.NodeState.INITIALIZING: + # The method below adds the tx as a child of the parents + # This needs to be called right before the save because we were adding the children + # in the tx parents even if the tx was invalid (failing the verifications above) + # then I would have a children that was not in the storage + tx.update_initial_metadata() self.tx_storage.save_transaction(tx) - else: - self.tx_storage._add_to_cache(tx) - if self._full_verification: - tx.reset_metadata() - else: - # When doing a fast init, we don't update the consensus, so we must trust the data on the metadata - # For transactions, we don't store them on the tips index if they are voided - # We have to execute _add_to_cache before because _del_from_cache does not remove from all indexes - metadata = tx.get_metadata() - if not tx.is_block and metadata.voided_by: - self.tx_storage._del_from_cache(tx) - - if self.state != self.NodeState.INITIALIZING or self._full_verification: - try: - tx.update_initial_metadata() - self.consensus_algorithm.update(tx) - except Exception: - self.log.exception('unexpected error when processing tx', tx=tx) - self.tx_storage.remove_transaction(tx) - raise + self.tx_storage.add_to_deps_index(tx.hash, tx.get_all_dependencies()) + self.tx_storage.add_needed_deps(tx) + + if tx.is_transaction: + self.tx_storage.remove_from_needed_index(tx.hash) + + try: + self.step_validations([tx]) + except (AssertionError, HathorError) as e: + if not fails_silently: + raise InvalidNewTransaction('step validations failed') from e + self.log.warn('on_new_tx(): step validations failed', tx=tx.hash_hex) + return False if not quiet: ts_date = datetime.datetime.fromtimestamp(tx.timestamp) @@ -699,14 +805,51 @@ def on_new_tx(self, tx: BaseTransaction, *, conn: Optional[HathorProtocol] = Non # Propagate to our peers. self.connections.send_tx_to_peers(tx) - if self.wallet: - # TODO Remove it and use pubsub instead. - self.wallet.on_new_tx(tx) + return True + + def step_validations(self, txs: Iterable[BaseTransaction]) -> None: + """ Step all validations until none can be stepped anymore. + """ + # cur_txs will be empty when there are no more new txs that reached full + # validation because of an initial trigger + for ready_tx in txs: + assert ready_tx.hash is not None + self.tx_storage.remove_ready_for_validation(ready_tx.hash) + for tx in map(self.tx_storage.get_transaction, self.tx_storage.next_ready_for_validation()): + assert tx.hash is not None + tx.update_initial_metadata() + try: + assert tx.validate_full() + except (AssertionError, HathorError): + # TODO + raise + else: + self.tx_storage.save_transaction(tx, only_metadata=True, add_to_indexes=True) + self.consensus_algorithm.update(tx) + # save and process its dependencies even if it became invalid + # because invalidation state also has to propagate to children + self.tx_storage.remove_ready_for_validation(tx.hash) + self.tx_fully_validated(tx) - # Publish to pubsub manager the new tx accepted + def tx_fully_validated(self, tx: BaseTransaction) -> None: + """ Handle operations that need to happen once the tx becomes fully validated. + + This might happen immediately after we receive the tx, if we have all dependencies + already. Or it might happen later. + """ + assert tx.hash is not None + + # Publish to pubsub manager the new tx accepted, now that it's full validated self.pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=tx) - return True + self.tx_storage.del_from_deps_index(tx.hash) + self.tx_storage.update_tx_tips(tx) + if tx.is_block: + self.tx_storage.add_to_parent_blocks_index(tx.hash) + + if self.wallet: + # TODO Remove it and use pubsub instead. + self.wallet.on_new_tx(tx) def listen(self, description: str, use_ssl: Optional[bool] = None) -> None: endpoint = self.connections.listen(description, use_ssl) diff --git a/hathor/p2p/downloader.py b/hathor/p2p/downloader.py index 3cf2d6d585..93fd7e2d5b 100644 --- a/hathor/p2p/downloader.py +++ b/hathor/p2p/downloader.py @@ -203,12 +203,12 @@ def on_new_tx(self, tx: 'BaseTransaction') -> None: """ This is called when a new transaction arrives. """ assert tx.hash is not None - self.log.debug('new tx', tx=tx.hash_hex) + self.log.debug('new tx/block', tx=tx.hash_hex) details = self.pending_transactions.get(tx.hash, None) if not details: # Something is wrong! It should never happen. - self.log.warn('new tx arrived but tx detail does not exist', tx=tx.hash_hex) + self.log.warn('new tx/block arrived but tx detail does not exist', tx=tx.hash_hex) return assert len(self.downloading_deque) > 0 diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index 37233405e8..c9141f2eef 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -35,9 +35,9 @@ from hathor.transaction import BaseTransaction if TYPE_CHECKING: - from hathor.manager import HathorManager # noqa: F401 - from hathor.p2p.factory import HathorClientFactory, HathorServerFactory # noqa: F401 - from hathor.p2p.node_sync import NodeSyncTimestamp # noqa: F401 + from hathor.manager import HathorManager + from hathor.p2p.factory import HathorClientFactory, HathorServerFactory + from hathor.p2p.node_sync import NodeSyncTimestamp logger = get_logger() settings = HathorSettings() diff --git a/hathor/p2p/node_sync.py b/hathor/p2p/node_sync.py index 9ad841931c..930dabea9f 100644 --- a/hathor/p2p/node_sync.py +++ b/hathor/p2p/node_sync.py @@ -686,6 +686,10 @@ def handle_data(self, payload: str) -> None: assert tx.timestamp is not None self.requested_data_arrived(tx.timestamp) deferred.callback(tx) + elif self.manager.tx_storage.transaction_exists(tx.hash): + # transaction already added to the storage, ignore it + # XXX: maybe we could add a hash blacklist and punish peers propagating known bad txs + return else: # If we have not requested the data, it is a new transaction being propagated # in the network, thus, we propagate it as well. @@ -719,7 +723,7 @@ def get_data_key(self, hash_bytes: bytes) -> str: key = 'get-data-{}'.format(hash_bytes.hex()) return key - def remove_deferred(self, reason: 'Failure', hash_bytes: bytes) -> None: + def remove_deferred(self, reason: 'Failure', hash_bytes: bytes) -> None: """ Remove the deferred from the deferred_by_key Used when a requested tx deferred fails for some reason (not found, or timeout) """ diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index 03618168b0..7be96ae61e 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -331,15 +331,21 @@ def get_time_from_now(self, now: Optional[Any] = None) -> str: minutes, seconds = divmod(seconds, 60) return '{} days, {:02d}:{:02d}:{:02d}'.format(dt.days, hours, minutes, seconds) - def get_parents(self) -> Iterator['BaseTransaction']: + def get_parents(self, *, existing_only: bool = False) -> Iterator['BaseTransaction']: """Return an iterator of the parents :return: An iterator of the parents :rtype: Iter[BaseTransaction] """ + from hathor.transaction.storage.exceptions import TransactionDoesNotExist + for parent_hash in self.parents: assert self.storage is not None - yield self.storage.get_transaction(parent_hash) + try: + yield self.storage.get_transaction(parent_hash) + except TransactionDoesNotExist: + if not existing_only: + raise @property def is_genesis(self) -> bool: @@ -442,6 +448,9 @@ def can_validate_full(self) -> bool: """ Check if this transaction is ready to be fully validated, either all deps are full-valid or one is invalid. """ assert self.storage is not None + assert self.hash is not None + if self.is_genesis: + return True deps = self.get_all_dependencies() all_exist = True all_valid = True @@ -855,10 +864,11 @@ def update_initial_metadata(self) -> None: assert self.hash is not None assert self.storage is not None - for parent in self.get_parents(): + for parent in self.get_parents(existing_only=True): metadata = parent.get_metadata() - metadata.children.append(self.hash) - self.storage.save_transaction(parent, only_metadata=True) + if self.hash not in metadata.children: + metadata.children.append(self.hash) + self.storage.save_transaction(parent, only_metadata=True) def update_timestamp(self, now: int) -> None: """Update this tx's timestamp diff --git a/hathor/transaction/resources/__init__.py b/hathor/transaction/resources/__init__.py index 3c5897ad2f..ad66008da6 100644 --- a/hathor/transaction/resources/__init__.py +++ b/hathor/transaction/resources/__init__.py @@ -18,7 +18,6 @@ from hathor.transaction.resources.graphviz import GraphvizFullResource, GraphvizNeighboursResource from hathor.transaction.resources.mining import GetBlockTemplateResource, SubmitBlockResource from hathor.transaction.resources.push_tx import PushTxResource -from hathor.transaction.resources.tips_histogram import TipsHistogramResource from hathor.transaction.resources.transaction import TransactionResource from hathor.transaction.resources.transaction_confirmation import TransactionAccWeightResource from hathor.transaction.resources.tx_parents import TxParentsResource @@ -35,7 +34,6 @@ 'TransactionAccWeightResource', 'TransactionResource', 'DashboardTransactionResource', - 'TipsHistogramResource', 'TxParentsResource', 'ValidateAddressResource', ] diff --git a/hathor/transaction/resources/tips_histogram.py b/hathor/transaction/resources/tips_histogram.py deleted file mode 100644 index 0558c8c712..0000000000 --- a/hathor/transaction/resources/tips_histogram.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2021 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from twisted.web import resource - -from hathor.api_util import parse_get_arguments, set_cors -from hathor.cli.openapi_files.register import register_resource - -ARGS = ['begin', 'end'] - - -@register_resource -class TipsHistogramResource(resource.Resource): - """ Implements a web server API to return the tips in a timestamp interval. - Returns a list of timestamps and numbers of tips. - - You must run with option `--status `. - """ - isLeaf = True - - def __init__(self, manager): - self.manager = manager - - def render_GET(self, request): - """ Get request to /tips-histogram/ that return the number of tips between two timestamp - We expect two GET parameters: 'begin' and 'end' - - 'begin': int that indicates the beginning of the interval - 'end': int that indicates the end of the interval - - :rtype: string (json) - """ - request.setHeader(b'content-type', b'application/json; charset=utf-8') - set_cors(request, 'GET') - - parsed = parse_get_arguments(request.args, ARGS) - if not parsed['success']: - return json.dumps({ - 'success': False, - 'message': 'Missing parameter: {}'.format(parsed['missing']) - }).encode('utf-8') - - args = parsed['args'] - - # Get quantity for each - try: - begin = int(args['begin']) - except ValueError: - return json.dumps({ - 'success': False, - 'message': 'Invalid parameter, cannot convert to int: begin' - }).encode('utf-8') - - try: - end = int(args['end']) - except ValueError: - return json.dumps({ - 'success': False, - 'message': 'Invalid parameter, cannot convert to int: end' - }).encode('utf-8') - - v = [] - for timestamp in range(begin, end + 1): - tx_tips = self.manager.tx_storage.get_tx_tips(timestamp) - v.append((timestamp, len(tx_tips))) - - return json.dumps({'success': True, 'tips': v}).encode('utf-8') - - -TipsHistogramResource.openapi = { - '/tips-histogram': { - 'x-visibility': 'private', - 'get': { - 'tags': ['transaction'], - 'operationId': 'tips_histogram', - 'summary': 'Histogram of tips', - 'description': ('Returns a list of tuples (timestamp, quantity)' - 'for each timestamp in the requested interval'), - 'parameters': [ - { - 'name': 'begin', - 'in': 'query', - 'description': 'Beggining of the timestamp interval', - 'required': True, - 'schema': { - 'type': 'int' - } - }, - { - 'name': 'end', - 'in': 'query', - 'description': 'End of the timestamp interval', - 'required': True, - 'schema': { - 'type': 'int' - } - } - ], - 'responses': { - '200': { - 'description': 'Success', - 'content': { - 'application/json': { - 'examples': { - 'success': { - 'summary': 'Success', - 'value': [ - [ - 1547163020, - 1 - ], - [ - 1547163021, - 4 - ], - [ - 1547163022, - 2 - ] - ] - }, - 'error': { - 'summary': 'Invalid parameter', - 'value': { - 'success': False, - 'message': 'Missing parameter: begin' - } - } - } - } - } - } - } - } - } -} diff --git a/hathor/transaction/storage/binary_storage.py b/hathor/transaction/storage/binary_storage.py index c8a801e800..fdcec9a31b 100644 --- a/hathor/transaction/storage/binary_storage.py +++ b/hathor/transaction/storage/binary_storage.py @@ -59,8 +59,8 @@ def remove_transaction(self, tx): except FileNotFoundError: pass - def save_transaction(self, tx, *, only_metadata=False): - super().save_transaction(tx, only_metadata=only_metadata) + def save_transaction(self, tx, *, only_metadata=False, add_to_indexes=False): + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) self._save_transaction(tx, only_metadata=only_metadata) self._save_to_weakref(tx) diff --git a/hathor/transaction/storage/block_height_index.py b/hathor/transaction/storage/block_height_index.py new file mode 100644 index 0000000000..19b1aa4792 --- /dev/null +++ b/hathor/transaction/storage/block_height_index.py @@ -0,0 +1,62 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Tuple + +from hathor.transaction.genesis import BLOCK_GENESIS +from hathor.util import not_none + +BLOCK_GENESIS_HASH: bytes = not_none(BLOCK_GENESIS.hash) + + +class BlockHeightIndex: + """Store the block hash for each given height + """ + def __init__(self) -> None: + self._index: List[bytes] = [BLOCK_GENESIS_HASH] + + def add(self, height: int, block_hash: bytes, *, can_reorg: bool = False) -> None: + """Add new element to the cache. Must not be called for re-orgs. + """ + if len(self._index) < height: + raise ValueError(f'parent hash required (current height: {len(self._index)}, new height: {height})') + elif len(self._index) == height: + self._index.append(block_hash) + elif self._index[height] != block_hash: + if can_reorg: + del self._index[height:] + self._index.append(block_hash) + else: + raise ValueError('adding would cause a re-org, use can_reorg=True to accept re-orgs') + else: + # nothing to do (there are more blocks, but the block at height currently matches the added block) + pass + + def get(self, height: int) -> Optional[bytes]: + """ Return the block hash for the given height, or None if it is not set. + """ + if len(self._index) <= height: + return None + return self._index[height] + + def get_tip(self) -> bytes: + """ Return the best block hash, it returns the genesis when there is no other block + """ + return self._index[-1] + + def get_height_tip(self) -> Tuple[int, bytes]: + """ Return the best block height and hash, it returns the genesis when there is no other block + """ + height = len(self._index) - 1 + return height, self._index[height] diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index cb808da01e..3c3afeb1fb 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -119,12 +119,13 @@ def remove_transaction(self, tx: BaseTransaction) -> None: self.dirty_txs.discard(tx.hash) self._remove_from_weakref(tx) - def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: + def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: self._save_transaction(tx) self._save_to_weakref(tx) # call super which adds to index if needed - super().save_transaction(tx, only_metadata=only_metadata) + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) def get_all_genesis(self) -> Set[BaseTransaction]: return self.store.get_all_genesis() diff --git a/hathor/transaction/storage/compact_storage.py b/hathor/transaction/storage/compact_storage.py index e97dfc495f..1048531b66 100644 --- a/hathor/transaction/storage/compact_storage.py +++ b/hathor/transaction/storage/compact_storage.py @@ -73,8 +73,9 @@ def remove_transaction(self, tx: 'BaseTransaction') -> None: except FileNotFoundError: pass - def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - super().save_transaction(tx, only_metadata=only_metadata) + def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) self._save_transaction(tx, only_metadata=only_metadata) self._save_to_weakref(tx) diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index 5a4f350be8..33f41af199 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -48,8 +48,9 @@ def remove_transaction(self, tx: BaseTransaction) -> None: self.transactions.pop(tx.hash, None) self.metadata.pop(tx.hash, None) - def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: - super().save_transaction(tx, only_metadata=only_metadata) + def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) self._save_transaction(tx, only_metadata=only_metadata) def _save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: diff --git a/hathor/transaction/storage/old_rocksdb_storage.py b/hathor/transaction/storage/old_rocksdb_storage.py index 7035672f4b..f3ef0fa34a 100644 --- a/hathor/transaction/storage/old_rocksdb_storage.py +++ b/hathor/transaction/storage/old_rocksdb_storage.py @@ -54,8 +54,9 @@ def remove_transaction(self, tx: 'BaseTransaction') -> None: self._db.delete(tx.hash) self._remove_from_weakref(tx) - def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - super().save_transaction(tx, only_metadata=only_metadata) + def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) self._save_transaction(tx, only_metadata=only_metadata) self._save_to_weakref(tx) diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index 0706af9518..b35698b322 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -128,8 +128,9 @@ def remove_transaction(self, tx: 'BaseTransaction') -> None: self._db.delete((self._cf_meta, tx.hash)) self._remove_from_weakref(tx) - def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: - super().save_transaction(tx, only_metadata=only_metadata) + def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: + super().save_transaction(tx, only_metadata=only_metadata, add_to_indexes=add_to_indexes) self._save_transaction(tx, only_metadata=only_metadata) self._save_to_weakref(tx) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index e49302331c..9070e6522f 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -16,7 +16,7 @@ from abc import ABC, abstractmethod, abstractproperty from collections import deque from threading import Lock -from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Set, Tuple, cast +from typing import Any, Dict, FrozenSet, Iterable, Iterator, List, NamedTuple, Optional, Set, Tuple, cast from weakref import WeakValueDictionary from intervaltree.interval import Interval @@ -25,14 +25,21 @@ from hathor.conf import HathorSettings from hathor.indexes import IndexesManager, TokensIndex, TransactionsIndex, WalletIndex from hathor.pubsub import HathorEvents, PubSubManager +from hathor.transaction.base_transaction import BaseTransaction from hathor.transaction.block import Block +from hathor.transaction.storage.block_height_index import BlockHeightIndex from hathor.transaction.storage.exceptions import TransactionDoesNotExist, TransactionIsNotABlock -from hathor.transaction.transaction import BaseTransaction -from hathor.transaction.transaction_metadata import TransactionMetadata +from hathor.transaction.storage.traversal import BFSWalk +from hathor.transaction.transaction import Transaction +from hathor.transaction.transaction_metadata import TransactionMetadata, ValidationState +from hathor.util import not_none settings = HathorSettings() +INF_HEIGHT: int = 1_000_000_000_000 + + class AllTipsCache(NamedTuple): timestamp: int tips: Set[Interval] @@ -40,6 +47,14 @@ class AllTipsCache(NamedTuple): hashes: List[bytes] +class _DirDepValue(Dict[bytes, ValidationState]): + """This class is used to add a handy method to values on dependency indexes.""" + + def is_ready(self) -> bool: + """True if all deps' validation are fully connected.""" + return all(val.is_fully_connected() for val in self.values()) + + class TransactionStorage(ABC): """Legacy sync interface, please copy @deprecated decorator when implementing methods.""" @@ -53,6 +68,8 @@ class TransactionStorage(ABC): log = get_logger() def __init__(self): + from hathor.transaction.genesis import BLOCK_GENESIS + # Weakref is used to guarantee that there is only one instance of each transaction in memory. self._tx_weakref: WeakValueDictionary[bytes, BaseTransaction] = WeakValueDictionary() self._tx_weakref_disabled: bool = False @@ -94,6 +111,310 @@ def __init__(self): # Key storage attribute to save if the node has clean db self._clean_db_attribute: str = 'clean_db' + # Cache of block hash by height + self._block_height_index = BlockHeightIndex() + + # Direct and reverse dependency mapping (i.e. needs and needed by) + self._dir_dep_index: Dict[bytes, _DirDepValue] = {} + self._rev_dep_index: Dict[bytes, Set[bytes]] = {} + self._txs_with_deps_ready: Set[bytes] = set() + + # Needed txs (key: tx missing, value: requested by) + self._needed_txs_index: Dict[bytes, Tuple[int, bytes]] = {} + + # Hold txs that have not been confirmed + self._tx_tips_index: Set[bytes] = set() + + # Hold blocks that can be used as the next parent block + # XXX: if there is more than one they must all have the same score, must always have at least one hash + self._parent_blocks_index: Set[bytes] = {BLOCK_GENESIS.hash} + + # rev-dep-index methods: + + def count_deps_index(self) -> int: + """Count total number of txs with dependencies.""" + return len(self._dir_dep_index) + + def _get_validation_state(self, tx: bytes) -> ValidationState: + """Query database for the validation state of a transaction, returns INITIAL when tx does not exist.""" + tx_meta = self.get_metadata(tx) + if tx_meta is None: + return ValidationState.INITIAL + return tx_meta.validation + + def _update_deps(self, deps: _DirDepValue) -> None: + """Propagate the new validation state of the given deps.""" + for tx, validation in deps.items(): + self._update_validation(tx, validation) + + def _update_validation(self, tx: bytes, validation: ValidationState) -> None: + """Propagate the new validation state of a given dep.""" + for cousin in self._rev_dep_index[tx].copy(): + deps = self._dir_dep_index[cousin] + # XXX: this check serves to avoid calling is_ready() when nothing changed + if deps[tx] != validation: + deps[tx] = validation + if deps.is_ready(): + self.del_from_deps_index(cousin) + self._txs_with_deps_ready.add(cousin) + + def add_to_deps_index(self, tx: bytes, deps: Iterable[bytes]) -> None: + """Call to add all dependencies a transaction has.""" + # deps are immutable for a given hash + _deps = _DirDepValue((dep, self._get_validation_state(dep)) for dep in deps) + # short circuit add directly to ready + if _deps.is_ready(): + self._txs_with_deps_ready.add(tx) + return + # add direct deps + if __debug__ and tx in self._dir_dep_index: + # XXX: dependencies set must be immutable + assert self._dir_dep_index[tx].keys() == _deps.keys() + self._dir_dep_index[tx] = _deps + # add reverse dep + for rev_dep in _deps: + if rev_dep not in self._rev_dep_index: + self._rev_dep_index[rev_dep] = set() + self._rev_dep_index[rev_dep].add(tx) + + def del_from_deps_index(self, tx: bytes) -> None: + """Call to remove tx from all reverse dependencies, for example when validation is complete.""" + _deps = self._dir_dep_index.pop(tx, _DirDepValue()) + for rev_dep in _deps.keys(): + rev_deps = self._rev_dep_index[rev_dep] + if tx in rev_deps: + rev_deps.remove(tx) + if not rev_deps: + del self._rev_dep_index[rev_dep] + + def is_ready_for_validation(self, tx: bytes) -> bool: + """ Whether a tx can be fully validated (implies fully connected). + """ + return tx in self._txs_with_deps_ready + + def remove_ready_for_validation(self, tx: bytes) -> None: + """ Removes from ready for validation set. + """ + self._txs_with_deps_ready.discard(tx) + + def next_ready_for_validation(self, *, dry_run: bool = False) -> Iterator[bytes]: + """ Yields and removes all txs ready for validation even if they become ready while iterating. + """ + if dry_run: + cur_ready = self._txs_with_deps_ready.copy() + else: + cur_ready, self._txs_with_deps_ready = self._txs_with_deps_ready, set() + while cur_ready: + yield from iter(cur_ready) + if dry_run: + cur_ready = self._txs_with_deps_ready - cur_ready + else: + cur_ready, self._txs_with_deps_ready = self._txs_with_deps_ready, set() + + def iter_deps_index(self) -> Iterator[bytes]: + """Iterate through all hashes depended by any tx or block.""" + yield from self._rev_dep_index.keys() + + def get_rev_deps(self, tx: bytes) -> FrozenSet[bytes]: + """Get all txs that depend on the given tx (i.e. its reverse depdendencies).""" + return frozenset(self._rev_dep_index.get(tx, set())) + + # needed-txs-index methods: + + def has_needed_tx(self) -> bool: + """Whether there is any tx on the needed tx index.""" + return bool(self._needed_txs_index) + + def is_tx_needed(self, tx: bytes) -> bool: + """Whether a tx is in the requested tx list.""" + return tx in self._needed_txs_index + + def needed_index_height(self, tx: bytes) -> int: + """Indexed height from the needed tx index.""" + return self._needed_txs_index[tx][0] + + def remove_from_needed_index(self, tx: bytes) -> None: + """Remove tx from needed txs index, tx doesn't need to be in the index.""" + self._needed_txs_index.pop(tx, None) + + def get_next_needed_tx(self) -> bytes: + """Choose the start hash for downloading the needed txs""" + # This strategy maximizes the chance to download multiple txs on the same stream + # find the tx with highest "height" + # XXX: we could cache this onto `needed_txs` so we don't have to fetch txs every time + height, start_hash, tx = max((h, s, t) for t, (h, s) in self._needed_txs_index.items()) + self.log.debug('next needed tx start', needed=len(self._needed_txs_index), start=start_hash.hex(), + height=height, needed_tx=tx.hex()) + return start_hash + + def add_needed_deps(self, tx: BaseTransaction) -> None: + if isinstance(tx, Block): + height = tx.get_metadata().height + else: + assert isinstance(tx, Transaction) + first_block = tx.get_metadata().first_block + if first_block is None: + # XXX: consensus did not run yet to update first_block, what should we do? + # I'm defaulting the height to `inf` (practically), this should make it heightest priority when + # choosing which transactions to fetch next + height = INF_HEIGHT + else: + block = self.get_transaction(first_block) + assert isinstance(block, Block) + height = block.get_metadata().height + # get_tx_parents is used instead of get_tx_dependencies because the remote will traverse the parent + # tree, not # the dependency tree, eventually we should receive all tx dependencies and be able to validate + # this transaction + for tx_hash in tx.get_tx_parents(): + # It may happen that we have one of the dependencies already, so just add the ones we don't + # have. We should add at least one dependency, otherwise this tx should be full validated + if not self.transaction_exists(tx_hash): + self._needed_txs_index[tx_hash] = (height, not_none(tx.hash)) + + # parent-blocks-index methods: + + def add_to_parent_blocks_index(self, block: bytes) -> None: + from math import isclose + meta = not_none(self.get_metadata(block)) + new_score = not_none(meta).score + cur_score = not_none(self.get_metadata(next(iter(self._parent_blocks_index)))).score + if isclose(new_score, cur_score): + self.log.debug('same score: new competing parent block', block=block.hex()) + self._parent_blocks_index.add(block) + elif new_score > cur_score and not meta.voided_by: + # If it's a high score, then I can't add one that is voided + self.log.debug('high score: new best parent block', block=block.hex()) + self._parent_blocks_index.clear() + self._parent_blocks_index.add(block) + else: + self.log.debug('low score: skip parent block', block=block.hex()) + + # tx-tips-index methods: + + def iter_tx_tips(self, max_timestamp: Optional[float] = None) -> Iterator[Transaction]: + """ + Iterate over txs that are tips, a subset of the mempool (i.e. not tx-parent of another tx on the mempool). + """ + it = map(self.get_transaction, self._tx_tips_index) + if max_timestamp is not None: + it = filter(lambda tx: tx.timestamp < not_none(max_timestamp), it) + yield from cast(Iterator[Transaction], it) + + def remove_from_tx_tips_index(self, remove_txs: Set[bytes]) -> None: + """ + This should be called to remove a transaction from the "mempool", usually when it is confirmed by a block. + """ + self._tx_tips_index -= remove_txs + + def iter_mempool(self) -> Iterator[Transaction]: + """ + Iterate over the transactions on the "mempool", even the ones that are not tips. + """ + bfs = BFSWalk(self, is_dag_verifications=True, is_left_to_right=False) + for tx in bfs.run(map(self.get_transaction, self._tx_tips_index), skip_root=False): + assert isinstance(tx, Transaction) + yield tx + if tx.get_metadata().first_block is not None: + bfs.skip_neighbors(tx) + + def update_tx_tips(self, tx: BaseTransaction) -> None: + """ + This should be called when a new `tx` is created and added to the "mempool". + """ + # A new tx/block added might cause a tx in the tips to become voided. For instance, + # there might be a tx1 a double spending tx2, where tx1 is valid and tx2 voided. A new block + # confirming tx2 will make it valid while tx1 becomes voided, so it has to be removed + # from the tips. + assert tx.hash is not None + to_remove: Set[bytes] = set() + to_remove_parents: Set[bytes] = set() + for tip_tx in self.iter_tx_tips(): + assert tip_tx.hash is not None + # A new tx/block added might cause a tx in the tips to become voided. For instance, + # there might be twin txs, tx1 and tx2, where tx1 is valid and tx2 voided. A new block + # confirming tx2 will make it valid while tx1 becomes voided, so it has to be removed + # from the tips. The txs confirmed by tx1 need to be double checked, as they might + # themselves become tips (hence we use to_remove_parents) + meta = tip_tx.get_metadata() + if meta.voided_by: + to_remove.add(tip_tx.hash) + to_remove_parents.update(tip_tx.parents) + continue + + # might also happen that a tip has a child that became valid, so it's not a tip anymore + confirmed = False + for child_meta in map(self.get_metadata, meta.children): + assert child_meta is not None + if not child_meta.voided_by: + confirmed = True + break + if confirmed: + to_remove.add(tip_tx.hash) + + if to_remove: + self.remove_from_tx_tips_index(to_remove) + self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove]) + + # Check if any of the txs being confirmed by the voided txs is a tip again. This happens + # if it doesn't have any other valid child. + to_add = set() + for tx_hash in to_remove_parents: + confirmed = False + # check if it has any valid children + meta = not_none(self.get_metadata(tx_hash)) + if meta.voided_by: + continue + children = meta.children + for child_meta in map(self.get_metadata, children): + assert child_meta is not None + if not child_meta.voided_by: + confirmed = True + break + if not confirmed: + to_add.add(tx_hash) + + if to_add: + self._tx_tips_index.update(to_add) + self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add]) + + if tx.get_metadata().voided_by: + # this tx is voided, don't need to update the tips + self.log.debug('voided tx, won\'t add it as a tip', tx=tx.hash_hex) + return + + self.remove_from_tx_tips_index(set(tx.parents)) + + if tx.is_transaction and tx.get_metadata().first_block is None: + self._tx_tips_index.add(tx.hash) + + # block height index methods: + + def update_block_height_cache_new_chain(self, height: int, block: Block) -> None: + """ When we have a new winner chain we must update all the height index + until the first height with a common block + """ + assert self.get_from_block_height_index(height) != block.hash + + block_height = height + side_chain_block = block + add_to_cache: List[Tuple[int, bytes]] = [] + while self.get_from_block_height_index(block_height) != side_chain_block.hash: + add_to_cache.append((block_height, not_none(side_chain_block.hash))) + + side_chain_block = side_chain_block.get_block_parent() + new_block_height = side_chain_block.get_metadata().height + assert new_block_height + 1 == block_height + block_height = new_block_height + + # Reverse the data because I was adding in the array from the highest block + reversed_add_to_cache = add_to_cache[::-1] + + for height, tx_hash in reversed_add_to_cache: + # Add it to the index + self.add_reorg_to_block_height_index(height, tx_hash) + + # all other methods: + def is_empty(self) -> bool: """True when only genesis is present, useful for checking for a fresh database.""" return self.get_count_tx_blocks() <= 3 @@ -102,6 +423,13 @@ def pre_init(self) -> None: """Storages can implement this to run code before transaction loading starts""" pass + def get_best_block(self) -> Block: + """The block with highest score or one of the blocks with highest scores. Can be used for mining.""" + block_hash = self._block_height_index.get_tip() + block = self.get_transaction(block_hash) + assert isinstance(block, Block) + return block + def _save_or_verify_genesis(self) -> None: """Save all genesis in the storage.""" for tx in self._get_genesis_from_settings(): @@ -110,7 +438,7 @@ def _save_or_verify_genesis(self) -> None: tx2 = self.get_transaction(tx.hash) assert tx == tx2 except TransactionDoesNotExist: - self.save_transaction(tx) + self.save_transaction(tx, add_to_indexes=True) tx2 = tx assert tx2.hash is not None self._genesis_cache[tx2.hash] = tx2 @@ -158,23 +486,32 @@ def _disable_weakref(self) -> None: self._tx_weakref_disabled = True @abstractmethod - def save_transaction(self: 'TransactionStorage', tx: BaseTransaction, *, only_metadata: bool = False) -> None: + def save_transaction(self: 'TransactionStorage', tx: BaseTransaction, *, only_metadata: bool = False, + add_to_indexes: bool = False) -> None: # XXX: although this method is abstract (because a subclass must implement it) the implementer # should call the base implementation for correctly interacting with the index """Saves the tx. :param tx: Transaction to save :param only_metadata: Don't save the transaction, only the metadata of this transaction + :param add_to_indexes: Add this transaction to the indexes """ meta = tx.get_metadata() + if tx.hash in self._rev_dep_index: + self._update_validation(tx.hash, meta.validation) + + # XXX: we can only add to cache and publish txs that are complete, having the parents and all inputs + if not meta.validation.is_valid(): + return + if self.pubsub: if not meta.voided_by: self.pubsub.publish(HathorEvents.STORAGE_TX_WINNER, tx=tx) else: self.pubsub.publish(HathorEvents.STORAGE_TX_VOIDED, tx=tx) - if self.with_index and not only_metadata: - self._add_to_cache(tx) + if self.with_index and add_to_indexes: + self.add_to_indexes(tx) @abstractmethod def remove_transaction(self, tx: BaseTransaction) -> None: @@ -185,8 +522,8 @@ def remove_transaction(self, tx: BaseTransaction) -> None: if self.with_index: assert self.all_index is not None - self._del_from_cache(tx, relax_assert=True) - # TODO Move it to self._del_from_cache. We cannot simply do it because + self.del_from_indexes(tx, relax_assert=True) + # TODO Move it to self.del_from_indexes. We cannot simply do it because # this method is used by the consensus algorithm which does not # expect to have it removed from self.all_index. self.all_index.del_tx(tx, relax_assert=True) @@ -294,10 +631,11 @@ def get_best_block_tips(self, timestamp: Optional[float] = None, *, skip_cache: """ if timestamp is None and not skip_cache and self._best_block_tips is not None: return self._best_block_tips + best_score = 0.0 - best_tip_blocks = [] # List[bytes(hash)] - tip_blocks = [x.data for x in self.get_block_tips(timestamp)] - for block_hash in tip_blocks: + best_tip_blocks: List[bytes] = [] + + for block_hash in (x.data for x in self.get_block_tips(timestamp)): meta = self.get_metadata(block_hash) assert meta is not None if meta.voided_by and meta.voided_by != set([block_hash]): @@ -456,11 +794,11 @@ def _topological_sort(self) -> Iterator[BaseTransaction]: raise NotImplementedError @abstractmethod - def _add_to_cache(self, tx: BaseTransaction) -> None: + def add_to_indexes(self, tx: BaseTransaction) -> None: raise NotImplementedError @abstractmethod - def _del_from_cache(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: + def del_from_indexes(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: raise NotImplementedError @abstractmethod @@ -563,6 +901,18 @@ def is_db_clean(self) -> bool: """ return self.get_value(self._clean_db_attribute) == '1' + def add_new_to_block_height_index(self, height: int, block_hash: bytes) -> None: + """Add a new block to the height index that must not result in a re-org""" + self._block_height_index.add(height, block_hash) + + def add_reorg_to_block_height_index(self, height: int, block_hash: bytes) -> None: + """Add a new block to the height index that can result in a re-org""" + # XXX: in the future we can make this more strict so that it MUST result in a re-orgr + self._block_height_index.add(height, block_hash, can_reorg=True) + + def get_from_block_height_index(self, height: int) -> bytes: + return self._block_height_index.get(height) + class BaseTransactionStorage(TransactionStorage): def __init__(self, with_index: bool = True, pubsub: Optional[Any] = None) -> None: @@ -640,11 +990,12 @@ def get_tx_tips(self, timestamp: Optional[float] = None) -> Set[Interval]: timestamp = self.latest_timestamp tips = self.tx_index.tips_index[timestamp] - # This `for` is for assert only. How to skip it when running with `-O` parameter? - for interval in tips: - meta = self.get_metadata(interval.data) - assert meta is not None - assert not meta.voided_by + if __debug__: + # XXX: this `for` is for assert only and thus is inside `if __debug__:` + for interval in tips: + meta = self.get_metadata(interval.data) + assert meta is not None + # assert not meta.voided_by return tips @@ -721,7 +1072,7 @@ def _manually_initialize(self) -> None: # We need to construct a topological sort, then iterate from # genesis to tips. for tx in self._topological_sort(): - self._add_to_cache(tx) + self.add_to_indexes(tx) def _topological_sort(self) -> Iterator[BaseTransaction]: # TODO We must optimize this algorithm to remove the `visited` set. @@ -761,15 +1112,25 @@ def _topological_sort_dfs(self, root: BaseTransaction, visited: Dict[bytes, int] # matter. for parent_hash in tx.parents[::-1]: if parent_hash not in visited: - parent = self.get_transaction(parent_hash) - stack.append(parent) + try: + parent = self.get_transaction(parent_hash) + except TransactionDoesNotExist: + # XXX: it's possible transactions won't exist because of missing dependencies + pass + else: + stack.append(parent) for txin in tx.inputs: if txin.tx_id not in visited: - txinput = self.get_transaction(txin.tx_id) - stack.append(txinput) - - def _add_to_cache(self, tx: BaseTransaction) -> None: + try: + txinput = self.get_transaction(txin.tx_id) + except TransactionDoesNotExist: + # XXX: it's possible transactions won't exist because of missing dependencies + pass + else: + stack.append(txinput) + + def add_to_indexes(self, tx: BaseTransaction) -> None: if not self.with_index: raise NotImplementedError assert self.all_index is not None @@ -794,7 +1155,7 @@ def _add_to_cache(self, tx: BaseTransaction) -> None: if self.tx_index.add_tx(tx): self._cache_tx_count += 1 - def _del_from_cache(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: + def del_from_indexes(self, tx: BaseTransaction, *, relax_assert: bool = False) -> None: if not self.with_index: raise NotImplementedError assert self.block_index is not None diff --git a/hathor/wallet/base_wallet.py b/hathor/wallet/base_wallet.py index 30aaebb0c6..0377effddb 100644 --- a/hathor/wallet/base_wallet.py +++ b/hathor/wallet/base_wallet.py @@ -521,6 +521,7 @@ def on_new_tx(self, tx: BaseTransaction) -> None: script_type_out = parse_address_script(output.script) if script_type_out: if script_type_out.address in self.keys: + self.log.debug('detected tx output', tx=tx.hash_hex, index=index, address=script_type_out.address) token_id = tx.get_token_uid(output.get_token_index()) # this wallet received tokens utxo = UnspentTx(tx.hash, index, output.value, tx.timestamp, script_type_out.address, diff --git a/tests/resources/transaction/test_tips_histogram.py b/tests/resources/transaction/test_tips_histogram.py deleted file mode 100644 index d0cf5736b8..0000000000 --- a/tests/resources/transaction/test_tips_histogram.py +++ /dev/null @@ -1,72 +0,0 @@ -import time - -from twisted.internet.defer import inlineCallbacks - -from hathor.transaction.resources import TipsHistogramResource -from tests.resources.base_resource import StubSite, _BaseResourceTest -from tests.utils import add_blocks_unlock_reward, add_new_blocks, add_new_transactions - - -class TipsTest(_BaseResourceTest._ResourceTest): - def setUp(self): - super().setUp() - self.web = StubSite(TipsHistogramResource(self.manager)) - self.manager.wallet.unlock(b'MYPASS') - self.manager.reactor.advance(time.time()) - - @inlineCallbacks - def test_get_tips_histogram(self): - # Add blocks to have funds - add_new_blocks(self.manager, 2, 2) - add_blocks_unlock_reward(self.manager) - - txs = add_new_transactions(self.manager, 10, 2) - - response1 = yield self.web.get("tips-histogram", { - b'begin': str(txs[0].timestamp).encode(), - b'end': str(txs[0].timestamp).encode() - }) - data1 = response1.json_value() - self.assertTrue(data1['success']) - self.assertEqual(len(data1['tips']), 1) - self.assertEqual([txs[0].timestamp, 1], data1['tips'][0]) - - response2 = yield self.web.get("tips-histogram", { - b'begin': str(txs[0].timestamp).encode(), - b'end': str(txs[0].timestamp + 1).encode() - }) - data2 = response2.json_value() - self.assertTrue(data2['success']) - self.assertEqual(len(data2['tips']), 2) - self.assertEqual([txs[0].timestamp, 1], data2['tips'][0]) - self.assertEqual([txs[0].timestamp + 1, 1], data2['tips'][1]) - - response3 = yield self.web.get("tips-histogram", { - b'begin': str(txs[0].timestamp).encode(), - b'end': str(txs[-1].timestamp).encode() - }) - data3 = response3.json_value() - self.assertTrue(data3['success']) - self.assertEqual(len(data3['tips']), 19) - - @inlineCallbacks - def test_invalid_params(self): - # missing end param - response = yield self.web.get("tips-histogram", {b'begin': b'0'}) - data = response.json_value() - self.assertFalse(data['success']) - - # wrong end param - response = yield self.web.get("tips-histogram", {b'begin': b'a', b'end': b'10'}) - data = response.json_value() - self.assertFalse(data['success']) - - # missing begin param - response = yield self.web.get("tips-histogram", {b'end': b'0'}) - data = response.json_value() - self.assertFalse(data['success']) - - # wrong begin param - response = yield self.web.get("tips-histogram", {b'begin': b'0', b'end': b'a'}) - data = response.json_value() - self.assertFalse(data['success']) diff --git a/tests/tx/test_indexes.py b/tests/tx/test_indexes.py new file mode 100644 index 0000000000..9f61930c48 --- /dev/null +++ b/tests/tx/test_indexes.py @@ -0,0 +1,134 @@ +from hathor.crypto.util import decode_address +from hathor.transaction import Transaction +from hathor.transaction.storage import TransactionMemoryStorage +from hathor.wallet import Wallet +from tests import unittest +from tests.utils import add_blocks_unlock_reward, add_new_blocks, get_genesis_key + + +class BasicTransaction(unittest.TestCase): + def setUp(self): + super().setUp() + self.wallet = Wallet() + self.tx_storage = TransactionMemoryStorage() + self.genesis = self.tx_storage.get_all_genesis() + self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] + self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] + + # read genesis keys + self.genesis_private_key = get_genesis_key() + self.genesis_public_key = self.genesis_private_key.public_key() + + # this makes sure we can spend the genesis outputs + self.manager = self.create_peer('testnet', tx_storage=self.tx_storage, unlock_wallet=True, wallet_index=True) + blocks = add_blocks_unlock_reward(self.manager) + self.last_block = blocks[-1] + + def test_tx_tips_with_conflict(self): + from hathor.wallet.base_wallet import WalletOutputInfo + + add_new_blocks(self.manager, 5, advance_clock=15) + add_blocks_unlock_reward(self.manager) + + address = self.get_address(0) + value = 500 + + outputs = [WalletOutputInfo(address=decode_address(address), value=value, timelock=None)] + + tx1 = self.manager.wallet.prepare_transaction_compute_inputs(Transaction, outputs, self.manager.tx_storage) + tx1.weight = 2.0 + tx1.parents = self.manager.get_new_tx_parents() + tx1.timestamp = int(self.clock.seconds()) + tx1.resolve() + self.assertTrue(self.manager.propagate_tx(tx1, False)) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + {tx1.hash} + ) + + outputs = [WalletOutputInfo(address=decode_address(address), value=value, timelock=None)] + + tx2 = self.manager.wallet.prepare_transaction_compute_inputs(Transaction, outputs, self.manager.tx_storage) + tx2.weight = 2.0 + tx2.parents = [tx1.hash] + self.manager.get_new_tx_parents()[1:] + self.assertIn(tx1.hash, tx2.parents) + tx2.timestamp = int(self.clock.seconds()) + 1 + tx2.resolve() + self.assertTrue(self.manager.propagate_tx(tx2, False)) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + {tx2.hash} + ) + + tx3 = Transaction.create_from_struct(tx2.get_struct()) + tx3.timestamp = tx2.timestamp + 1 + self.assertIn(tx1.hash, tx3.parents) + tx3.resolve() + self.assertNotEqual(tx2.hash, tx3.hash) + self.assertTrue(self.manager.propagate_tx(tx3, False)) + self.assertIn(tx3.hash, tx2.get_metadata().conflict_with) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + # XXX: what should we expect here? I don't think we should exclude both tx2 and tx3, but maybe let the + # function using the index decide + # {tx1.hash, tx3.hash} + {tx1.hash} + ) + + def test_tx_tips_voided(self): + from hathor.wallet.base_wallet import WalletOutputInfo + + add_new_blocks(self.manager, 5, advance_clock=15) + add_blocks_unlock_reward(self.manager) + + address1 = self.get_address(0) + address2 = self.get_address(1) + address3 = self.get_address(2) + output1 = WalletOutputInfo(address=decode_address(address1), value=123, timelock=None) + output2 = WalletOutputInfo(address=decode_address(address2), value=234, timelock=None) + output3 = WalletOutputInfo(address=decode_address(address3), value=345, timelock=None) + outputs = [output1, output2, output3] + + tx1 = self.manager.wallet.prepare_transaction_compute_inputs(Transaction, outputs, self.manager.tx_storage) + tx1.weight = 2.0 + tx1.parents = self.manager.get_new_tx_parents() + tx1.timestamp = int(self.clock.seconds()) + tx1.resolve() + self.assertTrue(self.manager.propagate_tx(tx1, False)) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + {tx1.hash} + ) + + tx2 = self.manager.wallet.prepare_transaction_compute_inputs(Transaction, outputs, self.manager.tx_storage) + tx2.weight = 2.0 + tx2.parents = [tx1.hash] + self.manager.get_new_tx_parents()[1:] + self.assertIn(tx1.hash, tx2.parents) + tx2.timestamp = int(self.clock.seconds()) + 1 + tx2.resolve() + self.assertTrue(self.manager.propagate_tx(tx2, False)) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + {tx2.hash} + ) + + tx3 = Transaction.create_from_struct(tx2.get_struct()) + tx3.weight = 3.0 + # tx3.timestamp = tx2.timestamp + 1 + tx3.parents = tx1.parents + # self.assertIn(tx1.hash, tx3.parents) + tx3.resolve() + self.assertNotEqual(tx2.hash, tx3.hash) + self.assertTrue(self.manager.propagate_tx(tx3, False)) + # self.assertIn(tx3.hash, tx2.get_metadata().voided_by) + self.assertIn(tx3.hash, tx2.get_metadata().conflict_with) + self.assertEqual( + {tx.hash for tx in self.manager.tx_storage.iter_tx_tips()}, + # XXX: what should we expect here? I don't think we should exclude both tx2 and tx3, but maybe let the + # function using the index decide + {tx1.hash, tx3.hash} + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tx/test_tx_storage.py b/tests/tx/test_tx_storage.py index ed403a6539..e0e87f3b20 100644 --- a/tests/tx/test_tx_storage.py +++ b/tests/tx/test_tx_storage.py @@ -24,6 +24,7 @@ TransactionRocksDBStorage, ) from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.transaction_metadata import ValidationState from hathor.wallet import Wallet from tests.utils import ( BURN_ADDRESS, @@ -47,6 +48,8 @@ class _BaseTransactionStorageTest: class _TransactionStorageTest(unittest.TestCase): def setUp(self, tx_storage, reactor=None): + from hathor.manager import HathorManager + if not reactor: self.reactor = Clock() else: @@ -61,7 +64,6 @@ def setUp(self, tx_storage, reactor=None): self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] - from hathor.manager import HathorManager self.tmpdir = tempfile.mkdtemp() wallet = Wallet(directory=self.tmpdir) wallet.unlock(b'teste') @@ -76,6 +78,7 @@ def setUp(self, tx_storage, reactor=None): nonce=100781, storage=tx_storage) self.block.resolve() self.block.verify() + self.block.get_metadata().validation = ValidationState.FULL tx_parents = [tx.hash for tx in self.genesis_txs] tx_input = TxInput( @@ -91,6 +94,7 @@ def setUp(self, tx_storage, reactor=None): tokens=[bytes.fromhex('0023be91834c973d6a6ddd1a0ae411807b7c8ef2a015afb5177ee64b666ce602')], parents=tx_parents, storage=tx_storage) self.tx.resolve() + self.tx.get_metadata().validation = ValidationState.FULL # Disable weakref to test the internal methods. Otherwise, most methods return objects from weakref. self.tx_storage._disable_weakref() @@ -151,7 +155,7 @@ def test_storage_basic(self): self.assertEqual(set(tx_parents_hash), {self.genesis_txs[0].hash, self.genesis_txs[1].hash}) def validate_save(self, obj): - self.tx_storage.save_transaction(obj) + self.tx_storage.save_transaction(obj, add_to_indexes=True) loaded_obj1 = self.tx_storage.get_transaction(obj.hash) @@ -170,7 +174,7 @@ def validate_save(self, obj): else: self.assertTrue(obj.hash in self.tx_storage.tx_index.tips_index.tx_last_interval) - self.tx_storage._del_from_cache(obj) + self.tx_storage.del_from_indexes(obj) if self.tx_storage.with_index: if obj.is_block: @@ -178,7 +182,7 @@ def validate_save(self, obj): else: self.assertFalse(obj.hash in self.tx_storage.tx_index.tips_index.tx_last_interval) - self.tx_storage._add_to_cache(obj) + self.tx_storage.add_to_indexes(obj) if self.tx_storage.with_index: if obj.is_block: self.assertTrue(obj.hash in self.tx_storage.block_index.tips_index.tx_last_interval) @@ -193,6 +197,7 @@ def test_save_tx(self): def test_save_token_creation_tx(self): tx = create_tokens(self.manager, propagate=False) + tx.get_metadata().validation = ValidationState.FULL self.validate_save(tx) def _validate_not_in_index(self, tx, index): From ba9fc463d29f1eb9dbc2553583803daa97a3dc83 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 18 Mar 2021 16:24:40 -0300 Subject: [PATCH 13/15] feat: add block at height API --- hathor/cli/run_node.py | 2 + hathor/transaction/resources/__init__.py | 2 + .../transaction/resources/block_at_height.py | 173 ++++++++++++++++++ .../transaction/test_block_at_height.py | 50 +++++ 4 files changed, 227 insertions(+) create mode 100644 hathor/transaction/resources/block_at_height.py create mode 100644 tests/resources/transaction/test_block_at_height.py diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 0b5e2df614..944594e3d7 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -291,6 +291,7 @@ def register_resources(self, args: Namespace) -> None: from hathor.profiler.resources import CPUProfilerResource, ProfilerResource from hathor.prometheus import PrometheusMetricsExporter from hathor.transaction.resources import ( + BlockAtHeightResource, CreateTxResource, DashboardTransactionResource, DecodeTxResource, @@ -374,6 +375,7 @@ def register_resources(self, args: Namespace) -> None: root), (b'graphviz', graphviz, root), (b'transaction', TransactionResource(self.manager), root), + (b'block_at_height', BlockAtHeightResource(self.manager), root), (b'transaction_acc_weight', TransactionAccWeightResource(self.manager), root), (b'dashboard_tx', DashboardTransactionResource(self.manager), root), (b'profiler', ProfilerResource(self.manager), root), diff --git a/hathor/transaction/resources/__init__.py b/hathor/transaction/resources/__init__.py index ad66008da6..30d9e3c265 100644 --- a/hathor/transaction/resources/__init__.py +++ b/hathor/transaction/resources/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from hathor.transaction.resources.block_at_height import BlockAtHeightResource from hathor.transaction.resources.create_tx import CreateTxResource from hathor.transaction.resources.dashboard import DashboardTransactionResource from hathor.transaction.resources.decode_tx import DecodeTxResource @@ -24,6 +25,7 @@ from hathor.transaction.resources.validate_address import ValidateAddressResource __all__ = [ + 'BlockAtHeightResource', 'CreateTxResource', 'DecodeTxResource', 'PushTxResource', diff --git a/hathor/transaction/resources/block_at_height.py b/hathor/transaction/resources/block_at_height.py new file mode 100644 index 0000000000..af6b15c5ec --- /dev/null +++ b/hathor/transaction/resources/block_at_height.py @@ -0,0 +1,173 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import TYPE_CHECKING + +from twisted.web import resource + +from hathor.api_util import get_missing_params_msg, parse_get_arguments, set_cors +from hathor.cli.openapi_files.register import register_resource + +if TYPE_CHECKING: + from twisted.web.http import Request + + from hathor.manager import HathorManager + + +@register_resource +class BlockAtHeightResource(resource.Resource): + """ Implements a web server API to return the block at specific height. + + You must run with option `--status `. + """ + isLeaf = True + + def __init__(self, manager: 'HathorManager'): + # Important to have the manager so we can know the tx_storage + self.manager = manager + + def render_GET(self, request: 'Request') -> bytes: + """ Get request /block_at_height/ that returns a block at height in parameter + + 'height': int, the height of block to get + + :rtype: string (json) + """ + request.setHeader(b'content-type', b'application/json; charset=utf-8') + set_cors(request, 'GET') + + # Height parameter is required + parsed = parse_get_arguments(request.args, ['height']) + if not parsed['success']: + return get_missing_params_msg(parsed['missing']) + + args = parsed['args'] + + # Height parameter must be an integer + try: + height = int(args['height']) + except ValueError: + return json.dumps({ + 'success': False, + 'message': 'Invalid \'height\' parameter, expected an integer' + }).encode('utf-8') + + # Get hash of the block with the height + block_hash = self.manager.tx_storage.get_from_block_height_index(height) + + # If there is no block in the index with this height, block_hash will be None + if block_hash is None: + return json.dumps({ + 'success': False, + 'message': 'No block with height {}.'.format(height) + }).encode('utf-8') + + block = self.manager.tx_storage.get_transaction(block_hash) + + data = {'success': True, 'block': block.to_json_extended()} + return json.dumps(data, indent=4).encode('utf-8') + + +BlockAtHeightResource.openapi = { + '/block_at_height': { + 'x-visibility': 'public', + 'x-rate-limit': { + 'global': [ + { + 'rate': '50r/s', + 'burst': 100, + 'delay': 50 + } + ], + 'per-ip': [ + { + 'rate': '3r/s', + 'burst': 10, + 'delay': 3 + } + ] + }, + 'get': { + 'tags': ['block'], + 'operationId': 'block', + 'summary': 'Get block at height', + 'description': 'Returns the block at specific height in the best chain.', + 'parameters': [ + { + 'name': 'height', + 'in': 'query', + 'description': 'Height of the block to get', + 'required': True, + 'schema': { + 'type': 'int' + } + }, + ], + 'responses': { + '200': { + 'description': 'Success', + 'content': { + 'application/json': { + 'examples': { + 'success': { + 'summary': 'Success block height 1', + 'value': { + 'success': True, + 'block': { + 'tx_id': ('080c8086376ab7105d17df1127a68ede' + 'df54029a21b5d98841448cc23b5123ff'), + 'version': 0, + 'weight': 1.0, + 'timestamp': 1616094323, + 'is_voided': False, + 'inputs': [], + 'outputs': [ + { + 'value': 6400, + 'token_data': 0, + 'script': 'dqkU4yipgEZjbphR/M3gUGjsbyb1s76IrA==', + 'decoded': { + 'type': 'P2PKH', + 'address': 'HTEEV9FJeqBCYLUvkEHsWAAi6UGs9yxJKj', + 'timelock': None + }, + 'token': '00', + 'spent_by': None + } + ], + 'parents': [ + '339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', + '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', + '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869' + ], + 'height': 1 + } + } + }, + 'error': { + 'summary': 'Block not found', + 'value': { + 'success': False, + 'message': 'Does not have a block with height 100.' + } + }, + } + } + } + } + } + } + } +} diff --git a/tests/resources/transaction/test_block_at_height.py b/tests/resources/transaction/test_block_at_height.py new file mode 100644 index 0000000000..73e82c2990 --- /dev/null +++ b/tests/resources/transaction/test_block_at_height.py @@ -0,0 +1,50 @@ +from twisted.internet.defer import inlineCallbacks + +from hathor.transaction.resources import BlockAtHeightResource +from tests.resources.base_resource import StubSite, _BaseResourceTest +from tests.utils import add_new_blocks + + +class BlockAtHeightTest(_BaseResourceTest._ResourceTest): + def setUp(self): + super().setUp() + self.web = StubSite(BlockAtHeightResource(self.manager)) + self.manager.wallet.unlock(b'MYPASS') + + @inlineCallbacks + def test_get(self): + blocks = add_new_blocks(self.manager, 4, advance_clock=1) + + # Error1: No parameter + response1 = yield self.web.get("block_at_height") + data1 = response1.json_value() + self.assertFalse(data1['success']) + + # Error2: Invalid parameter + response2 = yield self.web.get("block_at_height", {b'height': b'c'}) + data2 = response2.json_value() + self.assertFalse(data2['success']) + + # Success genesis + genesis_block = next(x for x in self.manager.tx_storage.get_all_genesis() if x.is_block) + response3 = yield self.web.get("block_at_height", {b'height': b'0'}) + data3 = response3.json_value() + self.assertTrue(data3['success']) + self.assertEqual(data3['block']['tx_id'], genesis_block.hash.hex()) + + # Success height 1 + response4 = yield self.web.get("block_at_height", {b'height': b'1'}) + data4 = response4.json_value() + self.assertTrue(data4['success']) + self.assertEqual(data4['block']['tx_id'], blocks[0].hash.hex()) + + # Success height 5 + response5 = yield self.web.get("block_at_height", {b'height': b'4'}) + data5 = response5.json_value() + self.assertTrue(data5['success']) + self.assertEqual(data5['block']['tx_id'], blocks[3].hash.hex()) + + # Error 3: height 5 (does not have this block) + response6 = yield self.web.get("block_at_height", {b'height': b'5'}) + data6 = response6.json_value() + self.assertFalse(data6['success']) From 278c4b7593d82942d8022697d4c90502b7df8c87 Mon Sep 17 00:00:00 2001 From: Jan Segre Date: Tue, 22 Jun 2021 17:34:16 -0300 Subject: [PATCH 14/15] chore: bump version --- hathor/cli/openapi_files/openapi_base.json | 2 +- hathor/version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hathor/cli/openapi_files/openapi_base.json b/hathor/cli/openapi_files/openapi_base.json index 02f00e1798..52db6957fa 100644 --- a/hathor/cli/openapi_files/openapi_base.json +++ b/hathor/cli/openapi_files/openapi_base.json @@ -7,7 +7,7 @@ ], "info": { "title": "Hathor API", - "version": "0.39.1" + "version": "0.40.0" }, "consumes": [ "application/json" diff --git a/hathor/version.py b/hathor/version.py index 44e5c7466d..9cfcb4b4d9 100644 --- a/hathor/version.py +++ b/hathor/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.39.1' +__version__ = '0.40.0' diff --git a/pyproject.toml b/pyproject.toml index 482e854b22..97cd33591b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ [tool.poetry] name = "hathor" -version = "0.39.1" +version = "0.40.0" description = "Hathor Network full-node" authors = ["Hathor Team "] license = "Apache-2.0" From 5252212e276395238c1ef697ed09cb505347f9f4 Mon Sep 17 00:00:00 2001 From: Luis Helder Date: Wed, 23 Jun 2021 15:09:45 -0300 Subject: [PATCH 15/15] chore(ci): verify that the version on different files are the same --- Makefile | 6 +++++- extras/check_version.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100755 extras/check_version.sh diff --git a/Makefile b/Makefile index ea5ba19635..ed704d6d83 100644 --- a/Makefile +++ b/Makefile @@ -62,8 +62,12 @@ flake8: isort-check: isort --ac --check-only $(py_sources) +.PHONY: check-version +check-version: + bash ./extras/check_version.sh + .PHONY: check -check: flake8 isort-check mypy +check: flake8 isort-check mypy check-version # formatting: diff --git a/extras/check_version.sh b/extras/check_version.sh new file mode 100755 index 0000000000..f972fd5007 --- /dev/null +++ b/extras/check_version.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +OPENAPI_FILE="hathor/cli/openapi_files/openapi_base.json" +SRC_FILE="hathor/version.py" +PACKAGE_FILE="pyproject.toml" + +OPENAPI_VERSION=`grep "version\":" ${OPENAPI_FILE} | cut -d'"' -f4` +SRC_VERSION=`grep "__version__" ${SRC_FILE} | cut -d "'" -f2` +PACKAGE_VERSION=`grep '^version' ${PACKAGE_FILE} | cut -d '"' -f2` + +# For debugging: +# echo x${SRC_VERSION}x +# echo x${OPENAPI_VERSION}x +# echo x${PACKAGE_VERSION}x + +EXITCODE=0 + +if [[ x${PACKAGE_VERSION}x != x${SRC_VERSION}x ]]; then + echo "Version different in ${PACKAGE_FILE} and ${SRC_FILE}" + EXITCODE=-1 +fi + +if [[ x${PACKAGE_VERSION}x != x${OPENAPI_VERSION}x ]]; then + echo "Version different in ${PACKAGE_FILE} and ${OPENAPI_FILE}" + EXITCODE=-1 +fi + +if [[ ${EXITCODE} == 0 ]]; then + echo OK +fi +exit $EXITCODE