From 8f2d608fb2d90beecb62399ae1e99bd64d4e224b Mon Sep 17 00:00:00 2001 From: Gabriel Levcovitz Date: Fri, 29 Sep 2023 15:16:06 -0300 Subject: [PATCH 1/2] refactor(verification): create vertex verifiers --- hathor/builder/builder.py | 18 +++++- hathor/builder/cli_builder.py | 5 +- hathor/verification/block_verification.py | 47 -------------- hathor/verification/block_verifier.py | 50 +++++++++++++++ .../merge_mined_block_verifier.py | 19 ++++++ ...=> token_creation_transaction_verifier.py} | 17 ++--- .../verification/transaction_verification.py | 53 ---------------- hathor/verification/transaction_verifier.py | 56 +++++++++++++++++ hathor/verification/verification_service.py | 62 +++++++++++++++---- hathor/verification/vertex_verifier.py | 22 +++++++ 10 files changed, 227 insertions(+), 122 deletions(-) delete mode 100644 hathor/verification/block_verification.py create mode 100644 hathor/verification/block_verifier.py create mode 100644 hathor/verification/merge_mined_block_verifier.py rename hathor/verification/{token_creation_transaction_verification.py => token_creation_transaction_verifier.py} (55%) delete mode 100644 hathor/verification/transaction_verification.py create mode 100644 hathor/verification/transaction_verifier.py create mode 100644 hathor/verification/vertex_verifier.py diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index ecb2bd02c..c67d20d63 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -41,7 +41,7 @@ TransactionStorage, ) from hathor.util import Random, Reactor, get_environment_info -from hathor.verification.verification_service import VerificationService +from hathor.verification.verification_service import VerificationService, VertexVerifiers from hathor.wallet import BaseWallet, Wallet logger = get_logger() @@ -102,6 +102,7 @@ def __init__(self) -> None: self._feature_service: Optional[FeatureService] = None self._bit_signaling_service: Optional[BitSignalingService] = None + self._vertex_verifiers: Optional[VertexVerifiers] = None self._verification_service: Optional[VerificationService] = None self._rocksdb_path: Optional[str] = None @@ -432,10 +433,18 @@ def _get_or_create_bit_signaling_service(self, tx_storage: TransactionStorage) - def _get_or_create_verification_service(self) -> VerificationService: if self._verification_service is None: - self._verification_service = VerificationService() + verifiers = self._get_or_create_vertex_verifiers() + self._verification_service = VerificationService(verifiers=verifiers) return self._verification_service + def _get_or_create_vertex_verifiers(self) -> VertexVerifiers: + if self._vertex_verifiers is None: + settings = self._get_or_create_settings() + self._vertex_verifiers = VertexVerifiers.create(settings=settings) + + return self._vertex_verifiers + def use_memory(self) -> 'Builder': self.check_if_can_modify() self._storage_type = StorageType.MEMORY @@ -533,6 +542,11 @@ def set_verification_service(self, verification_service: VerificationService) -> self._verification_service = verification_service return self + def set_vertex_verifiers(self, vertex_verifiers: VertexVerifiers) -> 'Builder': + self.check_if_can_modify() + self._vertex_verifiers = vertex_verifiers + return self + def set_reactor(self, reactor: Reactor) -> 'Builder': self.check_if_can_modify() self._reactor = reactor diff --git a/hathor/builder/cli_builder.py b/hathor/builder/cli_builder.py index d49928f26..441c95855 100644 --- a/hathor/builder/cli_builder.py +++ b/hathor/builder/cli_builder.py @@ -35,7 +35,7 @@ from hathor.pubsub import PubSubManager from hathor.stratum import StratumFactory from hathor.util import Random, Reactor -from hathor.verification.verification_service import VerificationService +from hathor.verification.verification_service import VerificationService, VertexVerifiers from hathor.wallet import BaseWallet, HDWallet, Wallet logger = get_logger() @@ -202,7 +202,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager: not_support_features=self._args.signal_not_support ) - verification_service = VerificationService() + vertex_verifiers = VertexVerifiers.create(settings=settings) + verification_service = VerificationService(verifiers=vertex_verifiers) p2p_manager = ConnectionsManager( reactor, diff --git a/hathor/verification/block_verification.py b/hathor/verification/block_verification.py deleted file mode 100644 index 3e47aa254..000000000 --- a/hathor/verification/block_verification.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from hathor.profiler import get_cpu_profiler -from hathor.transaction import Block - -cpu = get_cpu_profiler() - - -def verify_basic(block: Block, *, skip_block_weight_verification: bool = False) -> None: - """Partially run validations, the ones that need parents/inputs are skipped.""" - if not skip_block_weight_verification: - block.verify_weight() - block.verify_reward() - - -@cpu.profiler(key=lambda block: 'block-verify!{}'.format(block.hash.hex())) -def verify(block: Block) -> None: - """ - (1) confirms at least two pending transactions and references last block - (2) solves the pow with the correct weight (done in HathorManager) - (3) creates the correct amount of tokens in the output (done in HathorManager) - (4) all parents must exist and have timestamp smaller than ours - (5) data field must contain at most BLOCK_DATA_MAX_SIZE bytes - """ - # TODO Should we validate a limit of outputs? - if block.is_genesis: - # TODO do genesis validation - return - - block.verify_without_storage() - - # (1) and (4) - block.verify_parents() - - block.verify_height() diff --git a/hathor/verification/block_verifier.py b/hathor/verification/block_verifier.py new file mode 100644 index 000000000..dd8903f72 --- /dev/null +++ b/hathor/verification/block_verifier.py @@ -0,0 +1,50 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.profiler import get_cpu_profiler +from hathor.transaction import Block +from hathor.verification.vertex_verifier import VertexVerifier + +cpu = get_cpu_profiler() + + +class BlockVerifier(VertexVerifier): + __slots__ = () + + def verify_basic(self, block: Block, *, skip_block_weight_verification: bool = False) -> None: + """Partially run validations, the ones that need parents/inputs are skipped.""" + if not skip_block_weight_verification: + block.verify_weight() + block.verify_reward() + + @cpu.profiler(key=lambda _, block: 'block-verify!{}'.format(block.hash.hex())) + def verify(self, block: Block) -> None: + """ + (1) confirms at least two pending transactions and references last block + (2) solves the pow with the correct weight (done in HathorManager) + (3) creates the correct amount of tokens in the output (done in HathorManager) + (4) all parents must exist and have timestamp smaller than ours + (5) data field must contain at most BLOCK_DATA_MAX_SIZE bytes + """ + # TODO Should we validate a limit of outputs? + if block.is_genesis: + # TODO do genesis validation + return + + block.verify_without_storage() + + # (1) and (4) + block.verify_parents() + + block.verify_height() diff --git a/hathor/verification/merge_mined_block_verifier.py b/hathor/verification/merge_mined_block_verifier.py new file mode 100644 index 000000000..efbfc4c07 --- /dev/null +++ b/hathor/verification/merge_mined_block_verifier.py @@ -0,0 +1,19 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.verification.block_verifier import BlockVerifier + + +class MergeMinedBlockVerifier(BlockVerifier): + __slots__ = () diff --git a/hathor/verification/token_creation_transaction_verification.py b/hathor/verification/token_creation_transaction_verifier.py similarity index 55% rename from hathor/verification/token_creation_transaction_verification.py rename to hathor/verification/token_creation_transaction_verifier.py index b1d9622b2..fee1bec0e 100644 --- a/hathor/verification/token_creation_transaction_verification.py +++ b/hathor/verification/token_creation_transaction_verifier.py @@ -13,13 +13,16 @@ # limitations under the License. from hathor.transaction.token_creation_tx import TokenCreationTransaction -from hathor.verification import transaction_verification +from hathor.verification.transaction_verifier import TransactionVerifier -def verify(tx: TokenCreationTransaction, *, reject_locked_reward: bool = True) -> None: - """ Run all validations as regular transactions plus validation on token info. +class TokenCreationTransactionVerifier(TransactionVerifier): + __slots__ = () - We also overload verify_sum to make some different checks - """ - transaction_verification.verify(tx, reject_locked_reward=reject_locked_reward) - tx.verify_token_info() + def verify(self, tx: TokenCreationTransaction, *, reject_locked_reward: bool = True) -> None: + """ Run all validations as regular transactions plus validation on token info. + + We also overload verify_sum to make some different checks + """ + super().verify(tx, reject_locked_reward=reject_locked_reward) + tx.verify_token_info() diff --git a/hathor/verification/transaction_verification.py b/hathor/verification/transaction_verification.py deleted file mode 100644 index 02d887a10..000000000 --- a/hathor/verification/transaction_verification.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2023 Hathor Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from hathor.profiler import get_cpu_profiler -from hathor.transaction import Transaction - -cpu = get_cpu_profiler() - - -def verify_basic(transaction: Transaction) -> None: - """Partially run validations, the ones that need parents/inputs are skipped.""" - if transaction.is_genesis: - # TODO do genesis validation? - return - transaction.verify_parents_basic() - transaction.verify_weight() - transaction.verify_without_storage() - - -@cpu.profiler(key=lambda tx: 'tx-verify!{}'.format(tx.hash.hex())) -def verify(tx: Transaction, *, reject_locked_reward: bool = True) -> None: - """ Common verification for all transactions: - (i) number of inputs is at most 256 - (ii) number of outputs is at most 256 - (iii) confirms at least two pending transactions - (iv) solves the pow (we verify weight is correct in HathorManager) - (v) validates signature of inputs - (vi) validates public key and output (of the inputs) addresses - (vii) validate that both parents are valid - (viii) validate input's timestamps - (ix) validate inputs and outputs sum - """ - if tx.is_genesis: - # TODO do genesis validation - return - tx.verify_without_storage() - tx.verify_sigops_input() - tx.verify_inputs() # need to run verify_inputs first to check if all inputs exist - tx.verify_parents() - tx.verify_sum() - if reject_locked_reward: - tx.verify_reward_locked() diff --git a/hathor/verification/transaction_verifier.py b/hathor/verification/transaction_verifier.py new file mode 100644 index 000000000..8c3711524 --- /dev/null +++ b/hathor/verification/transaction_verifier.py @@ -0,0 +1,56 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.profiler import get_cpu_profiler +from hathor.transaction import Transaction +from hathor.verification.vertex_verifier import VertexVerifier + +cpu = get_cpu_profiler() + + +class TransactionVerifier(VertexVerifier): + __slots__ = () + + def verify_basic(self, tx: Transaction) -> None: + """Partially run validations, the ones that need parents/inputs are skipped.""" + if tx.is_genesis: + # TODO do genesis validation? + return + tx.verify_parents_basic() + tx.verify_weight() + tx.verify_without_storage() + + @cpu.profiler(key=lambda _, tx: 'tx-verify!{}'.format(tx.hash.hex())) + def verify(self, tx: Transaction, *, reject_locked_reward: bool = True) -> None: + """ Common verification for all transactions: + (i) number of inputs is at most 256 + (ii) number of outputs is at most 256 + (iii) confirms at least two pending transactions + (iv) solves the pow (we verify weight is correct in HathorManager) + (v) validates signature of inputs + (vi) validates public key and output (of the inputs) addresses + (vii) validate that both parents are valid + (viii) validate input's timestamps + (ix) validate inputs and outputs sum + """ + if tx.is_genesis: + # TODO do genesis validation + return + tx.verify_without_storage() + tx.verify_sigops_input() + tx.verify_inputs() # need to run verify_inputs first to check if all inputs exist + tx.verify_parents() + tx.verify_sum() + if reject_locked_reward: + tx.verify_reward_locked() diff --git a/hathor/verification/verification_service.py b/hathor/verification/verification_service.py index 2a98cb662..ee6127234 100644 --- a/hathor/verification/verification_service.py +++ b/hathor/verification/verification_service.py @@ -12,15 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -from hathor.transaction import BaseTransaction, Block, Transaction, TxVersion +from typing import NamedTuple + +from hathor.conf.settings import HathorSettings +from hathor.transaction import BaseTransaction, Block, MergeMinedBlock, Transaction, TxVersion from hathor.transaction.exceptions import TxValidationError from hathor.transaction.token_creation_tx import TokenCreationTransaction from hathor.transaction.validation_state import ValidationState -from hathor.verification import block_verification, token_creation_transaction_verification, transaction_verification +from hathor.verification.block_verifier import BlockVerifier +from hathor.verification.merge_mined_block_verifier import MergeMinedBlockVerifier +from hathor.verification.token_creation_transaction_verifier import TokenCreationTransactionVerifier +from hathor.verification.transaction_verifier import TransactionVerifier + + +class VertexVerifiers(NamedTuple): + block: BlockVerifier + merge_mined_block: MergeMinedBlockVerifier + tx: TransactionVerifier + token_creation_tx: TokenCreationTransactionVerifier + + @classmethod + def create(cls, *, settings: HathorSettings) -> 'VertexVerifiers': + return VertexVerifiers( + block=BlockVerifier(settings=settings), + merge_mined_block=MergeMinedBlockVerifier(settings=settings), + tx=TransactionVerifier(settings=settings), + token_creation_tx=TokenCreationTransactionVerifier(settings=settings), + ) class VerificationService: - __slots__ = () + __slots__ = ('verifiers', ) + + def __init__(self, *, verifiers: VertexVerifiers) -> None: + self.verifiers = verifiers def validate_basic(self, vertex: BaseTransaction, *, skip_block_weight_verification: bool = False) -> bool: """ Run basic validations (all that are possible without dependencies) and update the validation state. @@ -70,12 +95,24 @@ def verify_basic(self, vertex: BaseTransaction, *, skip_block_weight_verificatio Used by `self.validate_basic`. Should not modify the validation state.""" match vertex.version: - case TxVersion.REGULAR_BLOCK | TxVersion.MERGE_MINED_BLOCK: + case TxVersion.REGULAR_BLOCK: assert isinstance(vertex, Block) - block_verification.verify_basic(vertex, skip_block_weight_verification=skip_block_weight_verification) - case TxVersion.REGULAR_TRANSACTION | TxVersion.TOKEN_CREATION_TRANSACTION: + self.verifiers.block.verify_basic( + vertex, + skip_block_weight_verification=skip_block_weight_verification + ) + case TxVersion.MERGE_MINED_BLOCK: + assert isinstance(vertex, MergeMinedBlock) + self.verifiers.merge_mined_block.verify_basic( + vertex, + skip_block_weight_verification=skip_block_weight_verification + ) + case TxVersion.REGULAR_TRANSACTION: assert isinstance(vertex, Transaction) - transaction_verification.verify_basic(vertex) + self.verifiers.tx.verify_basic(vertex) + case TxVersion.TOKEN_CREATION_TRANSACTION: + assert isinstance(vertex, TokenCreationTransaction) + self.verifiers.token_creation_tx.verify_basic(vertex) case _: raise NotImplementedError @@ -84,15 +121,18 @@ def verify(self, vertex: BaseTransaction, *, reject_locked_reward: bool = True) Used by `self.validate_full`. Should not modify the validation state.""" match vertex.version: - case TxVersion.REGULAR_BLOCK | TxVersion.MERGE_MINED_BLOCK: + case TxVersion.REGULAR_BLOCK: assert isinstance(vertex, Block) - block_verification.verify(vertex) + self.verifiers.block.verify(vertex) + case TxVersion.MERGE_MINED_BLOCK: + assert isinstance(vertex, MergeMinedBlock) + self.verifiers.merge_mined_block.verify(vertex) case TxVersion.REGULAR_TRANSACTION: assert isinstance(vertex, Transaction) - transaction_verification.verify(vertex, reject_locked_reward=reject_locked_reward) + self.verifiers.tx.verify(vertex, reject_locked_reward=reject_locked_reward) case TxVersion.TOKEN_CREATION_TRANSACTION: assert isinstance(vertex, TokenCreationTransaction) - token_creation_transaction_verification.verify(vertex, reject_locked_reward=reject_locked_reward) + self.verifiers.token_creation_tx.verify(vertex, reject_locked_reward=reject_locked_reward) case _: raise NotImplementedError diff --git a/hathor/verification/vertex_verifier.py b/hathor/verification/vertex_verifier.py new file mode 100644 index 000000000..360450116 --- /dev/null +++ b/hathor/verification/vertex_verifier.py @@ -0,0 +1,22 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.conf.settings import HathorSettings + + +class VertexVerifier: + __slots__ = ('_settings', ) + + def __init__(self, *, settings: HathorSettings): + self._settings = settings From 13398a63843893f45524c6025c1d5fab06b30865 Mon Sep 17 00:00:00 2001 From: Gabriel Levcovitz Date: Fri, 29 Sep 2023 15:33:15 -0300 Subject: [PATCH 2/2] refactor(verification): move verification methods signatures --- hathor/cli/mining.py | 7 +- hathor/stratum/stratum.py | 5 +- hathor/transaction/resources/create_tx.py | 2 +- hathor/verification/block_verifier.py | 39 +++++++- .../merge_mined_block_verifier.py | 7 ++ .../token_creation_transaction_verifier.py | 5 + hathor/verification/transaction_verifier.py | 92 +++++++++++++++++-- hathor/verification/verification_service.py | 17 ++++ hathor/verification/vertex_verifier.py | 42 +++++++++ tests/simulation/test_simulator.py | 3 +- tests/tx/test_genesis.py | 8 +- tests/tx/test_tx.py | 56 +++++------ tests/tx/test_tx_deserialization.py | 8 +- tests/wallet/test_wallet_hd.py | 5 +- 14 files changed, 246 insertions(+), 50 deletions(-) diff --git a/hathor/cli/mining.py b/hathor/cli/mining.py index 9a373be90..df99b2913 100644 --- a/hathor/cli/mining.py +++ b/hathor/cli/mining.py @@ -24,6 +24,9 @@ import requests +from hathor.conf.get_settings import get_settings +from hathor.verification.block_verifier import BlockVerifier + _SLEEP_ON_ERROR_SECONDS = 5 _MAX_CONN_RETRIES = math.inf @@ -134,7 +137,9 @@ def execute(args: Namespace) -> None: block.nonce, block.weight)) try: - block.verify_without_storage() + settings = get_settings() + verifier = BlockVerifier(settings=settings) + verifier.verify_without_storage(block) except HathorError: print('[{}] ERROR: Block has not been pushed because it is not valid.'.format(datetime.datetime.now())) else: diff --git a/hathor/stratum/stratum.py b/hathor/stratum/stratum.py index 6abc2dfbd..16b278661 100644 --- a/hathor/stratum/stratum.py +++ b/hathor/stratum/stratum.py @@ -41,6 +41,7 @@ from hathor.transaction import BaseTransaction, BitcoinAuxPow, Block, MergeMinedBlock, Transaction, sum_weights from hathor.transaction.exceptions import PowError, ScriptError, TxValidationError from hathor.util import Reactor, json_dumpb, json_loadb, reactor +from hathor.verification.vertex_verifier import VertexVerifier from hathor.wallet.exceptions import InvalidAddress if TYPE_CHECKING: @@ -526,7 +527,7 @@ def handle_submit(self, params: dict, msgid: Optional[str]) -> None: self.log.debug('share received', block=tx, block_base=block_base.hex(), block_base_hash=block_base_hash.hex()) try: - tx.verify_pow(job.weight) + VertexVerifier.verify_pow(tx, override_weight=job.weight) except PowError: self.log.error('bad share, discard', job_weight=job.weight, tx=tx) return self.send_error(INVALID_SOLUTION, msgid, { @@ -542,7 +543,7 @@ def handle_submit(self, params: dict, msgid: Optional[str]) -> None: self.manager.reactor.callLater(0, self.job_request) try: - tx.verify_pow() + VertexVerifier.verify_pow(tx) except PowError: # Transaction pow was not enough, but the share was succesfully submited self.log.info('high hash, keep mining', tx=tx) diff --git a/hathor/transaction/resources/create_tx.py b/hathor/transaction/resources/create_tx.py index 438d1f23d..dcec5d363 100644 --- a/hathor/transaction/resources/create_tx.py +++ b/hathor/transaction/resources/create_tx.py @@ -89,7 +89,7 @@ def render_POST(self, request): # conservative estimate of the input data size to estimate a valid weight tx_input.data = b'\0' * 107 tx.weight = minimum_tx_weight(fake_signed_tx) - tx.verify_unsigned_skip_pow() + self.manager.verification_service.verifiers.tx.verify_unsigned_skip_pow(tx) if tx.is_double_spending(): raise InvalidNewTransaction('At least one of your inputs has already been spent.') diff --git a/hathor/verification/block_verifier.py b/hathor/verification/block_verifier.py index dd8903f72..098e1014d 100644 --- a/hathor/verification/block_verifier.py +++ b/hathor/verification/block_verifier.py @@ -13,7 +13,7 @@ # limitations under the License. from hathor.profiler import get_cpu_profiler -from hathor.transaction import Block +from hathor.transaction import BaseTransaction, Block from hathor.verification.vertex_verifier import VertexVerifier cpu = get_cpu_profiler() @@ -25,8 +25,8 @@ class BlockVerifier(VertexVerifier): def verify_basic(self, block: Block, *, skip_block_weight_verification: bool = False) -> None: """Partially run validations, the ones that need parents/inputs are skipped.""" if not skip_block_weight_verification: - block.verify_weight() - block.verify_reward() + self.verify_weight(block) + self.verify_reward(block) @cpu.profiler(key=lambda _, block: 'block-verify!{}'.format(block.hash.hex())) def verify(self, block: Block) -> None: @@ -42,9 +42,38 @@ def verify(self, block: Block) -> None: # TODO do genesis validation return - block.verify_without_storage() + self.verify_without_storage(block) # (1) and (4) - block.verify_parents() + self.verify_parents(block) + + self.verify_height(block) + + def verify_without_storage(self, block: Block) -> None: + """ Run all verifications that do not need a storage. + """ + block.verify_without_storage() + @staticmethod + def verify_height(block: Block) -> None: + """Validate that the block height is enough to confirm all transactions being confirmed.""" block.verify_height() + + def verify_weight(self, block: Block) -> None: + """Validate minimum block difficulty.""" + block.verify_weight() + + @staticmethod + def verify_reward(block: Block) -> None: + """Validate reward amount.""" + block.verify_reward() + + @staticmethod + def verify_no_inputs(block: Block) -> None: + block.verify_no_inputs() + + def verify_outputs(self, block: BaseTransaction) -> None: + block.verify_outputs() + + def verify_data(self, block: Block) -> None: + block.verify_data() diff --git a/hathor/verification/merge_mined_block_verifier.py b/hathor/verification/merge_mined_block_verifier.py index efbfc4c07..74e488452 100644 --- a/hathor/verification/merge_mined_block_verifier.py +++ b/hathor/verification/merge_mined_block_verifier.py @@ -12,8 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from hathor.transaction import MergeMinedBlock from hathor.verification.block_verifier import BlockVerifier class MergeMinedBlockVerifier(BlockVerifier): __slots__ = () + + @staticmethod + def verify_aux_pow(block: MergeMinedBlock) -> None: + """ Verify auxiliary proof-of-work (for merged mining). + """ + block.verify_aux_pow() diff --git a/hathor/verification/token_creation_transaction_verifier.py b/hathor/verification/token_creation_transaction_verifier.py index fee1bec0e..b7f8b53d1 100644 --- a/hathor/verification/token_creation_transaction_verifier.py +++ b/hathor/verification/token_creation_transaction_verifier.py @@ -25,4 +25,9 @@ def verify(self, tx: TokenCreationTransaction, *, reject_locked_reward: bool = T We also overload verify_sum to make some different checks """ super().verify(tx, reject_locked_reward=reject_locked_reward) + self.verify_token_info(tx) + + def verify_token_info(self, tx: TokenCreationTransaction) -> None: + """ Validates token info + """ tx.verify_token_info() diff --git a/hathor/verification/transaction_verifier.py b/hathor/verification/transaction_verifier.py index 8c3711524..3aae1655e 100644 --- a/hathor/verification/transaction_verifier.py +++ b/hathor/verification/transaction_verifier.py @@ -13,7 +13,9 @@ # limitations under the License. from hathor.profiler import get_cpu_profiler -from hathor.transaction import Transaction +from hathor.transaction import BaseTransaction, Transaction, TxInput +from hathor.transaction.transaction import TokenInfo +from hathor.types import TokenUid from hathor.verification.vertex_verifier import VertexVerifier cpu = get_cpu_profiler() @@ -27,9 +29,9 @@ def verify_basic(self, tx: Transaction) -> None: if tx.is_genesis: # TODO do genesis validation? return - tx.verify_parents_basic() - tx.verify_weight() - tx.verify_without_storage() + self.verify_parents_basic(tx) + self.verify_weight(tx) + self.verify_without_storage(tx) @cpu.profiler(key=lambda _, tx: 'tx-verify!{}'.format(tx.hash.hex())) def verify(self, tx: Transaction, *, reject_locked_reward: bool = True) -> None: @@ -47,10 +49,84 @@ def verify(self, tx: Transaction, *, reject_locked_reward: bool = True) -> None: if tx.is_genesis: # TODO do genesis validation return + self.verify_without_storage(tx) + self.verify_sigops_input(tx) + self.verify_inputs(tx) # need to run verify_inputs first to check if all inputs exist + self.verify_parents(tx) + self.verify_sum(tx) + if reject_locked_reward: + self.verify_reward_locked(tx) + + def verify_unsigned_skip_pow(self, tx: Transaction) -> None: + """ Same as .verify but skipping pow and signature verification.""" + tx.verify_unsigned_skip_pow() + + @staticmethod + def verify_parents_basic(tx: Transaction) -> None: + """Verify number and non-duplicity of parents.""" + tx.verify_parents_basic() + + def verify_weight(self, tx: Transaction) -> None: + """Validate minimum tx difficulty.""" + tx.verify_weight() + + def verify_without_storage(self, tx: Transaction) -> None: + """ Run all verifications that do not need a storage. + """ tx.verify_without_storage() + + def verify_sigops_input(self, tx: Transaction) -> None: + """ Count sig operations on all inputs and verify that the total sum is below the limit + """ tx.verify_sigops_input() - tx.verify_inputs() # need to run verify_inputs first to check if all inputs exist - tx.verify_parents() + + def verify_inputs(self, tx: Transaction, *, skip_script: bool = False) -> None: + """Verify inputs signatures and ownership and all inputs actually exist""" + tx.verify_inputs(skip_script=skip_script) + + @staticmethod + def verify_script(*, tx: Transaction, input_tx: TxInput, spent_tx: BaseTransaction) -> None: + """ + :type tx: Transaction + :type input_tx: TxInput + :type spent_tx: Transaction + """ + tx.verify_script(input_tx, spent_tx) + + def verify_sum(self, tx: Transaction) -> None: + """Verify that the sum of outputs is equal of the sum of inputs, for each token. + + If there are authority UTXOs involved, tokens can be minted or melted, so the above rule may + not be respected. + + :raises InvalidToken: when there's an error in token operations + :raises InputOutputMismatch: if sum of inputs is not equal to outputs and there's no mint/melt + """ tx.verify_sum() - if reject_locked_reward: - tx.verify_reward_locked() + + @staticmethod + def verify_reward_locked(tx: Transaction) -> None: + """Will raise `RewardLocked` if any reward is spent before the best block height is enough, considering only + the block rewards spent by this tx itself, and not the inherited `min_height`.""" + tx.verify_reward_locked() + + def verify_number_of_inputs(self, tx: Transaction) -> None: + """Verify number of inputs is in a valid range""" + tx.verify_number_of_inputs() + + def verify_outputs(self, tx: BaseTransaction) -> None: + """Verify outputs reference an existing token uid in the tokens list + + :raises InvalidToken: output references non existent token uid + """ + tx.verify_outputs() + + @staticmethod + def update_token_info_from_outputs(tx: Transaction, *, token_dict: dict[TokenUid, TokenInfo]) -> None: + """Iterate over the outputs and add values to token info dict. Updates the dict in-place. + + Also, checks if no token has authorities on the outputs not present on the inputs + + :raises InvalidToken: when there's an error in token operations + """ + tx.update_token_info_from_outputs(token_dict) diff --git a/hathor/verification/verification_service.py b/hathor/verification/verification_service.py index ee6127234..f696e593d 100644 --- a/hathor/verification/verification_service.py +++ b/hathor/verification/verification_service.py @@ -136,6 +136,23 @@ def verify(self, vertex: BaseTransaction, *, reject_locked_reward: bool = True) case _: raise NotImplementedError + def verify_without_storage(self, vertex: BaseTransaction) -> None: + match vertex.version: + case TxVersion.REGULAR_BLOCK: + assert isinstance(vertex, Block) + self.verifiers.block.verify_without_storage(vertex) + case TxVersion.MERGE_MINED_BLOCK: + assert isinstance(vertex, MergeMinedBlock) + self.verifiers.merge_mined_block.verify_without_storage(vertex) + case TxVersion.REGULAR_TRANSACTION: + assert isinstance(vertex, Transaction) + self.verifiers.tx.verify_without_storage(vertex) + case TxVersion.TOKEN_CREATION_TRANSACTION: + assert isinstance(vertex, TokenCreationTransaction) + self.verifiers.token_creation_tx.verify_without_storage(vertex) + case _: + raise NotImplementedError + def validate_vertex_error(self, vertex: BaseTransaction) -> tuple[bool, str]: """ Verify if tx is valid and return success and possible error message diff --git a/hathor/verification/vertex_verifier.py b/hathor/verification/vertex_verifier.py index 360450116..85efa0089 100644 --- a/hathor/verification/vertex_verifier.py +++ b/hathor/verification/vertex_verifier.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional + from hathor.conf.settings import HathorSettings +from hathor.transaction import BaseTransaction class VertexVerifier: @@ -20,3 +23,42 @@ class VertexVerifier: def __init__(self, *, settings: HathorSettings): self._settings = settings + + def verify_parents(self, vertex: BaseTransaction) -> None: + """All parents must exist and their timestamps must be smaller than ours. + + Also, txs should have 2 other txs as parents, while blocks should have 2 txs + 1 block. + + Parents must be ordered with blocks first, followed by transactions. + + :raises TimestampError: when our timestamp is less or equal than our parent's timestamp + :raises ParentDoesNotExist: when at least one of our parents does not exist + :raises IncorrectParents: when tx does not confirm the correct number/type of parent txs + """ + vertex.verify_parents() + + @classmethod + def verify_pow(cls, vertex: BaseTransaction, *, override_weight: Optional[float] = None) -> None: + """Verify proof-of-work + + :raises PowError: when the hash is equal or greater than the target + """ + vertex.verify_pow(override_weight) + + def verify_outputs(self, vertex: BaseTransaction) -> None: + """Verify there are no hathor authority UTXOs and outputs are all positive + + :raises InvalidToken: when there's a hathor authority utxo + :raises InvalidOutputValue: output has negative value + :raises TooManyOutputs: when there are too many outputs + """ + vertex.verify_outputs() + + def verify_number_of_outputs(self, vertex: BaseTransaction) -> None: + """Verify number of outputs does not exceeds the limit""" + vertex.verify_number_of_outputs() + + def verify_sigops_output(self, vertex: BaseTransaction) -> None: + """ Count sig operations on all outputs and verify that the total sum is below the limit + """ + vertex.verify_sigops_output() diff --git a/tests/simulation/test_simulator.py b/tests/simulation/test_simulator.py index cce6c795b..a373af657 100644 --- a/tests/simulation/test_simulator.py +++ b/tests/simulation/test_simulator.py @@ -2,6 +2,7 @@ from hathor.simulator import FakeConnection from hathor.simulator.trigger import All as AllTriggers, StopWhenSynced +from hathor.verification.vertex_verifier import VertexVerifier from tests import unittest from tests.simulation.base import SimulatorTestCase @@ -12,7 +13,7 @@ def test_verify_pow(self): # just get one of the genesis, we don't really need to create any transaction tx = next(iter(manager1.tx_storage.get_all_genesis())) # optional argument must be valid, it just has to not raise any exception, there's no assert for that - tx.verify_pow(0.) + VertexVerifier.verify_pow(tx, override_weight=0.) def test_one_node(self): manager1 = self.create_peer() diff --git a/tests/tx/test_genesis.py b/tests/tx/test_genesis.py index a30759193..1db08b2de 100644 --- a/tests/tx/test_genesis.py +++ b/tests/tx/test_genesis.py @@ -1,6 +1,8 @@ from hathor.conf import HathorSettings from hathor.daa import TestMode, _set_test_mode, calculate_block_difficulty, minimum_tx_weight from hathor.transaction.storage import TransactionMemoryStorage +from hathor.verification.verification_service import VerificationService, VertexVerifiers +from hathor.verification.vertex_verifier import VertexVerifier from tests import unittest settings = HathorSettings() @@ -26,18 +28,20 @@ def get_genesis_output(): class GenesisTest(unittest.TestCase): def setUp(self): super().setUp() + verifiers = VertexVerifiers.create(settings=self._settings) + self._verification_service = VerificationService(verifiers=verifiers) self.storage = TransactionMemoryStorage() def test_pow(self): genesis = self.storage.get_all_genesis() for g in genesis: self.assertEqual(g.calculate_hash(), g.hash) - self.assertIsNone(g.verify_pow()) + self.assertIsNone(VertexVerifier.verify_pow(g)) def test_verify(self): genesis = self.storage.get_all_genesis() for g in genesis: - g.verify_without_storage() + self._verification_service.verify_without_storage(g) def test_output(self): # Test if block output is valid diff --git a/tests/tx/test_tx.py b/tests/tx/test_tx.py index 96cc51ce2..ae932dee2 100644 --- a/tests/tx/test_tx.py +++ b/tests/tx/test_tx.py @@ -30,6 +30,7 @@ from hathor.transaction.scripts import P2PKH, parse_address_script from hathor.transaction.util import int_to_bytes from hathor.transaction.validation_state import ValidationState +from hathor.verification.verification_service import VertexVerifiers from hathor.wallet import Wallet from tests import unittest from tests.utils import ( @@ -46,6 +47,7 @@ class BaseTransactionTest(unittest.TestCase): def setUp(self): super().setUp() + self._verifiers = VertexVerifiers.create(settings=self._settings) self.wallet = Wallet() # this makes sure we can spend the genesis outputs @@ -80,7 +82,7 @@ def test_input_output_match(self): _input.data = P2PKH.create_input_data(public_bytes, signature) with self.assertRaises(InputOutputMismatch): - tx.verify_sum() + self._verifiers.tx.verify_sum(tx) def test_validation(self): # add 100 blocks and check that walking through get_next_block_best_chain yields the same blocks @@ -120,7 +122,7 @@ def test_script(self): _input.data = data_wrong with self.assertRaises(InvalidInputData): - tx.verify_inputs() + self._verifiers.tx.verify_inputs(tx) def test_too_many_inputs(self): random_bytes = bytes.fromhex('0000184e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0fe902') @@ -131,13 +133,13 @@ def test_too_many_inputs(self): tx = Transaction(inputs=inputs, storage=self.tx_storage) with self.assertRaises(TooManyInputs): - tx.verify_number_of_inputs() + self._verifiers.tx.verify_number_of_inputs(tx) def test_no_inputs(self): tx = Transaction(inputs=[], storage=self.tx_storage) with self.assertRaises(NoInputError): - tx.verify_number_of_inputs() + self._verifiers.tx.verify_number_of_inputs(tx) def test_too_many_outputs(self): random_bytes = bytes.fromhex('0000184e64683b966b4268f387c269915cc61f6af5329823a93e3696cb0fe902') @@ -148,7 +150,7 @@ def test_too_many_outputs(self): tx = Transaction(outputs=outputs, storage=self.tx_storage) with self.assertRaises(TooManyOutputs): - tx.verify_number_of_outputs() + self._verifiers.tx.verify_number_of_outputs(tx) def _gen_tx_spending_genesis_block(self): parents = [tx.hash for tx in self.genesis_txs] @@ -246,11 +248,11 @@ def test_merge_mined_no_magic(self): ) with self.assertRaises(AuxPowNoMagicError): - b.verify_aux_pow() + self._verifiers.merge_mined_block.verify_aux_pow(b) # adding the MAGIC_NUMBER makes it work: b.aux_pow = b.aux_pow._replace(coinbase_head=b.aux_pow.coinbase_head + MAGIC_NUMBER) - b.verify_aux_pow() + self._verifiers.merge_mined_block.verify_aux_pow(b) def test_merge_mined_multiple_magic(self): from hathor.merged_mining import MAGIC_NUMBER @@ -312,9 +314,9 @@ def test_merge_mined_multiple_magic(self): assert bytes(b1) != bytes(b2) assert b1.calculate_hash() == b2.calculate_hash() - b1.verify_aux_pow() # OK + self._verifiers.merge_mined_block.verify_aux_pow(b1) # OK with self.assertRaises(AuxPowUnexpectedMagicError): - b2.verify_aux_pow() + self._verifiers.merge_mined_block.verify_aux_pow(b2) def test_merge_mined_long_merkle_path(self): from hathor.merged_mining import MAGIC_NUMBER @@ -341,11 +343,11 @@ def test_merge_mined_long_merkle_path(self): ) with self.assertRaises(AuxPowLongMerklePathError): - b.verify_aux_pow() + self._verifiers.merge_mined_block.verify_aux_pow(b) # removing one path makes it work b.aux_pow.merkle_path.pop() - b.verify_aux_pow() + self._verifiers.merge_mined_block.verify_aux_pow(b) def test_block_outputs(self): from hathor.transaction.exceptions import TooManyOutputs @@ -365,7 +367,7 @@ def test_block_outputs(self): storage=self.tx_storage) with self.assertRaises(TooManyOutputs): - block.verify_outputs() + self._verifiers.block.verify_outputs(block) def test_tx_number_parents(self): genesis_block = self.genesis_blocks[0] @@ -534,7 +536,7 @@ def test_tx_weight_too_high(self): tx.weight += self._settings.MAX_TX_WEIGHT_DIFF + 0.1 tx.update_hash() with self.assertRaises(WeightError): - tx.verify_weight() + self._verifiers.tx.verify_weight(tx) def test_weight_nan(self): # this should succeed @@ -682,34 +684,34 @@ def test_tx_methods(self): self.assertFalse(tx_equal.is_genesis) # Pow error - tx2.verify_pow() + self._verifiers.tx.verify_pow(tx2) tx2.weight = 100 with self.assertRaises(PowError): - tx2.verify_pow() + self._verifiers.tx.verify_pow(tx2) # Verify parent timestamps - tx2.verify_parents() + self._verifiers.tx.verify_parents(tx2) tx2_timestamp = tx2.timestamp tx2.timestamp = 2 with self.assertRaises(TimestampError): - tx2.verify_parents() + self._verifiers.tx.verify_parents(tx2) tx2.timestamp = tx2_timestamp # Verify inputs timestamps - tx2.verify_inputs() + self._verifiers.tx.verify_inputs(tx2) tx2.timestamp = 2 with self.assertRaises(TimestampError): - tx2.verify_inputs() + self._verifiers.tx.verify_inputs(tx2) tx2.timestamp = tx2_timestamp # Validate maximum distance between blocks block = blocks[0] block2 = blocks[1] block2.timestamp = block.timestamp + self._settings.MAX_DISTANCE_BETWEEN_BLOCKS - block2.verify_parents() + self._verifiers.block.verify_parents(block2) block2.timestamp += 1 with self.assertRaises(TimestampError): - block2.verify_parents() + self._verifiers.block.verify_parents(block2) def test_block_big_nonce(self): block = self.genesis_blocks[0] @@ -886,7 +888,7 @@ def _test_txout_script_limit(self, offset): _output = TxOutput(value, script) tx = Transaction(inputs=[_input], outputs=[_output], storage=self.tx_storage) - tx.verify_outputs() + self._verifiers.tx.verify_outputs(tx) def test_txout_script_limit_exceeded(self): with self.assertRaises(InvalidOutputScriptSize): @@ -910,7 +912,7 @@ def _test_txin_data_limit(self, offset): outputs=[_output], storage=self.tx_storage ) - tx.verify_inputs(skip_script=True) + self._verifiers.tx.verify_inputs(tx, skip_script=True) def test_txin_data_limit_exceeded(self): with self.assertRaises(InvalidInputDataSize): @@ -1063,7 +1065,7 @@ def test_sigops_output_single_below_limit(self) -> None: output3 = TxOutput(value, hscript) tx = Transaction(inputs=[_input], outputs=[output3], storage=self.tx_storage) tx.update_hash() - tx.verify_sigops_output() + self._verifiers.tx.verify_sigops_output(tx) def test_sigops_output_multi_below_limit(self) -> None: genesis_block = self.genesis_blocks[0] @@ -1075,7 +1077,7 @@ def test_sigops_output_multi_below_limit(self) -> None: output4 = TxOutput(value, hscript) tx = Transaction(inputs=[_input], outputs=[output4]*num_outputs, storage=self.tx_storage) tx.update_hash() - tx.verify_sigops_output() + self._verifiers.tx.verify_sigops_output(tx) def test_sigops_input_single_above_limit(self) -> None: genesis_block = self.genesis_blocks[0] @@ -1117,7 +1119,7 @@ def test_sigops_input_single_below_limit(self) -> None: input3 = TxInput(genesis_block.hash, 0, hscript) tx = Transaction(inputs=[input3], outputs=[_output], storage=self.tx_storage) tx.update_hash() - tx.verify_sigops_input() + self._verifiers.tx.verify_sigops_input(tx) def test_sigops_input_multi_below_limit(self) -> None: genesis_block = self.genesis_blocks[0] @@ -1131,7 +1133,7 @@ def test_sigops_input_multi_below_limit(self) -> None: input4 = TxInput(genesis_block.hash, 0, hscript) tx = Transaction(inputs=[input4]*num_inputs, outputs=[_output], storage=self.tx_storage) tx.update_hash() - tx.verify_sigops_input() + self._verifiers.tx.verify_sigops_input(tx) def test_compare_bytes_equal(self) -> None: # create some block diff --git a/tests/tx/test_tx_deserialization.py b/tests/tx/test_tx_deserialization.py index 7e15598f3..12f2753f5 100644 --- a/tests/tx/test_tx_deserialization.py +++ b/tests/tx/test_tx_deserialization.py @@ -1,10 +1,16 @@ from hathor.transaction import Block, MergeMinedBlock, Transaction, TxVersion from hathor.transaction.token_creation_tx import TokenCreationTransaction +from hathor.verification.verification_service import VerificationService, VertexVerifiers from tests import unittest class _BaseTest: class _DeserializationTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + verifiers = VertexVerifiers.create(settings=self._settings) + self._verification_service = VerificationService(verifiers=verifiers) + def test_deserialize(self): cls = self.get_tx_class() tx = cls.create_from_struct(self.tx_bytes) @@ -18,7 +24,7 @@ def verbose(key, value): cls = self.get_tx_class() tx = cls.create_from_struct(self.tx_bytes, verbose=verbose) - tx.verify_without_storage() + self._verification_service.verify_without_storage(tx) key, version = v[1] self.assertEqual(key, 'version') diff --git a/tests/wallet/test_wallet_hd.py b/tests/wallet/test_wallet_hd.py index 5c18648cb..fe0676630 100644 --- a/tests/wallet/test_wallet_hd.py +++ b/tests/wallet/test_wallet_hd.py @@ -1,6 +1,7 @@ from hathor.conf import HathorSettings from hathor.crypto.util import decode_address from hathor.transaction import Transaction +from hathor.verification.transaction_verifier import TransactionVerifier from hathor.wallet import HDWallet from hathor.wallet.base_wallet import WalletBalance, WalletInputInfo, WalletOutputInfo from hathor.wallet.exceptions import InsufficientFunds @@ -42,7 +43,7 @@ def test_transaction_and_balance(self): out = WalletOutputInfo(decode_address(new_address2), self.TOKENS, timelock=None) tx1 = self.wallet.prepare_transaction_compute_inputs(Transaction, [out], self.tx_storage) tx1.update_hash() - tx1.verify_script(tx1.inputs[0], block) + TransactionVerifier.verify_script(tx=tx1, input_tx=tx1.inputs[0], spent_tx=block) tx1.storage = self.tx_storage tx1.get_metadata().validation = ValidationState.FULL self.wallet.on_new_tx(tx1) @@ -62,7 +63,7 @@ def test_transaction_and_balance(self): tx2.storage = self.tx_storage tx2.update_hash() tx2.storage = self.tx_storage - tx2.verify_script(tx2.inputs[0], tx1) + TransactionVerifier.verify_script(tx=tx2, input_tx=tx2.inputs[0], spent_tx=tx1) tx2.get_metadata().validation = ValidationState.FULL self.tx_storage.save_transaction(tx2) self.wallet.on_new_tx(tx2)