diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ef6351562..714565cc0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,9 +26,6 @@ jobs: - '3.10' - '3.11' - '3.12' - include: - - python-impl: pypy - python-version: '3.10' steps: - name: Checkout uses: actions/checkout@v3 @@ -96,7 +93,6 @@ jobs: run: docker run --rm ${{ env.TEST_TAG }} quick_test --data / --testnet - name: Build and push uses: docker/build-push-action@v3 - continue-on-error: ${{ matrix.python-impl == 'pypy' }} # PyPy is not first-class and has been causing some build failures if: ${{ !env.ACT }} # Skip this step when testing locally with https://github.com/nektos/act with: context: . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 714e775ef..8e82414ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,10 +26,6 @@ jobs: 'python': ['3.10', '3.11', '3.12'], # available OS's: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on 'os': ['ubuntu-22.04', 'macos-12', 'windows-2022'], - 'include': [ - # XXX: tests fail on these, not sure why, when running them individually each on passes, but not on `make tests` - # {'os': 'ubuntu-22.04', 'python': 'pypy-3.10'}, - ], } # this is the fastest one: reduced_matrix = { diff --git a/Dockerfile.pypy b/Dockerfile.pypy deleted file mode 100644 index 4ad4b1c6d..000000000 --- a/Dockerfile.pypy +++ /dev/null @@ -1,36 +0,0 @@ -# before changing these variables, make sure the tag $PYTHON-alpine$ALPINE exists first -# list of valid tags hese: https://hub.docker.com/_/pypy -ARG PYTHON=3.10 -ARG DEBIAN=bullseye - -# stage-0: copy pyproject.toml/poetry.lock and install the production set of dependencies -FROM pypy:$PYTHON-slim-$DEBIAN as stage-0 -ARG PYTHON -# install runtime first deps to speedup the dev deps and because layers will be reused on stage-1 -RUN apt-get -qy update && apt-get -qy upgrade -RUN apt-get -qy install libssl1.1 graphviz librocksdb6.11 -# dev deps for this build start here -RUN apt-get -qy install libssl-dev libffi-dev build-essential zlib1g-dev libbz2-dev libsnappy-dev liblz4-dev librocksdb-dev cargo git pkg-config -# install all deps in a virtualenv so we can just copy it over to the final image -RUN pip --no-input --no-cache-dir install --upgrade pip wheel poetry -ENV POETRY_VIRTUALENVS_IN_PROJECT=true -WORKDIR /app/ -COPY pyproject.toml poetry.lock ./ -RUN poetry install -n -E sentry --no-root --only=main -COPY hathor ./hathor -COPY README.md ./ -RUN poetry build -f wheel -RUN poetry run pip install dist/hathor-*.whl - -# finally: use production .venv from before -# lean and mean: this image should be about ~50MB, would be about ~470MB if using the whole stage-1 -FROM pypy:$PYTHON-slim-$DEBIAN -ARG PYTHON -RUN apt-get -qy update && apt-get -qy upgrade -RUN apt-get -qy install libssl1.1 graphviz librocksdb6.11 -COPY --from=stage-0 /app/.venv/lib/pypy${PYTHON}/site-packages/ /opt/pypy/lib/pypy${PYTHON}/site-packages/ -# XXX: copy optional BUILD_VERSION file using ...VERSIO[N] instead of ...VERSION* to ensure only one file will be copied -# XXX: also copying the README.md because we need at least one existing file -COPY README.md BUILD_VERSIO[N] / -EXPOSE 40403 8080 -ENTRYPOINT ["pypy", "-m", "hathor"] diff --git a/Makefile b/Makefile index 563e86410..eed3cee4e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -py_sources = hathor/ tests/ +py_sources = hathor/ tests/ extras/custom_tests/ .PHONY: all all: check tests @@ -49,8 +49,12 @@ tests-genesis: tests-ci: pytest $(tests_ci) +.PHONY: tests-custom +tests-custom: + bash ./extras/custom_tests.sh + .PHONY: tests -tests: tests-cli tests-lib tests-genesis tests-ci +tests: tests-cli tests-lib tests-genesis tests-custom tests-ci .PHONY: tests-full tests-full: @@ -60,11 +64,11 @@ tests-full: .PHONY: mypy mypy: - mypy -p hathor -p tests + mypy -p hathor -p tests -p extras.custom_tests .PHONY: dmypy dmypy: - dmypy run --timeout 86400 -- -p hathor -p tests + dmypy run --timeout 86400 -- -p hathor -p tests -p extras.custom_tests .PHONY: flake8 flake8: @@ -134,10 +138,6 @@ endif docker: $(docker_dir)/Dockerfile docker build$(docker_build_flags) -t $(docker_tag) $(docker_dir) -.PHONY: docker-pypy -docker-pypy: $(docker_dir)/Dockerfile.pypy - docker build$(docker_build_flags) -f Dockerfile.pypy -t $(docker_tag) $(docker_dir) - .PHONY: docker-push docker-push: docker docker tag $(docker_tag) hathornetwork/hathor-core:$(docker_subtag) diff --git a/SECURITY.md b/SECURITY.md index 0b8927cf0..98aa9b701 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,3 @@ # Security -Hathor Labs has a bounty program to encourage white hat hackers to collaborate in identifying security breaches and vulnerabilities in Hathor core. To know more about this, see [Bug bounty program at Hathor docs](https://docs.hathor.network/references/besides-documentation#security). +Hathor Labs has a bounty program to encourage white hat hackers to collaborate in identifying security breaches and vulnerabilities in Hathor core. To know more about this, see [Bug bounty program at Hathor Network](https://hathor.network/bug-bounty/). diff --git a/extras/custom_tests.sh b/extras/custom_tests.sh new file mode 100644 index 000000000..ec548c25e --- /dev/null +++ b/extras/custom_tests.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Define colors +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +TESTS_DIR="extras/custom_tests" + +# List of test scripts to be executed +tests=( + /side_dag/test_one_fails.py + /side_dag/test_both_fail.py +) + +# Initialize a variable to track if any test fails +any_test_failed=0 + +# Loop over all tests +for test in "${tests[@]}"; do + echo -e "${BLUE}Testing $test${NC}" + PYTHONPATH=$TESTS_DIR python $TESTS_DIR/$test + result=$? + if [ $result -ne 0 ]; then + echo -e "${RED}Test $test FAILED${NC}" + any_test_failed=1 + else + echo -e "${GREEN}Test $test PASSED${NC}" + fi +done + +# Exit with code 0 if no test failed, otherwise exit with code 1 +if [ $any_test_failed -eq 0 ]; then + echo -e "${GREEN}All tests PASSED${NC}" + exit 0 +else + echo -e "${RED}Some tests FAILED${NC}" + exit 1 +fi diff --git a/extras/custom_tests/__init__.py b/extras/custom_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extras/custom_tests/side_dag/__init__.py b/extras/custom_tests/side_dag/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extras/custom_tests/side_dag/test_both_fail.py b/extras/custom_tests/side_dag/test_both_fail.py new file mode 100644 index 000000000..a715f744a --- /dev/null +++ b/extras/custom_tests/side_dag/test_both_fail.py @@ -0,0 +1,98 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shlex +import signal +import subprocess +import sys + +from utils import ( # type: ignore[import-not-found] + COMMAND, + HATHOR_PROCESS_NAME, + KILL_WAIT_DELAY, + MONITOR_PROCESS_NAME, + SIDE_DAG_PROCESS_NAME, + get_pid_by_name, + is_alive, + popen_is_alive, + wait_seconds, +) + +if sys.platform == 'win32': + print("test skipped on windows") + sys.exit(0) + + +def test_both_fail() -> None: + # Assert that there are no existing processes + assert get_pid_by_name(MONITOR_PROCESS_NAME) is None + assert get_pid_by_name(HATHOR_PROCESS_NAME) is None + assert get_pid_by_name(SIDE_DAG_PROCESS_NAME) is None + + # Run the python command + args = shlex.split(COMMAND) + monitor_process = subprocess.Popen(args) + print(f'running "run_node_with_side_dag" in the background with pid: {monitor_process.pid}') + print('awaiting subprocesses initialization...') + wait_seconds(5) + + monitor_process_pid = get_pid_by_name(MONITOR_PROCESS_NAME) + hathor_process_pid = get_pid_by_name(HATHOR_PROCESS_NAME) + side_dag_process_pid = get_pid_by_name(SIDE_DAG_PROCESS_NAME) + + # Assert that the processes exist and are alive + assert monitor_process_pid == monitor_process.pid + assert monitor_process_pid is not None + assert hathor_process_pid is not None + assert side_dag_process_pid is not None + + assert is_alive(monitor_process_pid) + assert is_alive(hathor_process_pid) + assert is_alive(side_dag_process_pid) + + print('processes are running:') + print(f' "{MONITOR_PROCESS_NAME}" pid: {monitor_process_pid}') + print(f' "{HATHOR_PROCESS_NAME}" pid: {hathor_process_pid}') + print(f' "{SIDE_DAG_PROCESS_NAME}" pid: {side_dag_process_pid}') + print('letting processes run for a while...') + wait_seconds(10) + + # Terminate both subprocess + print('terminating subprocesses...') + os.kill(hathor_process_pid, signal.SIGTERM) + os.kill(side_dag_process_pid, signal.SIGTERM) + print('awaiting processes termination...') + wait_seconds(KILL_WAIT_DELAY, break_function=lambda: not popen_is_alive(monitor_process)) + + # Assert that all process are terminated + assert not popen_is_alive(monitor_process) + assert not is_alive(monitor_process_pid) + assert not is_alive(hathor_process_pid) + assert not is_alive(side_dag_process_pid) + + print('all processes are dead. test succeeded!') + + +try: + test_both_fail() +except Exception: + if monitor_process_pid := get_pid_by_name(MONITOR_PROCESS_NAME): + os.kill(monitor_process_pid, signal.SIGKILL) + if hathor_process_pid := get_pid_by_name(HATHOR_PROCESS_NAME): + os.kill(hathor_process_pid, signal.SIGKILL) + if side_dag_process_pid := get_pid_by_name(SIDE_DAG_PROCESS_NAME): + os.kill(side_dag_process_pid, signal.SIGKILL) + + raise diff --git a/extras/custom_tests/side_dag/test_one_fails.py b/extras/custom_tests/side_dag/test_one_fails.py new file mode 100644 index 000000000..436b87179 --- /dev/null +++ b/extras/custom_tests/side_dag/test_one_fails.py @@ -0,0 +1,97 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shlex +import signal +import subprocess +import sys + +from utils import ( # type: ignore[import-not-found] + COMMAND, + HATHOR_PROCESS_NAME, + KILL_WAIT_DELAY, + MONITOR_PROCESS_NAME, + SIDE_DAG_PROCESS_NAME, + get_pid_by_name, + is_alive, + popen_is_alive, + wait_seconds, +) + +if sys.platform == 'win32': + print("test skipped on windows") + sys.exit(0) + + +def test_one_fails() -> None: + # Assert that there are no existing processes + assert get_pid_by_name(MONITOR_PROCESS_NAME) is None + assert get_pid_by_name(HATHOR_PROCESS_NAME) is None + assert get_pid_by_name(SIDE_DAG_PROCESS_NAME) is None + + # Run the python command + args = shlex.split(COMMAND) + monitor_process = subprocess.Popen(args) + print(f'running "run_node_with_side_dag" in the background with pid: {monitor_process.pid}') + print('awaiting subprocesses initialization...') + wait_seconds(5) + + monitor_process_pid = get_pid_by_name(MONITOR_PROCESS_NAME) + hathor_process_pid = get_pid_by_name(HATHOR_PROCESS_NAME) + side_dag_process_pid = get_pid_by_name(SIDE_DAG_PROCESS_NAME) + + # Assert that the processes exist and are alive + assert monitor_process_pid == monitor_process.pid + assert monitor_process_pid is not None + assert hathor_process_pid is not None + assert side_dag_process_pid is not None + + assert is_alive(monitor_process_pid) + assert is_alive(hathor_process_pid) + assert is_alive(side_dag_process_pid) + + print('processes are running:') + print(f' "{MONITOR_PROCESS_NAME}" pid: {monitor_process_pid}') + print(f' "{HATHOR_PROCESS_NAME}" pid: {hathor_process_pid}') + print(f' "{SIDE_DAG_PROCESS_NAME}" pid: {side_dag_process_pid}') + print('letting processes run for a while...') + wait_seconds(10) + + # Terminate one subprocess + print('terminating side-dag process...') + os.kill(side_dag_process_pid, signal.SIGTERM) + print('awaiting process termination...') + wait_seconds(KILL_WAIT_DELAY, break_function=lambda: not popen_is_alive(monitor_process)) + + # Assert that all process are terminated + assert not popen_is_alive(monitor_process) + assert not is_alive(monitor_process_pid) + assert not is_alive(hathor_process_pid) + assert not is_alive(side_dag_process_pid) + + print('all processes are dead. test succeeded!') + + +try: + test_one_fails() +except Exception: + if monitor_process_pid := get_pid_by_name(MONITOR_PROCESS_NAME): + os.kill(monitor_process_pid, signal.SIGKILL) + if hathor_process_pid := get_pid_by_name(HATHOR_PROCESS_NAME): + os.kill(hathor_process_pid, signal.SIGKILL) + if side_dag_process_pid := get_pid_by_name(SIDE_DAG_PROCESS_NAME): + os.kill(side_dag_process_pid, signal.SIGKILL) + + raise diff --git a/extras/custom_tests/side_dag/utils.py b/extras/custom_tests/side_dag/utils.py new file mode 100644 index 000000000..7793c30ac --- /dev/null +++ b/extras/custom_tests/side_dag/utils.py @@ -0,0 +1,78 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import time +from typing import Callable + +MONITOR_PROCESS_NAME = 'hathor-core: monitor' +PROCNAME_SUFFIX = 'hathor-core' +HATHOR_PROCESS_PREFIX = 'hathor:' +SIDE_DAG_PROCESS_PREFIX = 'side-dag:' +HATHOR_PROCESS_NAME = HATHOR_PROCESS_PREFIX + PROCNAME_SUFFIX +SIDE_DAG_PROCESS_NAME = SIDE_DAG_PROCESS_PREFIX + PROCNAME_SUFFIX +KILL_WAIT_DELAY = 305 + +COMMAND = f""" + python -m hathor run_node_with_side_dag + --disable-logs + --testnet + --memory-storage + --x-localhost-only + --procname-prefix {HATHOR_PROCESS_PREFIX} + --side-dag-testnet + --side-dag-memory-storage + --side-dag-x-localhost-only + --side-dag-procname-prefix {SIDE_DAG_PROCESS_PREFIX} +""" + + +def wait_seconds(seconds: int, *, break_function: Callable[[], bool] | None = None) -> None: + while seconds > 0: + print(f'waiting {seconds} seconds...') + time.sleep(1) + seconds -= 1 + if break_function and break_function(): + break + + +def get_pid_by_name(process_name: str) -> int | None: + try: + output = subprocess.check_output(['pgrep', '-f', process_name], text=True) + except subprocess.CalledProcessError: + return None + pids = output.strip().split() + assert len(pids) <= 1 + try: + return int(pids[0]) + except IndexError: + return None + + +def is_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +def popen_is_alive(popen: subprocess.Popen) -> bool: + try: + popen.wait(0) + except subprocess.TimeoutExpired: + return True + assert popen.returncode is not None + return False diff --git a/extras/github/docker.py b/extras/github/docker.py index f1702b81d..3a0c667e9 100644 --- a/extras/github/docker.py +++ b/extras/github/docker.py @@ -89,18 +89,9 @@ def extract_pyver(filename): for line in open(filename).readlines(): if line.startswith('ARG PYTHON'): return line.split('=')[1].strip() - dockerfile_cpython = 'Dockerfile' - dockerfile_pypy = 'Dockerfile.pypy' - default_python = 'python' + extract_pyver(dockerfile_cpython) - default_pypy = 'pypy' + extract_pyver(dockerfile_pypy) - - # Set which Dockerfile to use based on the versions matrix - if MATRIX_PYTHON_IMPL == 'pypy': - dockerfile = dockerfile_pypy - suffix = 'pypy' + MATRIX_PYTHON_VERSION - else: - dockerfile = dockerfile_cpython - suffix = 'python' + MATRIX_PYTHON_VERSION + dockerfile = 'Dockerfile' + default_python = 'python' + extract_pyver(dockerfile) + suffix = 'python' + MATRIX_PYTHON_VERSION # Build the tag list @@ -116,8 +107,6 @@ def extract_pyver(filename): if suffix == default_python: tags.add(base_version) output['slack-notification-version'] = base_version - elif suffix == default_pypy: - tags.add(base_version + '-pypy') # Check if this is a stable release if re.match(r'^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$', base_version): diff --git a/extras/github/test_docker.py b/extras/github/test_docker.py index eda6dbafc..6cfe2e2fc 100644 --- a/extras/github/test_docker.py +++ b/extras/github/test_docker.py @@ -165,35 +165,3 @@ def test_release_default_python(self): self.assertIn('mock_image:latest', output['tags'].split(',')) self.assertEqual(output['push'], 'true') self.assertEqual(output['dockerfile'], 'Dockerfile') - - def test_release_non_default_python(self): - os.environ.update({ - 'GITHUB_REF': 'refs/tags/v0.53.0', - 'GITHUB_EVENT_NAME': 'push', - 'GITHUB_SHA': '55629a7d0ae267cdd27618f452e9f1ad6764fd43', - 'GITHUB_EVENT_DEFAULT_BRANCH': 'master', - 'GITHUB_EVENT_NUMBER': '', - 'MATRIX_PYTHON_IMPL': 'pypy', - 'MATRIX_PYTHON_VERSION': '3.8', - 'SECRETS_DOCKERHUB_IMAGE': 'mock_image', - 'SECRETS_GHCR_IMAGE': '', - }) - - output, base_version, is_release_candidate, overwrite_hathor_core_version = prep_base_version(os.environ) - - self.assertTrue(overwrite_hathor_core_version) - self.assertFalse(is_release_candidate) - self.assertFalse(output['disable-slack-notification']) - self.assertEqual(base_version, 'v0.53.0') - - output = prep_tags(os.environ, base_version, is_release_candidate) - - self.assertNotIn('slack-notification-version', output) - self.assertEqual(output['version'], 'v0.53.0-pypy3.8') - self.assertEqual(output['login-dockerhub'], 'true') - self.assertEqual(output['login-ghcr'], 'false') - self.assertEqual(len(output['tags'].split(',')), 2) - self.assertIn('mock_image:v0.53-pypy3.8', output['tags'].split(',')) - self.assertIn('mock_image:v0.53.0-pypy3.8', output['tags'].split(',')) - self.assertEqual(output['push'], 'true') - self.assertEqual(output['dockerfile'], 'Dockerfile.pypy') diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index ec79c4af1..de8410bb2 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -19,9 +19,9 @@ from typing_extensions import assert_never from hathor.checkpoint import Checkpoint -from hathor.conf.get_settings import get_global_settings from hathor.conf.settings import HathorSettings as HathorSettingsType from hathor.consensus import ConsensusAlgorithm +from hathor.consensus.poa import PoaBlockProducer, PoaSigner from hathor.daa import DifficultyAdjustmentAlgorithm from hathor.event import EventManager from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage @@ -46,6 +46,7 @@ TransactionRocksDBStorage, TransactionStorage, ) +from hathor.transaction.vertex_parser import VertexParser from hathor.util import Random, get_environment_info, not_none from hathor.verification.verification_service import VerificationService from hathor.verification.vertex_verifiers import VertexVerifiers @@ -61,11 +62,14 @@ class SyncSupportLevel(IntEnum): ENABLED = 2 # available and enabled by default, possible to disable at runtime @classmethod - def add_factories(cls, - p2p_manager: ConnectionsManager, - sync_v1_support: 'SyncSupportLevel', - sync_v2_support: 'SyncSupportLevel', - ) -> None: + def add_factories( + cls, + settings: HathorSettingsType, + p2p_manager: ConnectionsManager, + sync_v1_support: 'SyncSupportLevel', + sync_v2_support: 'SyncSupportLevel', + vertex_parser: VertexParser, + ) -> None: """Adds the sync factory to the manager according to the support level.""" from hathor.p2p.sync_v1.factory import SyncV11Factory from hathor.p2p.sync_v2.factory import SyncV2Factory @@ -73,12 +77,14 @@ def add_factories(cls, # sync-v1 support: if sync_v1_support > cls.UNAVAILABLE: - p2p_manager.add_sync_factory(SyncVersion.V1_1, SyncV11Factory(p2p_manager)) + p2p_manager.add_sync_factory(SyncVersion.V1_1, SyncV11Factory(p2p_manager, vertex_parser=vertex_parser)) if sync_v1_support is cls.ENABLED: p2p_manager.enable_sync_version(SyncVersion.V1_1) # sync-v2 support: if sync_v2_support > cls.UNAVAILABLE: - p2p_manager.add_sync_factory(SyncVersion.V2, SyncV2Factory(p2p_manager)) + p2p_manager.add_sync_factory( + SyncVersion.V2, SyncV2Factory(settings, p2p_manager, vertex_parser=vertex_parser) + ) if sync_v2_support is cls.ENABLED: p2p_manager.enable_sync_version(SyncVersion.V2) @@ -186,8 +192,11 @@ def __init__(self) -> None: self._execution_manager: ExecutionManager | None = None self._vertex_handler: VertexHandler | None = None + self._vertex_parser: VertexParser | None = None self._consensus: ConsensusAlgorithm | None = None self._p2p_manager: ConnectionsManager | None = None + self._poa_signer: PoaSigner | None = None + self._poa_block_producer: PoaBlockProducer | None = None def build(self) -> BuildArtifacts: if self.artifacts is not None: @@ -220,6 +229,8 @@ def build(self) -> BuildArtifacts: daa = self._get_or_create_daa() cpu_mining_service = self._get_or_create_cpu_mining_service() vertex_handler = self._get_or_create_vertex_handler() + vertex_parser = self._get_or_create_vertex_parser() + poa_block_producer = self._get_or_create_poa_block_producer() if self._enable_address_index: indexes.enable_address_index(pubsub) @@ -259,10 +270,14 @@ def build(self) -> BuildArtifacts: cpu_mining_service=cpu_mining_service, execution_manager=execution_manager, vertex_handler=vertex_handler, + vertex_parser=vertex_parser, + poa_block_producer=poa_block_producer, **kwargs ) p2p_manager.set_manager(manager) + if poa_block_producer: + poa_block_producer.manager = manager stratum_factory: Optional[StratumFactory] = None if self._enable_stratum_server: @@ -330,7 +345,7 @@ def set_peer_id(self, peer_id: PeerId) -> 'Builder': def _get_or_create_settings(self) -> HathorSettingsType: """Return the HathorSettings instance set on this builder, or a new one if not set.""" if self._settings is None: - self._settings = get_global_settings() + raise ValueError('settings not set') return self._settings def _get_reactor(self) -> Reactor: @@ -406,7 +421,8 @@ def _get_or_create_p2p_manager(self) -> ConnectionsManager: assert self._network is not None self._p2p_manager = ConnectionsManager( - reactor, + settings=self._get_or_create_settings(), + reactor=reactor, network=self._network, my_peer=my_peer, pubsub=self._get_or_create_pubsub(), @@ -414,7 +430,13 @@ def _get_or_create_p2p_manager(self) -> ConnectionsManager: whitelist_only=False, rng=self._rng, ) - SyncSupportLevel.add_factories(self._p2p_manager, self._sync_v1_support, self._sync_v2_support) + SyncSupportLevel.add_factories( + self._get_or_create_settings(), + self._p2p_manager, + self._sync_v1_support, + self._sync_v2_support, + self._get_or_create_vertex_parser(), + ) return self._p2p_manager def _get_or_create_indexes_manager(self) -> IndexesManager: @@ -422,7 +444,7 @@ def _get_or_create_indexes_manager(self) -> IndexesManager: return self._indexes_manager if self._force_memory_index or self._storage_type == StorageType.MEMORY: - self._indexes_manager = MemoryIndexesManager() + self._indexes_manager = MemoryIndexesManager(settings=self._get_or_create_settings()) elif self._storage_type == StorageType.ROCKSDB: rocksdb_storage = self._get_or_create_rocksdb_storage() @@ -435,6 +457,7 @@ def _get_or_create_indexes_manager(self) -> IndexesManager: def _get_or_create_tx_storage(self) -> TransactionStorage: indexes = self._get_or_create_indexes_manager() + settings = self._get_or_create_settings() if self._tx_storage is not None: # If a tx storage is provided, set the indexes manager to it. @@ -446,11 +469,17 @@ def _get_or_create_tx_storage(self) -> TransactionStorage: store_indexes = None if self._storage_type == StorageType.MEMORY: - self._tx_storage = TransactionMemoryStorage(indexes=store_indexes) + self._tx_storage = TransactionMemoryStorage(indexes=store_indexes, settings=settings) elif self._storage_type == StorageType.ROCKSDB: rocksdb_storage = self._get_or_create_rocksdb_storage() - self._tx_storage = TransactionRocksDBStorage(rocksdb_storage, indexes=store_indexes) + vertex_parser = self._get_or_create_vertex_parser() + self._tx_storage = TransactionRocksDBStorage( + rocksdb_storage, + indexes=store_indexes, + settings=settings, + vertex_parser=vertex_parser, + ) else: raise NotImplementedError @@ -460,7 +489,9 @@ def _get_or_create_tx_storage(self) -> TransactionStorage: kwargs: dict[str, Any] = {} if self._tx_storage_cache_capacity is not None: kwargs['capacity'] = self._tx_storage_cache_capacity - self._tx_storage = TransactionCacheStorage(self._tx_storage, reactor, indexes=indexes, **kwargs) + self._tx_storage = TransactionCacheStorage( + self._tx_storage, reactor, indexes=indexes, settings=settings, **kwargs + ) return self._tx_storage @@ -530,8 +561,9 @@ def _get_or_create_bit_signaling_service(self) -> BitSignalingService: def _get_or_create_verification_service(self) -> VerificationService: if self._verification_service is None: + settings = self._get_or_create_settings() verifiers = self._get_or_create_vertex_verifiers() - self._verification_service = VerificationService(verifiers=verifiers) + self._verification_service = VerificationService(settings=settings, verifiers=verifiers) return self._verification_service @@ -590,6 +622,27 @@ def _get_or_create_vertex_handler(self) -> VertexHandler: return self._vertex_handler + def _get_or_create_vertex_parser(self) -> VertexParser: + if self._vertex_parser is None: + self._vertex_parser = VertexParser( + settings=self._get_or_create_settings() + ) + + return self._vertex_parser + + def _get_or_create_poa_block_producer(self) -> PoaBlockProducer | None: + if not self._poa_signer: + return None + + if self._poa_block_producer is None: + self._poa_block_producer = PoaBlockProducer( + settings=self._get_or_create_settings(), + reactor=self._get_reactor(), + poa_signer=self._poa_signer, + ) + + return self._poa_block_producer + def use_memory(self) -> 'Builder': self.check_if_can_modify() self._storage_type = StorageType.MEMORY @@ -785,3 +838,8 @@ def set_settings(self, settings: HathorSettingsType) -> 'Builder': self.check_if_can_modify() self._settings = settings return self + + def set_poa_signer(self, signer: PoaSigner) -> 'Builder': + self.check_if_can_modify() + self._poa_signer = signer + return self diff --git a/hathor/builder/cli_builder.py b/hathor/builder/cli_builder.py index 2d7cf1372..33acca41b 100644 --- a/hathor/builder/cli_builder.py +++ b/hathor/builder/cli_builder.py @@ -22,6 +22,7 @@ from structlog import get_logger from hathor.cli.run_node_args import RunNodeArgs +from hathor.cli.side_dag import SideDagArgs from hathor.consensus import ConsensusAlgorithm from hathor.daa import DifficultyAdjustmentAlgorithm from hathor.event import EventManager @@ -33,12 +34,14 @@ from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager from hathor.manager import HathorManager from hathor.mining.cpu_mining_service import CpuMiningService +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer_id import PeerId from hathor.p2p.utils import discover_hostname, get_genesis_short_hash from hathor.pubsub import PubSubManager from hathor.reactor import ReactorProtocol as Reactor from hathor.stratum import StratumFactory +from hathor.transaction.vertex_parser import VertexParser from hathor.util import Random, not_none from hathor.verification.verification_service import VerificationService from hathor.verification.vertex_verifiers import VertexVerifiers @@ -116,6 +119,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: self.check_or_raise(not settings.ENABLE_NANO_CONTRACTS, 'configuration error: NanoContracts can only be enabled on localnets for now') + vertex_parser = VertexParser(settings=settings) tx_storage: TransactionStorage event_storage: EventStorage indexes: IndexesManager @@ -127,7 +131,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: self.check_or_raise(not self._args.data, '--data should not be used with --memory-storage') # if using MemoryStorage, no need to have cache indexes = MemoryIndexesManager() - tx_storage = TransactionMemoryStorage(indexes) + tx_storage = TransactionMemoryStorage(indexes, settings=settings) event_storage = EventMemoryStorage() self.check_or_raise(not self._args.x_rocksdb_indexes, 'RocksDB indexes require RocksDB data') self.log.info('with storage', storage_class=type(tx_storage).__name__) @@ -150,14 +154,16 @@ def create_manager(self, reactor: Reactor) -> HathorManager: # We should only pass indexes if cache is disabled. Otherwise, # only TransactionCacheStorage should have indexes. kwargs['indexes'] = indexes - tx_storage = TransactionRocksDBStorage(self.rocksdb_storage, **kwargs) + tx_storage = TransactionRocksDBStorage( + self.rocksdb_storage, settings=settings, vertex_parser=vertex_parser, **kwargs + ) event_storage = EventRocksDBStorage(self.rocksdb_storage) feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage) self.log.info('with storage', storage_class=type(tx_storage).__name__, path=self._args.data) if self._args.cache: self.check_or_raise(not self._args.memory_storage, '--cache should not be used with --memory-storage') - tx_storage = TransactionCacheStorage(tx_storage, reactor, indexes=indexes) + tx_storage = TransactionCacheStorage(tx_storage, reactor, indexes=indexes, settings=settings) if self._args.cache_size: tx_storage.capacity = self._args.cache_size if self._args.cache_interval: @@ -176,8 +182,10 @@ def create_manager(self, reactor: Reactor) -> HathorManager: sync_choice: SyncChoice if self._args.sync_bridge: + self.log.warn('--sync-bridge is deprecated and will be removed') sync_choice = SyncChoice.BRIDGE_DEFAULT elif self._args.sync_v1_only: + self.log.warn('--sync-v1-only is deprecated and will be removed') sync_choice = SyncChoice.V1_DEFAULT elif self._args.sync_v2_only: self.log.warn('--sync-v2-only is the default, this parameter has no effect') @@ -185,10 +193,13 @@ def create_manager(self, reactor: Reactor) -> HathorManager: elif self._args.x_remove_sync_v1: sync_choice = SyncChoice.V2_ONLY elif self._args.x_sync_bridge: - self.log.warn('--x-sync-bridge is deprecated and will be removed, use --sync-bridge instead') + self.log.warn('--x-sync-bridge is deprecated and will be removed') sync_choice = SyncChoice.BRIDGE_DEFAULT + elif self._args.x_sync_v1_only: + self.log.warn('--x-sync-v1-only is deprecated and will be removed') + sync_choice = SyncChoice.V1_DEFAULT elif self._args.x_sync_v2_only: - self.log.warn('--x-sync-v2-only is deprecated and will be removed, use --sync-v2-only instead') + self.log.warn('--x-sync-v2-only is deprecated and will be removed') sync_choice = SyncChoice.V2_DEFAULT else: # XXX: this is the default behavior when no parameter is given @@ -284,12 +295,13 @@ def create_manager(self, reactor: Reactor) -> HathorManager: daa=daa, feature_service=self.feature_service ) - verification_service = VerificationService(verifiers=vertex_verifiers) + verification_service = VerificationService(settings=settings, verifiers=vertex_verifiers) cpu_mining_service = CpuMiningService() p2p_manager = ConnectionsManager( - reactor, + settings=settings, + reactor=reactor, network=network, my_peer=peer_id, pubsub=pubsub, @@ -297,7 +309,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager: whitelist_only=False, rng=Random(), ) - SyncSupportLevel.add_factories(p2p_manager, sync_v1_support, sync_v2_support) + SyncSupportLevel.add_factories(settings, p2p_manager, sync_v1_support, sync_v2_support, vertex_parser) vertex_handler = VertexHandler( reactor=reactor, @@ -312,6 +324,18 @@ def create_manager(self, reactor: Reactor) -> HathorManager: log_vertex_bytes=self._args.log_vertex_bytes, ) + from hathor.consensus.poa import PoaBlockProducer, PoaSignerFile + poa_block_producer: PoaBlockProducer | None = None + if settings.CONSENSUS_ALGORITHM.is_poa(): + assert isinstance(self._args, SideDagArgs) + if self._args.poa_signer_file: + poa_signer_file = PoaSignerFile.parse_file(self._args.poa_signer_file) + poa_block_producer = PoaBlockProducer( + settings=settings, + reactor=reactor, + poa_signer=poa_signer_file.get_signer(), + ) + self.manager = HathorManager( reactor, settings=settings, @@ -334,6 +358,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager: cpu_mining_service=cpu_mining_service, execution_manager=execution_manager, vertex_handler=vertex_handler, + vertex_parser=vertex_parser, + poa_block_producer=poa_block_producer, ) if self._args.x_ipython_kernel: @@ -342,6 +368,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager: self._start_ipykernel() p2p_manager.set_manager(self.manager) + if poa_block_producer: + poa_block_producer.manager = self.manager if self._args.stratum: stratum_factory = StratumFactory(self.manager, reactor=reactor) @@ -368,7 +396,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager: p2p_manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts)) if self._args.bootstrap: - p2p_manager.add_peer_discovery(BootstrapPeerDiscovery(self._args.bootstrap)) + entrypoints = [Entrypoint.parse(desc) for desc in self._args.bootstrap] + p2p_manager.add_peer_discovery(BootstrapPeerDiscovery(entrypoints)) if self._args.x_rocksdb_indexes: self.log.warn('--x-rocksdb-indexes is now the default, no need to specify it') diff --git a/hathor/builder/resources_builder.py b/hathor/builder/resources_builder.py index 0e89448ab..470f0c613 100644 --- a/hathor/builder/resources_builder.py +++ b/hathor/builder/resources_builder.py @@ -261,14 +261,18 @@ def create_resources(self) -> server.Site: # Websocket resource assert self.manager.tx_storage.indexes is not None - ws_factory = HathorAdminWebsocketFactory(metrics=self.manager.metrics, + ws_factory = HathorAdminWebsocketFactory(manager=self.manager, + metrics=self.manager.metrics, address_index=self.manager.tx_storage.indexes.addresses) + if self._args.disable_ws_history_streaming: + ws_factory.disable_history_streaming() ws_factory.start() root.putChild(b'ws', WebSocketResource(ws_factory)) - # Mining websocket resource - mining_ws_factory = MiningWebsocketFactory(self.manager) - root.putChild(b'mining_ws', WebSocketResource(mining_ws_factory)) + if settings.CONSENSUS_ALGORITHM.is_pow(): + # Mining websocket resource + mining_ws_factory = MiningWebsocketFactory(self.manager) + root.putChild(b'mining_ws', WebSocketResource(mining_ws_factory)) ws_factory.subscribe(self.manager.pubsub) diff --git a/hathor/cli/db_import.py b/hathor/cli/db_import.py index 6369890e4..8063b31d2 100644 --- a/hathor/cli/db_import.py +++ b/hathor/cli/db_import.py @@ -73,7 +73,10 @@ def run(self) -> None: self.log.info('imported', tx_count=tx_count, block_count=block_count) def _import_txs(self) -> Iterator['BaseTransaction']: - from hathor.transaction.base_transaction import tx_or_block_from_bytes + from hathor.conf.get_settings import get_global_settings + from hathor.transaction.vertex_parser import VertexParser + settings = get_global_settings() + parser = VertexParser(settings=settings) while True: # read tx tx_len_bytes = self.in_file.read(4) @@ -84,7 +87,7 @@ def _import_txs(self) -> Iterator['BaseTransaction']: if len(tx_bytes) != tx_len: self.log.error('unexpected end of file', expected=tx_len, got=len(tx_bytes)) sys.exit(2) - tx = tx_or_block_from_bytes(tx_bytes) + tx = parser.deserialize(tx_bytes) assert tx is not None tx.storage = self.tx_storage self.manager.on_new_tx(tx, quiet=True, fails_silently=False) diff --git a/hathor/cli/events_simulator/scenario.py b/hathor/cli/events_simulator/scenario.py index 8a2d20251..cf03f9cce 100644 --- a/hathor/cli/events_simulator/scenario.py +++ b/hathor/cli/events_simulator/scenario.py @@ -26,6 +26,7 @@ class Scenario(Enum): SINGLE_CHAIN_BLOCKS_AND_TRANSACTIONS = 'SINGLE_CHAIN_BLOCKS_AND_TRANSACTIONS' REORG = 'REORG' UNVOIDED_TRANSACTION = 'UNVOIDED_TRANSACTION' + INVALID_MEMPOOL_TRANSACTION = 'INVALID_MEMPOOL_TRANSACTION' def simulate(self, simulator: 'Simulator', manager: 'HathorManager') -> None: simulate_fns = { @@ -34,6 +35,7 @@ def simulate(self, simulator: 'Simulator', manager: 'HathorManager') -> None: Scenario.SINGLE_CHAIN_BLOCKS_AND_TRANSACTIONS: simulate_single_chain_blocks_and_transactions, Scenario.REORG: simulate_reorg, Scenario.UNVOIDED_TRANSACTION: simulate_unvoided_transaction, + Scenario.INVALID_MEMPOOL_TRANSACTION: simulate_invalid_mempool_transaction, } simulate_fn = simulate_fns[self] @@ -140,3 +142,38 @@ def simulate_unvoided_transaction(simulator: 'Simulator', manager: 'HathorManage # The first tx gets voided and the second gets unvoided assert tx.get_metadata().voided_by assert not tx2.get_metadata().voided_by + + +def simulate_invalid_mempool_transaction(simulator: 'Simulator', manager: 'HathorManager') -> None: + from hathor.conf.get_settings import get_global_settings + from hathor.simulator.utils import add_new_blocks, gen_new_tx + from hathor.transaction import Block + + settings = get_global_settings() + assert manager.wallet is not None + address = manager.wallet.get_unused_address(mark_as_used=False) + + blocks = add_new_blocks(manager, settings.REWARD_SPEND_MIN_BLOCKS + 1) + simulator.run(60) + + tx = gen_new_tx(manager, address, 1000) + tx.weight = manager.daa.minimum_tx_weight(tx) + tx.update_hash() + assert manager.propagate_tx(tx, fails_silently=False) + simulator.run(60) + + # re-org: replace last two blocks with one block, new height will be just one short of enough + block_to_replace = blocks[-2] + tb0 = manager.make_custom_block_template(block_to_replace.parents[0], block_to_replace.parents[1:]) + b0: Block = tb0.generate_mining_block(manager.rng, storage=manager.tx_storage) + b0.weight = 10 + manager.cpu_mining_service.resolve(b0) + assert manager.propagate_tx(b0, fails_silently=False) + simulator.run(60) + + # the transaction should have been removed from the mempool + assert tx not in manager.tx_storage.iter_mempool_from_best_index() + + # additionally the transaction should have been marked as invalid and removed from the storage after the re-org + assert tx.get_metadata().validation.is_invalid() + assert not manager.tx_storage.transaction_exists(tx.hash) diff --git a/hathor/cli/generate_genesis.py b/hathor/cli/generate_genesis.py new file mode 100644 index 000000000..014c49891 --- /dev/null +++ b/hathor/cli/generate_genesis.py @@ -0,0 +1,63 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from hathor.utils.pydantic import BaseModel # skip-cli-import-custom-check + + +class GenerateGenesisArgs(BaseModel): + tokens: int + address: str + block_timestamp: int + min_block_weight: int + min_tx_weight: int + + +def main() -> None: + from hathor.cli.util import create_parser + from hathor.transaction.genesis import generate_new_genesis + + parser = create_parser() + parser.add_argument('--tokens', type=int, help='Amount of genesis tokens, including decimals', required=True) + parser.add_argument('--address', type=str, help='Address for genesis tokens', required=True) + parser.add_argument('--block-timestamp', type=int, help='Timestamp for the genesis block', required=True) + parser.add_argument('--min-block-weight', type=float, help='The MIN_BLOCK_WEIGHT', required=True) + parser.add_argument('--min-tx-weight', type=float, help='The MIN_TX_WEIGHT', required=True) + + raw_args = parser.parse_args(sys.argv[1:]) + args = GenerateGenesisArgs.parse_obj((vars(raw_args))) + + block, tx1, tx2 = generate_new_genesis( + tokens=args.tokens, + address=args.address, + block_timestamp=args.block_timestamp, + min_block_weight=args.min_block_weight, + min_tx_weight=args.min_tx_weight, + ) + + print('# Paste this output into your network\'s yaml configuration file') + print() + print('GENESIS_BLOCK_HASH:', block.hash_hex) + print('GENESIS_TX1_HASH:', tx1.hash_hex) + print('GENESIS_TX2_HASH:', tx2.hash_hex) + print() + print('GENESIS_OUTPUT_SCRIPT:', block.outputs[0].script.hex()) + print('GENESIS_BLOCK_TIMESTAMP:', block.timestamp) + print('GENESIS_BLOCK_NONCE:', block.nonce) + print('GENESIS_TX1_NONCE:', tx1.nonce) + print('GENESIS_TX2_NONCE:', tx2.nonce) + print() + print('MIN_BLOCK_WEIGHT:', args.min_block_weight) + print('MIN_TX_WEIGHT:', args.min_tx_weight) diff --git a/hathor/cli/generate_poa_keys.py b/hathor/cli/generate_poa_keys.py new file mode 100644 index 000000000..cbefe0db2 --- /dev/null +++ b/hathor/cli/generate_poa_keys.py @@ -0,0 +1,49 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import sys + +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKeyWithSerialization + + +def main(): + from hathor.cli.util import create_parser + from hathor.crypto.util import ( + get_address_b58_from_public_key, + get_private_key_bytes, + get_public_key_bytes_compressed, + ) + parser = create_parser() + parser.add_argument('--config-yaml', type=str, help='Configuration yaml filepath') + args = parser.parse_args(sys.argv[1:]) + if not args.config_yaml: + raise Exception('`--config-yaml` is required') + # We have to set the config file because the `get_address_b58_from_public_key()` call below accesses it indirectly + # to use the version bytes. + os.environ['HATHOR_CONFIG_YAML'] = args.config_yaml + + private_key = ec.generate_private_key(ec.SECP256K1()) + public_key = private_key.public_key() + assert isinstance(private_key, EllipticCurvePrivateKeyWithSerialization) + + data = dict( + private_key_hex=get_private_key_bytes(private_key=private_key).hex(), + public_key_hex=get_public_key_bytes_compressed(public_key=public_key).hex(), + address=get_address_b58_from_public_key(public_key=public_key), + ) + + print(json.dumps(data, indent=4)) diff --git a/hathor/cli/load_from_logs.py b/hathor/cli/load_from_logs.py index c5a34e427..1e39979f6 100644 --- a/hathor/cli/load_from_logs.py +++ b/hathor/cli/load_from_logs.py @@ -37,7 +37,10 @@ def prepare(self, *, register_resources: bool = True) -> None: super().prepare(register_resources=False) def run(self) -> None: - from hathor.transaction.base_transaction import tx_or_block_from_bytes + from hathor.conf.get_settings import get_global_settings + from hathor.transaction.vertex_parser import VertexParser + settings = get_global_settings() + parser = VertexParser(settings=settings) pattern = r'new (tx|block) .*bytes=([^ ]*) ' pattern = r'new (tx|block) .*bytes=([^ ]*) ' @@ -57,7 +60,7 @@ def run(self) -> None: _, vertex_bytes_hex = matches[0] vertex_bytes = bytes.fromhex(vertex_bytes_hex) - vertex = tx_or_block_from_bytes(vertex_bytes) + vertex = parser.deserialize(vertex_bytes) self.manager.on_new_tx(vertex) diff --git a/hathor/cli/main.py b/hathor/cli/main.py index 6c4c84f4e..1c1633fb2 100644 --- a/hathor/cli/main.py +++ b/hathor/cli/main.py @@ -34,6 +34,8 @@ def __init__(self) -> None: from . import ( db_export, db_import, + generate_genesis, + generate_poa_keys, generate_valid_words, load_from_logs, merged_mining, @@ -53,6 +55,7 @@ def __init__(self) -> None: reset_feature_settings, run_node, shell, + side_dag, stratum_mining, twin_tx, tx_generator, @@ -69,6 +72,10 @@ def __init__(self) -> None: if sys.platform != 'win32': from . import top self.add_cmd('hathor', 'top', top, 'CPU profiler viewer') + self.add_cmd('side-dag', 'run_node_with_side_dag', side_dag, 'Run a side-dag') + self.add_cmd('side-dag', 'gen_poa_keys', generate_poa_keys, 'Generate a private/public key pair and its ' + 'address to be used in Proof-of-Authority') + self.add_cmd('side-dag', 'gen_genesis', generate_genesis, 'Generate a new genesis') self.add_cmd('docs', 'generate_openapi_json', openapi_json, 'Generate OpenAPI json for API docs') self.add_cmd('multisig', 'gen_multisig_address', multisig_address, 'Generate a new multisig address') self.add_cmd('multisig', 'spend_multisig_output', multisig_spend, 'Generate tx that spends a multisig output') @@ -119,7 +126,7 @@ def help(self) -> None: print() def execute_from_command_line(self): - from hathor.cli.util import setup_logging + from hathor.cli.util import process_logging_options, process_logging_output, setup_logging if len(sys.argv) < 2: self.help() @@ -138,9 +145,6 @@ def execute_from_command_line(self): sys.argv[0] = '{} {}'.format(sys.argv[0], cmd) module = self.command_list[cmd] - debug = '--debug' in sys.argv - if debug: - sys.argv.remove('--debug') if '--help' in sys.argv: capture_stdout = False else: @@ -154,14 +158,14 @@ def execute_from_command_line(self): pudb.set_trace(paused=False) capture_stdout = False - json_logs = '--json-logs' in sys.argv - if json_logs: - sys.argv.remove('--json-logs') - - sentry = '--sentry-dsn' in sys.argv - - setup_logging(debug, capture_stdout, json_logs, sentry=sentry) - module.main() + pre_setup_logging = getattr(module, 'PRE_SETUP_LOGGING', True) + if pre_setup_logging: + output = process_logging_output(sys.argv) + options = process_logging_options(sys.argv) + setup_logging(logging_output=output, logging_options=options, capture_stdout=capture_stdout) + module.main() + else: + module.main(capture_stdout=capture_stdout) def main(): diff --git a/hathor/cli/mining.py b/hathor/cli/mining.py index a769ed3a0..bb8655a82 100644 --- a/hathor/cli/mining.py +++ b/hathor/cli/mining.py @@ -142,7 +142,7 @@ def execute(args: Namespace) -> None: settings = get_global_settings() daa = DifficultyAdjustmentAlgorithm(settings=settings) verifiers = VertexVerifiers.create_defaults(settings=settings, daa=daa, feature_service=Mock()) - verification_service = VerificationService(verifiers=verifiers) + verification_service = VerificationService(settings=settings, verifiers=verifiers) verification_service.verify_without_storage(block) except HathorError: print('[{}] ERROR: Block has not been pushed because it is not valid.'.format(datetime.datetime.now())) diff --git a/hathor/cli/openapi_files/openapi_base.json b/hathor/cli/openapi_files/openapi_base.json index 237df7466..c92cbb0b0 100644 --- a/hathor/cli/openapi_files/openapi_base.json +++ b/hathor/cli/openapi_files/openapi_base.json @@ -7,7 +7,7 @@ ], "info": { "title": "Hathor API", - "version": "0.60.1" + "version": "0.62.0" }, "consumes": [ "application/json" diff --git a/hathor/cli/quick_test.py b/hathor/cli/quick_test.py index 8fe2e4fee..30f8852ac 100644 --- a/hathor/cli/quick_test.py +++ b/hathor/cli/quick_test.py @@ -14,6 +14,7 @@ import os from argparse import ArgumentParser +from typing import Any from hathor.cli.run_node import RunNode @@ -36,7 +37,7 @@ def prepare(self, *, register_resources: bool = True) -> None: self.log.info('patching on_new_tx to quit on success') orig_on_new_tx = self.manager.on_new_tx - def patched_on_new_tx(*args, **kwargs): + def patched_on_new_tx(*args: Any, **kwargs: Any) -> bool: res = orig_on_new_tx(*args, **kwargs) msg: str | None = None diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index 32f5848f9..399560307 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -50,13 +50,18 @@ class RunNode: UNSAFE_ARGUMENTS: list[tuple[str, Callable[['RunNodeArgs'], bool]]] = [ ('--test-mode-tx-weight', lambda args: bool(args.test_mode_tx_weight)), ('--enable-crash-api', lambda args: bool(args.enable_crash_api)), + ('--sync-bridge', lambda args: bool(args.sync_bridge)), + ('--sync-v1-only', lambda args: bool(args.sync_v1_only)), ('--x-sync-bridge', lambda args: bool(args.x_sync_bridge)), + ('--x-sync-v1-only', lambda args: bool(args.x_sync_v1_only)), ('--x-sync-v2-only', lambda args: bool(args.x_sync_v2_only)), ('--x-enable-event-queue', lambda args: bool(args.x_enable_event_queue)), ('--x-asyncio-reactor', lambda args: bool(args.x_asyncio_reactor)), ('--x-ipython-kernel', lambda args: bool(args.x_ipython_kernel)), ] + env_vars_prefix: str | None = None + @classmethod def create_parser(cls) -> ArgumentParser: """ @@ -65,7 +70,7 @@ def create_parser(cls) -> ArgumentParser: """ from hathor.cli.util import create_parser from hathor.feature_activation.feature import Feature - parser = create_parser() + parser = create_parser(prefix=cls.env_vars_prefix) parser.add_argument('--hostname', help='Hostname used to be accessed by other peers') parser.add_argument('--auto-hostname', action='store_true', help='Try to discover the hostname automatically') @@ -127,14 +132,14 @@ def create_parser(cls) -> ArgumentParser: parser.add_argument('--enable-debug-api', action='store_true', help='Enable _debug/* endpoints') parser.add_argument('--enable-crash-api', action='store_true', help='Enable _crash/* endpoints') sync_args = parser.add_mutually_exclusive_group() - sync_args.add_argument('--sync-bridge', action='store_true', - help='Enable running both sync protocols.') - sync_args.add_argument('--sync-v1-only', action='store_true', help='Disable support for running sync-v2.') - sync_args.add_argument('--sync-v2-only', action='store_true', help='Disable support for running sync-v1.') + sync_args.add_argument('--sync-bridge', action='store_true', help=SUPPRESS) # moved to --x-sync-bridge + sync_args.add_argument('--sync-v1-only', action='store_true', help=SUPPRESS) # moved to --x-sync-v1-only + sync_args.add_argument('--sync-v2-only', action='store_true', help=SUPPRESS) # already default sync_args.add_argument('--x-remove-sync-v1', action='store_true', help='Make sync-v1 unavailable, thus ' - 'impossible to be enable in runtime.') + 'impossible to be enabled in runtime.') + sync_args.add_argument('--x-sync-v1-only', action='store_true', help='Disable support for running sync-v2.') sync_args.add_argument('--x-sync-v2-only', action='store_true', help=SUPPRESS) # old argument - sync_args.add_argument('--x-sync-bridge', action='store_true', help=SUPPRESS) # old argument + sync_args.add_argument('--x-sync-bridge', action='store_true', help='Enable running both sync protocols.') parser.add_argument('--x-localhost-only', action='store_true', help='Only connect to peers on localhost') parser.add_argument('--x-rocksdb-indexes', action='store_true', help=SUPPRESS) parser.add_argument('--x-enable-event-queue', action='store_true', help='Enable event queue mechanism') @@ -153,6 +158,8 @@ def create_parser(cls) -> ArgumentParser: help='Launch embedded IPython kernel for remote debugging') parser.add_argument('--log-vertex-bytes', action='store_true', help='Log tx bytes for debugging') + parser.add_argument('--disable-ws-history-streaming', action='store_true', + help='Disable websocket history streaming API') return parser def prepare(self, *, register_resources: bool = True) -> None: @@ -455,7 +462,6 @@ def check_python_version(self) -> None: ])) def __init__(self, *, argv=None): - from hathor.cli.run_node_args import RunNodeArgs from hathor.conf import NANO_TESTNET_SETTINGS_FILEPATH, TESTNET_SETTINGS_FILEPATH from hathor.conf.get_settings import get_global_settings self.log = logger.new() @@ -467,7 +473,7 @@ def __init__(self, *, argv=None): self.parser = self.create_parser() raw_args = self.parse_args(argv) - self._args = RunNodeArgs.parse_obj(vars(raw_args)) + self._args = self._parse_args_obj(vars(raw_args)) if self._args.config_yaml: os.environ['HATHOR_CONFIG_YAML'] = self._args.config_yaml @@ -529,6 +535,10 @@ def init_sysctl(self, description: str, sysctl_init_file: Optional[str] = None) def parse_args(self, argv: list[str]) -> Namespace: return self.parser.parse_args(argv) + def _parse_args_obj(self, args: dict[str, Any]) -> 'RunNodeArgs': + from hathor.cli.run_node_args import RunNodeArgs + return RunNodeArgs.parse_obj(args) + def run(self) -> None: self.reactor.run() diff --git a/hathor/cli/run_node_args.py b/hathor/cli/run_node_args.py index c67aaeebb..36b137e3f 100644 --- a/hathor/cli/run_node_args.py +++ b/hathor/cli/run_node_args.py @@ -64,6 +64,7 @@ class RunNodeArgs(BaseModel, extra=Extra.allow): enable_debug_api: bool enable_crash_api: bool x_sync_bridge: bool + x_sync_v1_only: bool x_sync_v2_only: bool x_remove_sync_v1: bool sync_bridge: bool @@ -80,3 +81,4 @@ class RunNodeArgs(BaseModel, extra=Extra.allow): x_ipython_kernel: bool nano_testnet: bool log_vertex_bytes: bool + disable_ws_history_streaming: bool diff --git a/hathor/cli/side_dag.py b/hathor/cli/side_dag.py new file mode 100644 index 000000000..20132f2d5 --- /dev/null +++ b/hathor/cli/side_dag.py @@ -0,0 +1,309 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import os +import signal +import sys +import time +from argparse import ArgumentParser +from dataclasses import dataclass +from enum import Enum +from multiprocessing import Process +from typing import TYPE_CHECKING, Any + +from setproctitle import setproctitle +from typing_extensions import assert_never, override + +from hathor.cli.run_node_args import RunNodeArgs # skip-cli-import-custom-check + +if TYPE_CHECKING: + from hathor.cli.util import LoggingOutput + +from structlog import get_logger + +from hathor.cli.run_node import RunNode + +logger = get_logger() + +PRE_SETUP_LOGGING: bool = False + +# Period in seconds for polling subprocesses' state. +MONITOR_WAIT_PERIOD: int = 10 + +# Delay in seconds before killing a subprocess after trying to terminate it. +KILL_WAIT_DELAY = 300 + + +class SideDagArgs(RunNodeArgs): + poa_signer_file: str | None + + +class SideDagRunNode(RunNode): + env_vars_prefix = 'hathor_side_dag_' + + @override + def _parse_args_obj(self, args: dict[str, Any]) -> RunNodeArgs: + return SideDagArgs.parse_obj(args) + + @classmethod + @override + def create_parser(cls) -> ArgumentParser: + parser = super().create_parser() + parser.add_argument('--poa-signer-file', help='File containing the Proof-of-Authority signer private key.') + return parser + + +def main(capture_stdout: bool) -> None: + """ + This command runs two full node instances in separate processes. + + The main process runs a side-dag full node, and it accepts the same options as the `run_node` command. Options + with the `--side-dag` prefix are passed to the side-dag full node, while options without this prefix are passed + to the non-side-dag full node, which runs in a background process and is commonly just a Hathor full node. + Whenever one of the full nodes fail, the other is automatically terminated. + + The only exception is log configuration, which is set using a single option. By default, both full nodes output + logs to stdout. Here's an example changing both logs to json: + + ```bash + $ python -m hathor run_node_with_side_dag + --testnet + --procname-prefix testnet- + --memory-storage + --side-dag-config-yaml ./my-side-dag.yml + --side-dag-procname-prefix my-side-dag- + --side-dag-memory-storage + --json-logs both + ``` + + In this example, Hathor testnet logs would be disabled, while side-dag logs would be outputted to stdout as json. + """ + from hathor.cli.util import process_logging_options, setup_logging + argv = sys.argv[1:] + hathor_logging_output, side_dag_logging_output = _process_logging_output(argv) + hathor_node_argv, side_dag_argv = _partition_argv(argv) + + # the main process uses the same configuration as the hathor process + log_options = process_logging_options(hathor_node_argv.copy()) + setup_logging( + logging_output=hathor_logging_output, + logging_options=log_options, + capture_stdout=capture_stdout, + extra_log_info=_get_extra_log_info('monitor') + ) + + hathor_node_process = _start_node_process( + argv=hathor_node_argv, + runner=RunNode, + logging_output=hathor_logging_output, + capture_stdout=capture_stdout, + name='hathor', + ) + side_dag_node_process = _start_node_process( + argv=side_dag_argv, + runner=SideDagRunNode, + logging_output=side_dag_logging_output, + capture_stdout=capture_stdout, + name='side-dag', + ) + + logger.info( + 'starting nodes', + monitor_pid=os.getpid(), + hathor_node_pid=hathor_node_process.pid, + side_dag_node_pid=side_dag_node_process.pid + ) + + _run_monitor_process(hathor_node_process, side_dag_node_process) + + +def _process_logging_output(argv: list[str]) -> tuple[LoggingOutput, LoggingOutput]: + """Extract logging output before argv parsing.""" + from hathor.cli.util import LoggingOutput, create_parser + + class LogOutputConfig(str, Enum): + HATHOR = 'hathor' + SIDE_DAG = 'side-dag' + BOTH = 'both' + + parser = create_parser(add_help=False) + log_args = parser.add_mutually_exclusive_group() + log_args.add_argument('--json-logs', nargs='?', const='both', type=LogOutputConfig) + log_args.add_argument('--disable-logs', nargs='?', const='both', type=LogOutputConfig) + + args, remaining_argv = parser.parse_known_args(argv) + argv.clear() + argv.extend(remaining_argv) + + def proces_log_output_config( + config: LogOutputConfig, + target: LoggingOutput + ) -> tuple[LoggingOutput, LoggingOutput]: + hathor_output, side_dag_output = LoggingOutput.PRETTY, LoggingOutput.PRETTY + match config: + case LogOutputConfig.HATHOR: + hathor_output = target + case LogOutputConfig.SIDE_DAG: + side_dag_output = target + case LogOutputConfig.BOTH: + hathor_output, side_dag_output = target, target + case _: + assert_never(config) + return hathor_output, side_dag_output + + if args.json_logs: + return proces_log_output_config(args.json_logs, LoggingOutput.JSON) + + if args.disable_logs: + return proces_log_output_config(args.disable_logs, LoggingOutput.NULL) + + return LoggingOutput.PRETTY, LoggingOutput.PRETTY + + +def _partition_argv(argv: list[str]) -> tuple[list[str], list[str]]: + """Partition arguments into hathor node args and side-dag args, based on the `--side-dag` prefix.""" + hathor_node_argv: list[str] = [] + side_dag_argv: list[str] = [] + + def is_option(arg_: str) -> bool: + return arg_.startswith('--') + + for i, arg in enumerate(argv): + if not is_option(arg): + continue + + try: + value = None if is_option(argv[i + 1]) else argv[i + 1] + except IndexError: + value = None + + argv_list = hathor_node_argv + if arg.startswith('--side-dag'): + arg = arg.replace('--side-dag-', '--') + argv_list = side_dag_argv + + argv_list.append(arg) + if value is not None: + argv_list.append(value) + + return hathor_node_argv, side_dag_argv + + +def _run_monitor_process(*processes: Process) -> None: + """Function to be called by the main process to run the side-dag full node.""" + setproctitle('hathor-core: monitor') + signal.signal(signal.SIGINT, lambda _, __: _terminate_and_exit(processes, reason='received SIGINT')) + signal.signal(signal.SIGTERM, lambda _, __: _terminate_and_exit(processes, reason='received SIGTERM')) + + while True: + time.sleep(MONITOR_WAIT_PERIOD) + for process in processes: + if not process.is_alive(): + _terminate_and_exit( + processes, + reason=f'process "{process.name}" (pid: {process.pid}) exited' + ) + + +def _terminate_and_exit(processes: tuple[Process, ...], *, reason: str) -> None: + """Terminate all processes that are alive. Kills them if they're not terminated after a while. + Then, exits the program.""" + logger.critical(f'terminating all nodes. reason: {reason}') + + for process in processes: + if process.is_alive(): + logger.critical(f'terminating process "{process.name}" (pid: {process.pid})...') + process.terminate() + + now = time.time() + while True: + time.sleep(MONITOR_WAIT_PERIOD) + if time.time() >= now + KILL_WAIT_DELAY: + _kill_all(processes) + break + + all_are_dead = all(not process.is_alive() for process in processes) + if all_are_dead: + break + + logger.critical('all nodes terminated.') + sys.exit(0) + + +def _kill_all(processes: tuple[Process, ...]) -> None: + """Kill all processes that are alive.""" + for process in processes: + if process.is_alive(): + logger.critical(f'process "{process.name}" (pid: {process.pid}) still alive, killing it...') + process.kill() + + +def _start_node_process( + *, + argv: list[str], + runner: type[RunNode], + logging_output: LoggingOutput, + capture_stdout: bool, + name: str, +) -> Process: + """Create and start a Hathor node process.""" + args = _RunNodeArgs( + argv=argv, + runner=runner, + logging_output=logging_output, + capture_stdout=capture_stdout, + name=name + ) + process = Process(target=_run_node, args=(args,), name=name) + process.start() + return process + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _RunNodeArgs: + argv: list[str] + runner: type[RunNode] + logging_output: LoggingOutput + capture_stdout: bool + name: str + + +def _run_node(args: _RunNodeArgs) -> None: + from hathor.cli.util import process_logging_options, setup_logging + try: + log_options = process_logging_options(args.argv) + setup_logging( + logging_output=args.logging_output, + logging_options=log_options, + capture_stdout=args.capture_stdout, + extra_log_info=_get_extra_log_info(args.name) + ) + logger.info(f'initializing node "{args.name}"') + node = args.runner(argv=args.argv) + except KeyboardInterrupt: + logger.warn(f'{args.name} node interrupted by user') + return + except (BaseException, Exception): + logger.exception(f'process "{args.name}" terminated due to exception in initialization') + return + + node.run() + logger.critical(f'node "{args.name}" gracefully terminated') + + +def _get_extra_log_info(process_name: str) -> dict[str, str]: + """Return a dict to be used as extra log info for each process.""" + return dict(_source_process=process_name) diff --git a/hathor/cli/util.py b/hathor/cli/util.py index 84ac878fe..3e11b83c5 100644 --- a/hathor/cli/util.py +++ b/hathor/cli/util.py @@ -12,18 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import sys +import traceback from argparse import ArgumentParser from collections import OrderedDict from datetime import datetime -from typing import Any +from enum import IntEnum, auto +from typing import Any, NamedTuple import configargparse import structlog +from structlog.typing import EventDict +from typing_extensions import assert_never -def create_parser() -> ArgumentParser: - return configargparse.ArgumentParser(auto_env_var_prefix='hathor_') +def create_parser(*, prefix: str | None = None, add_help: bool = True) -> ArgumentParser: + return configargparse.ArgumentParser(auto_env_var_prefix=prefix or 'hathor_', add_help=add_help) # docs at http://www.structlog.org/en/stable/api.html#structlog.dev.ConsoleRenderer @@ -119,14 +124,59 @@ def _repr(self, val): return super()._repr(val) +class LoggingOutput(IntEnum): + NULL = auto() + PRETTY = auto() + JSON = auto() + + +class LoggingOptions(NamedTuple): + debug: bool + sentry: bool + + +def process_logging_output(argv: list[str]) -> LoggingOutput: + """Extract logging output before argv parsing.""" + parser = create_parser(add_help=False) + + log_args = parser.add_mutually_exclusive_group() + log_args.add_argument('--json-logs', action='store_true') + log_args.add_argument('--disable-logs', action='store_true') + + args, remaining_argv = parser.parse_known_args(argv) + argv.clear() + argv.extend(remaining_argv) + + if args.json_logs: + return LoggingOutput.JSON + + if args.disable_logs: + return LoggingOutput.NULL + + return LoggingOutput.PRETTY + + +def process_logging_options(argv: list[str]) -> LoggingOptions: + """Extract logging-specific options that are processed before argv parsing.""" + parser = create_parser(add_help=False) + parser.add_argument('--debug', action='store_true') + + args, remaining_argv = parser.parse_known_args(argv) + argv.clear() + argv.extend(remaining_argv) + + sentry = '--sentry-dsn' in argv + return LoggingOptions(debug=args.debug, sentry=sentry) + + def setup_logging( - debug: bool = False, - capture_stdout: bool = False, - json_logging: bool = False, - *, - sentry: bool = False, - _test_logging: bool = False, - ) -> None: + *, + logging_output: LoggingOutput, + logging_options: LoggingOptions, + capture_stdout: bool = False, + _test_logging: bool = False, + extra_log_info: dict[str, str] | None = None, +) -> None: import logging import logging.config @@ -152,13 +202,18 @@ def setup_logging( timestamper, ] - if json_logging: - handlers = ['json'] - else: - handlers = ['pretty'] + match logging_output: + case LoggingOutput.NULL: + handlers = ['null'] + case LoggingOutput.PRETTY: + handlers = ['pretty'] + case LoggingOutput.JSON: + handlers = ['json'] + case _: + assert_never(logging_output) # Flag to enable debug level for both sync-v1 and sync-v2. - debug_sync = False and debug + debug_sync = False and logging_options.debug # See: https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema logging.config.dictConfig({ @@ -192,6 +247,9 @@ def setup_logging( 'class': 'logging.StreamHandler', 'formatter': 'json', }, + 'null': { + 'class': 'logging.NullHandler', + }, # 'file': { # 'level': 'DEBUG', # 'class': 'logging.handlers.WatchedFileHandler', @@ -203,12 +261,12 @@ def setup_logging( # set twisted verbosity one level lower than hathor's 'twisted': { 'handlers': handlers, - 'level': 'INFO' if debug else 'WARN', + 'level': 'INFO' if logging_options.debug else 'WARN', 'propagate': False, }, 'tornado': { # used by ipykernel's zmq 'handlers': handlers, - 'level': 'INFO' if debug else 'WARN', + 'level': 'INFO' if logging_options.debug else 'WARN', 'propagate': False, }, 'hathor.p2p.sync_v1': { @@ -223,23 +281,37 @@ def setup_logging( }, '': { 'handlers': handlers, - 'level': 'DEBUG' if debug else 'INFO', + 'level': 'DEBUG' if logging_options.debug else 'INFO', }, } }) def kwargs_formatter(_, __, event_dict): if event_dict and event_dict.get('event') and isinstance(event_dict['event'], str): - event_dict['event'] = event_dict['event'].format(**event_dict) + try: + event_dict['event'] = event_dict['event'].format(**event_dict) + except KeyError: + # The event string may contain '{}'s that are not used for formatting, resulting in a KeyError in the + # event_dict. In this case, we don't format it. + pass + return event_dict + + extra_log_info = extra_log_info or {} + + def add_extra_log_info(_logger: logging.Logger, _method_name: str, event_dict: EventDict) -> EventDict: + for key, value in extra_log_info.items(): + assert key not in event_dict, 'extra log info conflicting with existing log key' + event_dict[key] = value return event_dict processors: list[Any] = [ structlog.stdlib.filter_by_level, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, + add_extra_log_info, ] - if sentry: + if logging_options.sentry: from structlog_sentry import SentryProcessor processors.append(SentryProcessor(level=logging.ERROR)) @@ -275,10 +347,14 @@ def twisted_structlog_observer(event): if failure is not None: kwargs['exc_info'] = (failure.type, failure.value, failure.getTracebackObject()) twisted_logger.log(level, msg, **kwargs) - except Exception as e: - print('error when logging event', e) - for k, v in event.items(): - print(k, v) + except Exception: + new_event = dict( + event='error when logging event', + original_event=event, + traceback=traceback.format_exc() + ) + new_event_json = json.dumps(new_event, default=str) + print(new_event_json, file=sys.stderr) # start logging to std logger so structlog can catch it twisted.python.log.startLoggingWithObserver(twisted_structlog_observer, setStdout=capture_stdout) diff --git a/hathor/conf/get_settings.py b/hathor/conf/get_settings.py index 6bdbd88b6..fefd74c5c 100644 --- a/hathor/conf/get_settings.py +++ b/hathor/conf/get_settings.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import importlib import os -from typing import NamedTuple, Optional +from typing import TYPE_CHECKING, NamedTuple, Optional from structlog import get_logger -from hathor import conf -from hathor.conf.settings import HathorSettings as Settings +if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings as Settings logger = get_logger() @@ -51,6 +53,7 @@ def HathorSettings() -> Settings: if settings_module_filepath is not None: return _load_settings_singleton(settings_module_filepath, is_yaml=False) + from hathor import conf settings_yaml_filepath = os.environ.get('HATHOR_CONFIG_YAML', conf.MAINNET_SETTINGS_FILEPATH) return _load_settings_singleton(settings_yaml_filepath, is_yaml=True) @@ -94,9 +97,11 @@ def _load_module_settings(module_path: str) -> Settings: ) settings_module = importlib.import_module(module_path) settings = getattr(settings_module, 'SETTINGS') + from hathor.conf.settings import HathorSettings as Settings assert isinstance(settings, Settings) return settings def _load_yaml_settings(filepath: str) -> Settings: + from hathor.conf.settings import HathorSettings as Settings return Settings.from_yaml(filepath=filepath) diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index 682279f6c..d6509b4b6 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -15,11 +15,12 @@ import os from math import log from pathlib import Path -from typing import NamedTuple, Optional, Union +from typing import Any, NamedTuple, Optional, Union import pydantic from hathor.checkpoint import Checkpoint +from hathor.consensus.consensus_settings import ConsensusSettings, PowSettings from hathor.feature_activation.settings import Settings as FeatureActivationSettings from hathor.utils import yaml from hathor.utils.named_tuple import validated_named_tuple_from_dict @@ -436,6 +437,13 @@ def GENESIS_TX2_TIMESTAMP(self) -> int: # List of enabled blueprints. BLUEPRINTS: dict[bytes, 'str'] = {} + # The consensus algorithm protocol settings. + CONSENSUS_ALGORITHM: ConsensusSettings = PowSettings() + + # The name and symbol of the native token. This is only used in APIs to serve clients. + NATIVE_TOKEN_NAME: str = 'Hathor' + NATIVE_TOKEN_SYMBOL: str = 'HTR' + @classmethod def from_yaml(cls, *, filepath: str) -> 'HathorSettings': """Takes a filepath to a yaml file and returns a validated HathorSettings instance.""" @@ -473,7 +481,7 @@ def _parse_blueprints(blueprints_raw: dict[str, str]) -> dict[bytes, str]: return blueprints -def _parse_hex_str(hex_str: Union[str, bytes]) -> bytes: +def parse_hex_str(hex_str: Union[str, bytes]) -> bytes: """Parse a raw hex string into bytes.""" if isinstance(hex_str, str): return bytes.fromhex(hex_str.lstrip('x')) @@ -484,6 +492,40 @@ def _parse_hex_str(hex_str: Union[str, bytes]) -> bytes: return hex_str +def _validate_consensus_algorithm(consensus_algorithm: ConsensusSettings, values: dict[str, Any]) -> ConsensusSettings: + """Validate that if Proof-of-Authority is enabled, block rewards must not be set.""" + if consensus_algorithm.is_pow(): + return consensus_algorithm + + assert consensus_algorithm.is_poa() + blocks_per_halving = values.get('BLOCKS_PER_HALVING') + initial_token_units_per_block = values.get('INITIAL_TOKEN_UNITS_PER_BLOCK') + minimum_token_units_per_block = values.get('MINIMUM_TOKEN_UNITS_PER_BLOCK') + assert initial_token_units_per_block is not None, 'INITIAL_TOKEN_UNITS_PER_BLOCK must be set' + assert minimum_token_units_per_block is not None, 'MINIMUM_TOKEN_UNITS_PER_BLOCK must be set' + + if blocks_per_halving is not None or initial_token_units_per_block != 0 or minimum_token_units_per_block != 0: + raise ValueError('PoA networks do not support block rewards') + + return consensus_algorithm + + +def _validate_tokens(genesis_tokens: int, values: dict[str, Any]) -> int: + """Validate genesis tokens.""" + genesis_token_units = values.get('GENESIS_TOKEN_UNITS') + decimal_places = values.get('DECIMAL_PLACES') + assert genesis_token_units is not None, 'GENESIS_TOKEN_UNITS must be set' + assert decimal_places is not None, 'DECIMAL_PLACES must be set' + + if genesis_tokens != genesis_token_units * (10**decimal_places): + raise ValueError( + f'invalid tokens: GENESIS_TOKENS={genesis_tokens}, GENESIS_TOKEN_UNITS={genesis_token_units}, ' + f'DECIMAL_PLACES={decimal_places}', + ) + + return genesis_tokens + + _VALIDATORS = dict( _parse_hex_str=pydantic.validator( 'P2PKH_VERSION_BYTE', @@ -494,13 +536,13 @@ def _parse_hex_str(hex_str: Union[str, bytes]) -> bytes: 'GENESIS_TX2_HASH', pre=True, allow_reuse=True - )(_parse_hex_str), + )(parse_hex_str), _parse_soft_voided_tx_id=pydantic.validator( 'SOFT_VOIDED_TX_IDS', pre=True, allow_reuse=True, each_item=True - )(_parse_hex_str), + )(parse_hex_str), _parse_checkpoints=pydantic.validator( 'CHECKPOINTS', pre=True @@ -508,5 +550,11 @@ def _parse_hex_str(hex_str: Union[str, bytes]) -> bytes: _parse_blueprints=pydantic.validator( 'BLUEPRINTS', pre=True - )(_parse_blueprints) + )(_parse_blueprints), + _validate_consensus_algorithm=pydantic.validator( + 'CONSENSUS_ALGORITHM' + )(_validate_consensus_algorithm), + _validate_tokens=pydantic.validator( + 'GENESIS_TOKENS' + )(_validate_tokens), ) diff --git a/hathor/consensus/block_consensus.py b/hathor/consensus/block_consensus.py index 7ce12f458..e7a0186cb 100644 --- a/hathor/consensus/block_consensus.py +++ b/hathor/consensus/block_consensus.py @@ -18,7 +18,6 @@ from structlog import get_logger from hathor.conf.get_settings import get_global_settings -from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, Block, Transaction, sum_weights from hathor.util import classproperty @@ -26,7 +25,6 @@ from hathor.consensus.context import ConsensusAlgorithmContext logger = get_logger() -cpu = get_cpu_profiler() _base_transaction_log = logger.new() diff --git a/hathor/consensus/consensus.py b/hathor/consensus/consensus.py index 0317c2fab..fb27e9895 100644 --- a/hathor/consensus/consensus.py +++ b/hathor/consensus/consensus.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + from structlog import get_logger from hathor.conf.get_settings import get_global_settings @@ -24,6 +29,9 @@ from hathor.transaction import BaseTransaction from hathor.util import not_none +if TYPE_CHECKING: + from hathor.transaction.storage import TransactionStorage + logger = get_logger() cpu = get_cpu_profiler() @@ -114,21 +122,20 @@ def _unsafe_update(self, base: BaseTransaction) -> None: raise NotImplementedError new_best_height, new_best_tip = storage.indexes.height.get_height_tip() + txs_to_remove: list[BaseTransaction] = [] if new_best_height < best_height: self.log.warn('height decreased, re-checking mempool', prev_height=best_height, new_height=new_best_height, prev_block_tip=best_tip.hex(), new_block_tip=new_best_tip.hex()) # XXX: this method will mark as INVALID all transactions in the mempool that became invalid because of a # reward lock - to_remove = storage.compute_transactions_that_became_invalid(new_best_height) - if to_remove: + txs_to_remove = storage.compute_transactions_that_became_invalid(new_best_height) + if txs_to_remove: self.log.warn('some transactions on the mempool became invalid and will be removed', - count=len(to_remove)) - # XXX: because transactions in `to_remove` are marked as invalid, we need this context to be able to - # remove them + count=len(txs_to_remove)) + # XXX: because transactions in `txs_to_remove` are marked as invalid, we need this context to be + # able to remove them with storage.allow_invalid_context(): - storage.remove_transactions(to_remove) - for tx_removed in to_remove: - context.pubsub.publish(HathorEvents.CONSENSUS_TX_REMOVED, tx_hash=tx_removed.hash) + self._remove_transactions(txs_to_remove, storage, context) # emit the reorg started event if needed if context.reorg_common_block is not None: @@ -150,6 +157,10 @@ def _unsafe_update(self, base: BaseTransaction) -> None: tx_affected.storage.indexes.update(tx_affected) context.pubsub.publish(HathorEvents.CONSENSUS_TX_UPDATE, tx=tx_affected) + # And emit events for txs that were removed + for tx_removed in txs_to_remove: + context.pubsub.publish(HathorEvents.CONSENSUS_TX_REMOVED, vertex_id=tx_removed.hash) + # and also emit the reorg finished event if needed if context.reorg_common_block is not None: context.pubsub.publish(HathorEvents.REORG_FINISHED) @@ -175,6 +186,52 @@ def filter_out_soft_voided_entries(self, tx: BaseTransaction, voided_by: set[byt ret.add(h) return ret + def _remove_transactions( + self, + txs: list[BaseTransaction], + storage: TransactionStorage, + context: ConsensusAlgorithmContext, + ) -> None: + """Will remove all the transactions on the list from the database. + + Special notes: + + - will refuse and raise an error when removing all transactions would leave dangling transactions, that is, + transactions without existing parent. That is, it expects the `txs` list to include all children of deleted + txs, from both the confirmation and funds DAGs + - inputs's spent_outputs should not have any of the transactions being removed as spending transactions, + this method will update and save those transaction's metadata + - parent's children metadata will be updated to reflect the removals + - all indexes will be updated + """ + parents_to_update: dict[bytes, list[bytes]] = defaultdict(list) + dangling_children: set[bytes] = set() + txset = {tx.hash for tx in txs} + for tx in txs: + tx_meta = tx.get_metadata() + assert not tx_meta.validation.is_checkpoint() + for parent in set(tx.parents) - txset: + parents_to_update[parent].append(tx.hash) + dangling_children.update(set(tx_meta.children) - txset) + for spending_txs in tx_meta.spent_outputs.values(): + dangling_children.update(set(spending_txs) - txset) + for tx_input in tx.inputs: + spent_tx = tx.get_spent_tx(tx_input) + spent_tx_meta = spent_tx.get_metadata() + if tx.hash in spent_tx_meta.spent_outputs[tx_input.index]: + spent_tx_meta.spent_outputs[tx_input.index].remove(tx.hash) + context.save(spent_tx) + assert not dangling_children, 'It is an error to try to remove transactions that would leave a gap in the DAG' + for parent_hash, children_to_remove in parents_to_update.items(): + parent_tx = storage.get_transaction(parent_hash) + parent_meta = parent_tx.get_metadata() + for child in children_to_remove: + parent_meta.children.remove(child) + context.save(parent_tx) + for tx in txs: + self.log.debug('remove transaction', tx=tx.hash_hex) + storage.remove_transaction(tx) + def _sorted_affected_txs(affected_txs: set[BaseTransaction]) -> list[BaseTransaction]: """ diff --git a/hathor/consensus/consensus_settings.py b/hathor/consensus/consensus_settings.py new file mode 100644 index 000000000..259b35f99 --- /dev/null +++ b/hathor/consensus/consensus_settings.py @@ -0,0 +1,149 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +from abc import ABC, abstractmethod +from enum import Enum, unique +from typing import Annotated, Any, Literal, TypeAlias + +from pydantic import Field, NonNegativeInt, PrivateAttr, validator +from typing_extensions import override + +from hathor.transaction import TxVersion +from hathor.util import json_dumpb +from hathor.utils.pydantic import BaseModel + + +@unique +class ConsensusType(str, Enum): + PROOF_OF_WORK = 'PROOF_OF_WORK' + PROOF_OF_AUTHORITY = 'PROOF_OF_AUTHORITY' + + +class _BaseConsensusSettings(ABC, BaseModel): + type: ConsensusType + _peer_hello_hash: str | None = PrivateAttr(default=None) + + def is_pow(self) -> bool: + """Return whether this is a Proof-of-Work consensus.""" + return self.type is ConsensusType.PROOF_OF_WORK + + def is_poa(self) -> bool: + """Return whether this is a Proof-of-Authority consensus.""" + return self.type is ConsensusType.PROOF_OF_AUTHORITY + + @abstractmethod + def _get_valid_vertex_versions(self, include_genesis: bool) -> set[TxVersion]: + """Return a set of `TxVersion`s that are valid in for this consensus type.""" + raise NotImplementedError + + def is_vertex_version_valid(self, version: TxVersion, include_genesis: bool = False) -> bool: + """Return whether a `TxVersion` is valid for this consensus type.""" + return version in self._get_valid_vertex_versions(include_genesis) + + def get_peer_hello_hash(self) -> str | None: + """Return a hash of consensus settings to be used in peer hello validation.""" + if self._peer_hello_hash is None: + self._peer_hello_hash = self._calculate_peer_hello_hash() + return self._peer_hello_hash + + def _calculate_peer_hello_hash(self) -> str | None: + """Calculate a hash of consensus settings to be used in peer hello validation.""" + return None + + +class PowSettings(_BaseConsensusSettings): + type: Literal[ConsensusType.PROOF_OF_WORK] = ConsensusType.PROOF_OF_WORK + + @override + def _get_valid_vertex_versions(self, include_genesis: bool) -> set[TxVersion]: + return { + TxVersion.REGULAR_BLOCK, + TxVersion.REGULAR_TRANSACTION, + TxVersion.TOKEN_CREATION_TRANSACTION, + TxVersion.MERGE_MINED_BLOCK + } + + @override + def get_peer_hello_hash(self) -> str | None: + return None + + +class PoaSignerSettings(BaseModel): + public_key: bytes + start_height: NonNegativeInt = 0 + end_height: NonNegativeInt | None = None + + @validator('public_key', pre=True) + def _parse_hex_str(cls, hex_str: str | bytes) -> bytes: + from hathor.conf.settings import parse_hex_str + return parse_hex_str(hex_str) + + @validator('end_height') + def _validate_end_height(cls, end_height: int | None, values: dict[str, Any]) -> int | None: + start_height = values.get('start_height') + assert start_height is not None, 'start_height must be set' + + if end_height is None: + return None + + if end_height <= start_height: + raise ValueError(f'end_height ({end_height}) must be greater than start_height ({start_height})') + + return end_height + + def to_json_dict(self) -> dict[str, Any]: + """Return this signer settings instance as a json dict.""" + json_dict = self.dict() + # TODO: We can use a custom serializer to convert bytes to hex when we update to Pydantic V2. + json_dict['public_key'] = self.public_key.hex() + return json_dict + + +class PoaSettings(_BaseConsensusSettings): + type: Literal[ConsensusType.PROOF_OF_AUTHORITY] = ConsensusType.PROOF_OF_AUTHORITY + + # A list of Proof-of-Authority signer public keys that have permission to produce blocks. + signers: tuple[PoaSignerSettings, ...] + + @validator('signers') + def _validate_signers(cls, signers: tuple[PoaSignerSettings, ...]) -> tuple[PoaSignerSettings, ...]: + if len(signers) == 0: + raise ValueError('At least one signer must be provided in PoA networks') + return signers + + @override + def _get_valid_vertex_versions(self, include_genesis: bool) -> set[TxVersion]: + versions = { + TxVersion.POA_BLOCK, + TxVersion.REGULAR_TRANSACTION, + TxVersion.TOKEN_CREATION_TRANSACTION, + } + + if include_genesis: + # TODO: We have to add REGULAR_BLOCK to allow genesis deserialization. + # This may be removed if we refactor the way genesis is constructed. + versions.add(TxVersion.REGULAR_BLOCK) + + return versions + + @override + def _calculate_peer_hello_hash(self) -> str | None: + data = b'' + for signer in self.signers: + data += json_dumpb(signer.to_json_dict()) + return hashlib.sha256(data).digest().hex() + + +ConsensusSettings: TypeAlias = Annotated[PowSettings | PoaSettings, Field(discriminator='type')] diff --git a/hathor/consensus/context.py b/hathor/consensus/context.py index 5896ed553..a83af60b2 100644 --- a/hathor/consensus/context.py +++ b/hathor/consensus/context.py @@ -16,7 +16,6 @@ from structlog import get_logger -from hathor.profiler import get_cpu_profiler from hathor.pubsub import PubSubManager from hathor.transaction import BaseTransaction, Block @@ -26,7 +25,6 @@ from hathor.consensus.transaction_consensus import TransactionConsensusAlgorithm logger = get_logger() -cpu = get_cpu_profiler() _base_transaction_log = logger.new() diff --git a/hathor/consensus/poa/__init__.py b/hathor/consensus/poa/__init__.py new file mode 100644 index 000000000..084ceec8a --- /dev/null +++ b/hathor/consensus/poa/__init__.py @@ -0,0 +1,30 @@ +from .poa import ( + BLOCK_WEIGHT_IN_TURN, + BLOCK_WEIGHT_OUT_OF_TURN, + SIGNER_ID_LEN, + InvalidSignature, + ValidSignature, + calculate_weight, + get_active_signers, + get_hashed_poa_data, + get_signer_index_distance, + verify_poa_signature, +) +from .poa_block_producer import PoaBlockProducer +from .poa_signer import PoaSigner, PoaSignerFile + +__all__ = [ + 'BLOCK_WEIGHT_IN_TURN', + 'BLOCK_WEIGHT_OUT_OF_TURN', + 'SIGNER_ID_LEN', + 'get_hashed_poa_data', + 'calculate_weight', + 'PoaBlockProducer', + 'PoaSigner', + 'PoaSignerFile', + 'verify_poa_signature', + 'InvalidSignature', + 'ValidSignature', + 'get_active_signers', + 'get_signer_index_distance', +] diff --git a/hathor/consensus/poa/poa.py b/hathor/consensus/poa/poa.py new file mode 100644 index 000000000..03ecbf3a5 --- /dev/null +++ b/hathor/consensus/poa/poa.py @@ -0,0 +1,107 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from cryptography.exceptions import InvalidSignature as CryptographyInvalidSignature +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec + +from hathor.consensus.consensus_settings import PoaSettings +from hathor.crypto.util import get_public_key_from_bytes_compressed +from hathor.transaction import Block + +if TYPE_CHECKING: + from hathor.transaction.poa import PoaBlock + +BLOCK_WEIGHT_IN_TURN = 2.0 +BLOCK_WEIGHT_OUT_OF_TURN = 1.0 +SIGNER_ID_LEN = 2 + + +def get_hashed_poa_data(block: PoaBlock) -> bytes: + """Get the data to be signed for the Proof-of-Authority.""" + poa_data = block.get_funds_struct() + poa_data += Block.get_graph_struct(block) # We call Block's to exclude poa fields + poa_data += block.get_struct_nonce() + hashed_poa_data = hashlib.sha256(poa_data).digest() + return hashed_poa_data + + +def get_active_signers(settings: PoaSettings, height: int) -> list[bytes]: + """Return a list of signers that are currently active considering the given block height.""" + active_signers = [] + for signer_settings in settings.signers: + end_height = float('inf') if signer_settings.end_height is None else signer_settings.end_height + + if signer_settings.start_height <= height <= end_height: + active_signers.append(signer_settings.public_key) + + return active_signers + + +def get_signer_index_distance(*, settings: PoaSettings, signer_index: int, height: int) -> int: + """Considering a block height, return the signer index distance to that block. When the distance is 0, it means it + is the signer's turn.""" + active_signers = get_active_signers(settings, height) + expected_index = height % len(active_signers) + signers = get_active_signers(settings, height) + index_distance = (signer_index - expected_index) % len(signers) + assert 0 <= index_distance < len(signers) + return index_distance + + +def calculate_weight(settings: PoaSettings, block: PoaBlock, signer_index: int) -> float: + """Return the weight for the given block and signer.""" + index_distance = get_signer_index_distance(settings=settings, signer_index=signer_index, height=block.get_height()) + return BLOCK_WEIGHT_IN_TURN if index_distance == 0 else BLOCK_WEIGHT_OUT_OF_TURN / index_distance + + +@dataclass(frozen=True, slots=True) +class InvalidSignature: + pass + + +@dataclass(frozen=True, slots=True) +class ValidSignature: + signer_index: int + public_key: bytes + + +def verify_poa_signature(settings: PoaSettings, block: PoaBlock) -> InvalidSignature | ValidSignature: + """Return whether the provided public key was used to sign the block Proof-of-Authority.""" + from hathor.consensus.poa import PoaSigner + active_signers = get_active_signers(settings, block.get_height()) + hashed_poa_data = get_hashed_poa_data(block) + + for signer_index, public_key_bytes in enumerate(active_signers): + signer_id = PoaSigner.get_poa_signer_id(public_key_bytes) + if block.signer_id != signer_id: + # this is not our signer + continue + + public_key = get_public_key_from_bytes_compressed(public_key_bytes) + try: + public_key.verify(block.signature, hashed_poa_data, ec.ECDSA(hashes.SHA256())) + except CryptographyInvalidSignature: + # the signer_id is correct, but not the signature + continue + # the signer and signature are valid! + return ValidSignature(signer_index, public_key_bytes) + + return InvalidSignature() diff --git a/hathor/consensus/poa/poa_block_producer.py b/hathor/consensus/poa/poa_block_producer.py new file mode 100644 index 000000000..267e7792c --- /dev/null +++ b/hathor/consensus/poa/poa_block_producer.py @@ -0,0 +1,209 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from structlog import get_logger +from twisted.internet.interfaces import IDelayedCall +from twisted.internet.task import LoopingCall + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings +from hathor.crypto.util import get_public_key_bytes_compressed +from hathor.pubsub import EventArguments, HathorEvents +from hathor.reactor import ReactorProtocol +from hathor.util import not_none + +if TYPE_CHECKING: + from hathor.consensus.poa import PoaSigner + from hathor.manager import HathorManager + from hathor.transaction import Block + from hathor.transaction.poa import PoaBlock + +logger = get_logger() + +# Number of seconds used between each signer depending on its distance to the expected signer +_SIGNER_TURN_INTERVAL: int = 10 + + +class PoaBlockProducer: + """ + This class is analogous to mining classes, but for Proof-of-Authority networks. + It waits for blocks to arrive, gets templates, and propagates new blocks accordingly. + """ + __slots__ = ( + '_log', + '_settings', + '_poa_settings', + '_reactor', + '_manager', + '_poa_signer', + '_last_seen_best_block', + '_delayed_call', + '_start_producing_lc', + ) + + def __init__(self, *, settings: HathorSettings, reactor: ReactorProtocol, poa_signer: PoaSigner) -> None: + assert isinstance(settings.CONSENSUS_ALGORITHM, PoaSettings) + self._log = logger.new() + self._settings = settings + self._poa_settings = settings.CONSENSUS_ALGORITHM + self._reactor = reactor + self._manager: HathorManager | None = None + self._poa_signer = poa_signer + self._last_seen_best_block: Block | None = None + self._delayed_call: IDelayedCall | None = None + self._start_producing_lc: LoopingCall = LoopingCall(self._safe_start_producing) + self._start_producing_lc.clock = self._reactor + + @property + def manager(self) -> HathorManager: + assert self._manager is not None + return self._manager + + @manager.setter + def manager(self, manager: HathorManager) -> None: + self._manager = manager + + def start(self) -> None: + self.manager.pubsub.subscribe(HathorEvents.NETWORK_NEW_TX_ACCEPTED, self._on_new_vertex) + self._start_producing_lc.start(self._settings.AVG_TIME_BETWEEN_BLOCKS) + + def stop(self) -> None: + self.manager.pubsub.unsubscribe(HathorEvents.NETWORK_NEW_TX_ACCEPTED, self._on_new_vertex) + + if self._delayed_call and self._delayed_call.active(): + self._delayed_call.cancel() + + if self._start_producing_lc.running: + self._start_producing_lc.stop() + + def _get_signer_index(self, previous_block: Block) -> int | None: + """Return our signer index considering the active signers.""" + height = previous_block.get_height() + 1 + public_key = self._poa_signer.get_public_key() + public_key_bytes = get_public_key_bytes_compressed(public_key) + active_signers = poa.get_active_signers(self._poa_settings, height) + try: + return active_signers.index(public_key_bytes) + except ValueError: + return None + + def _safe_start_producing(self) -> None: + try: + return self._unsafe_start_producing() + except Exception: + self._log.exception('error while trying to start block production') + + def _unsafe_start_producing(self) -> None: + """Start producing new blocks.""" + if not self.manager.can_start_mining(): + # We're syncing, so we'll try again later + self._log.warn('cannot start producing new blocks, node not synced') + return + + self._log.info('started producing new blocks') + self._schedule_block() + + def _on_new_vertex(self, event: HathorEvents, args: EventArguments) -> None: + """Handle propagation of new blocks after a vertex is received.""" + assert event is HathorEvents.NETWORK_NEW_TX_ACCEPTED + block = args.tx + + from hathor.transaction import Block + if not isinstance(block, Block): + return + + from hathor.transaction.poa import PoaBlock + if isinstance(block, PoaBlock) and not block.weight == poa.BLOCK_WEIGHT_IN_TURN: + self._log.info('received out of turn block', block=block.hash_hex, signer_id=block.signer_id) + + self._schedule_block() + + def _schedule_block(self) -> None: + """Schedule propagation of a new block.""" + if not self.manager.can_start_mining(): + # We're syncing, so we'll try again later + self._log.info('cannot produce new block, node not synced') + return + + if self._start_producing_lc.running: + self._start_producing_lc.stop() + + previous_block = self.manager.tx_storage.get_best_block() + if previous_block == self._last_seen_best_block: + return + + self._last_seen_best_block = previous_block + signer_index = self._get_signer_index(previous_block) + if signer_index is None: + return + + now = self._reactor.seconds() + expected_timestamp = self._expected_block_timestamp(previous_block, signer_index) + propagation_delay = 0 if expected_timestamp < now else expected_timestamp - now + + if self._delayed_call and self._delayed_call.active(): + self._delayed_call.cancel() + + self._delayed_call = self._reactor.callLater(propagation_delay, self._produce_block, previous_block) + + self._log.debug( + 'scheduling block production', + previous_block=previous_block.hash_hex, + previous_block_height=previous_block.get_height(), + delay=propagation_delay, + ) + + def _produce_block(self, previous_block: PoaBlock) -> None: + """Create and propagate a new block.""" + from hathor.transaction.poa import PoaBlock + block_templates = self.manager.get_block_templates(parent_block_hash=previous_block.hash) + block = block_templates.generate_mining_block(self.manager.rng, cls=PoaBlock) + assert isinstance(block, PoaBlock) + + if block.get_height() <= self.manager.tx_storage.get_height_best_block(): + return + + signer_index = self._get_signer_index(previous_block) + block.weight = poa.calculate_weight(self._poa_settings, block, not_none(signer_index)) + self._poa_signer.sign_block(block) + block.update_hash() + + self._log.info( + 'produced new block', + block=block.hash_hex, + height=block.get_height(), + weight=block.weight, + parent=block.get_block_parent_hash().hex(), + voided=bool(block.get_metadata().voided_by), + ) + self.manager.on_new_tx(block, propagate_to_peers=True, fails_silently=False) + + def _expected_block_timestamp(self, previous_block: Block, signer_index: int) -> int: + """Calculate the expected timestamp for a new block.""" + height = previous_block.get_height() + 1 + index_distance = poa.get_signer_index_distance( + settings=self._poa_settings, + signer_index=signer_index, + height=height, + ) + delay = _SIGNER_TURN_INTERVAL * index_distance + if index_distance > 0: + # if it's not our turn, we add a constant offset to the delay + delay += self._settings.AVG_TIME_BETWEEN_BLOCKS + return previous_block.timestamp + self._settings.AVG_TIME_BETWEEN_BLOCKS + delay diff --git a/hathor/consensus/poa/poa_signer.py b/hathor/consensus/poa/poa_signer.py new file mode 100644 index 000000000..689ecee71 --- /dev/null +++ b/hathor/consensus/poa/poa_signer.py @@ -0,0 +1,114 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING, Any, NewType + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from pydantic import Field, validator + +from hathor.consensus import poa +from hathor.crypto.util import ( + get_address_b58_from_public_key, + get_private_key_from_bytes, + get_public_key_bytes_compressed, +) +from hathor.utils.pydantic import BaseModel + +if TYPE_CHECKING: + from hathor.transaction.poa import PoaBlock + + +class PoaSignerFile(BaseModel, arbitrary_types_allowed=True): + """Class that represents a Proof-of-Authority signer configuration file.""" + private_key: ec.EllipticCurvePrivateKeyWithSerialization = Field(alias='private_key_hex') + public_key: ec.EllipticCurvePublicKey = Field(alias='public_key_hex') + address: str + + @validator('private_key', pre=True) + def _parse_private_key(cls, private_key_hex: str) -> ec.EllipticCurvePrivateKeyWithSerialization: + """Parse a private key hex into a private key instance.""" + private_key_bytes = bytes.fromhex(private_key_hex) + return get_private_key_from_bytes(private_key_bytes) + + @validator('public_key', pre=True) + def _validate_public_key_first_bytes( + cls, + public_key_hex: str, + values: dict[str, Any] + ) -> ec.EllipticCurvePublicKey: + """Parse a public key hex into a public key instance, and validate that it corresponds to the private key.""" + private_key = values.get('private_key') + assert isinstance(private_key, ec.EllipticCurvePrivateKey), 'private_key must be set' + + public_key_bytes = bytes.fromhex(public_key_hex) + actual_public_key = private_key.public_key() + + if public_key_bytes != get_public_key_bytes_compressed(actual_public_key): + raise ValueError('invalid public key') + + return actual_public_key + + @validator('address') + def _validate_address(cls, address: str, values: dict[str, Any]) -> str: + """Validate that the provided address corresponds to the provided private key.""" + private_key = values.get('private_key') + assert isinstance(private_key, ec.EllipticCurvePrivateKey), 'private_key must be set' + + if address != get_address_b58_from_public_key(private_key.public_key()): + raise ValueError('invalid address') + + return address + + def get_signer(self) -> PoaSigner: + """Get a PoaSigner for this file.""" + return PoaSigner(self.private_key) + + +""" +The `PoaSignerId` is the first 2 bytes of the hashed public key of a signer(see `PoaSigner.get_poa_signer_id()`). +It is a non-unique ID that represents a signer and exists simply to skip unnecessary signature verifications during the +verification process of PoA blocks. +""" +PoaSignerId = NewType('PoaSignerId', bytes) + + +class PoaSigner: + """Class that represents a Proof-of-Authority signer.""" + __slots__ = ('_private_key', '_signer_id') + + def __init__(self, private_key: ec.EllipticCurvePrivateKey) -> None: + self._private_key = private_key + public_key_bytes = get_public_key_bytes_compressed(private_key.public_key()) + self._signer_id = self.get_poa_signer_id(public_key_bytes) + + def sign_block(self, block: PoaBlock) -> None: + """Sign the Proof-of-Authority for a block.""" + hashed_poa_data = poa.get_hashed_poa_data(block) + signature = self._private_key.sign(hashed_poa_data, ec.ECDSA(hashes.SHA256())) + block.signer_id = self._signer_id + block.signature = signature + + def get_public_key(self) -> ec.EllipticCurvePublicKey: + """Return this signer's public key.""" + return self._private_key.public_key() + + @staticmethod + def get_poa_signer_id(compressed_public_key_bytes: bytes) -> PoaSignerId: + """Get the PoaSignerId from the compressed public key bytes.""" + hashed_public_key = hashlib.sha256(compressed_public_key_bytes).digest() + return PoaSignerId(hashed_public_key[:poa.SIGNER_ID_LEN]) diff --git a/hathor/consensus/transaction_consensus.py b/hathor/consensus/transaction_consensus.py index 358503c78..311ebd9d7 100644 --- a/hathor/consensus/transaction_consensus.py +++ b/hathor/consensus/transaction_consensus.py @@ -17,7 +17,6 @@ from structlog import get_logger from hathor.conf.get_settings import get_global_settings -from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, Block, Transaction, TxInput, sum_weights from hathor.util import classproperty @@ -25,7 +24,6 @@ from hathor.consensus.context import ConsensusAlgorithmContext logger = get_logger() -cpu = get_cpu_profiler() _base_transaction_log = logger.new() diff --git a/hathor/event/event_manager.py b/hathor/event/event_manager.py index 748abe90a..6ce402b82 100644 --- a/hathor/event/event_manager.py +++ b/hathor/event/event_manager.py @@ -46,6 +46,7 @@ HathorEvents.REORG_STARTED, HathorEvents.REORG_FINISHED, HathorEvents.CONSENSUS_TX_UPDATE, + HathorEvents.CONSENSUS_TX_REMOVED, ] diff --git a/hathor/event/model/event_data.py b/hathor/event/model/event_data.py index 632d124a7..a0faee890 100644 --- a/hathor/event/model/event_data.py +++ b/hathor/event/model/event_data.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Union, cast +from typing import Optional, TypeAlias, Union, cast from pydantic import Extra, validator +from typing_extensions import Self from hathor.pubsub import EventArguments from hathor.utils.pydantic import BaseModel @@ -137,6 +138,14 @@ def from_event_arguments(cls, args: EventArguments) -> 'TxData': return cls(**tx_json) +class VertexIdData(BaseEventData): + vertex_id: str + + @classmethod + def from_event_arguments(cls, args: EventArguments) -> Self: + return cls(vertex_id=args.vertex_id.hex()) + + class ReorgData(BaseEventData): """Class that represents reorg data on an event.""" reorg_size: int @@ -155,4 +164,4 @@ def from_event_arguments(cls, args: EventArguments) -> 'ReorgData': # Union type to encompass BaseEventData polymorphism -EventData = Union[EmptyData, TxData, ReorgData] +EventData: TypeAlias = EmptyData | TxData | ReorgData | VertexIdData diff --git a/hathor/event/model/event_type.py b/hathor/event/model/event_type.py index 617ea74d8..c251e704c 100644 --- a/hathor/event/model/event_type.py +++ b/hathor/event/model/event_type.py @@ -14,7 +14,7 @@ from enum import Enum -from hathor.event.model.event_data import BaseEventData, EmptyData, ReorgData, TxData +from hathor.event.model.event_data import BaseEventData, EmptyData, ReorgData, TxData, VertexIdData from hathor.pubsub import HathorEvents @@ -25,6 +25,7 @@ class EventType(Enum): REORG_STARTED = 'REORG_STARTED' REORG_FINISHED = 'REORG_FINISHED' VERTEX_METADATA_CHANGED = 'VERTEX_METADATA_CHANGED' + VERTEX_REMOVED = 'VERTEX_REMOVED' FULL_NODE_CRASHED = 'FULL_NODE_CRASHED' @classmethod @@ -44,7 +45,8 @@ def data_type(self) -> type[BaseEventData]: HathorEvents.NETWORK_NEW_TX_ACCEPTED: EventType.NEW_VERTEX_ACCEPTED, HathorEvents.REORG_STARTED: EventType.REORG_STARTED, HathorEvents.REORG_FINISHED: EventType.REORG_FINISHED, - HathorEvents.CONSENSUS_TX_UPDATE: EventType.VERTEX_METADATA_CHANGED + HathorEvents.CONSENSUS_TX_UPDATE: EventType.VERTEX_METADATA_CHANGED, + HathorEvents.CONSENSUS_TX_REMOVED: EventType.VERTEX_REMOVED } _EVENT_TYPE_TO_EVENT_DATA: dict[EventType, type[BaseEventData]] = { @@ -54,5 +56,6 @@ def data_type(self) -> type[BaseEventData]: EventType.REORG_STARTED: ReorgData, EventType.REORG_FINISHED: EmptyData, EventType.VERTEX_METADATA_CHANGED: TxData, + EventType.VERTEX_REMOVED: VertexIdData, EventType.FULL_NODE_CRASHED: EmptyData, } diff --git a/hathor/indexes/base_index.py b/hathor/indexes/base_index.py index bc9195009..98b1c0721 100644 --- a/hathor/indexes/base_index.py +++ b/hathor/indexes/base_index.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Optional @@ -22,6 +24,7 @@ from hathor.transaction.base_transaction import BaseTransaction if TYPE_CHECKING: # pragma: no cover + from hathor.conf.settings import HathorSettings from hathor.indexes.manager import IndexesManager logger = get_logger() @@ -33,8 +36,8 @@ class BaseIndex(ABC): This class exists so we can interact with indexes without knowing anything specific to its implemented. It was created to generalize how we initialize indexes and keep track of which ones are up-to-date. """ - def __init__(self) -> None: - self._settings = get_global_settings() + def __init__(self, *, settings: HathorSettings | None = None) -> None: + self._settings = settings or get_global_settings() self.log = logger.new() def init_start(self, indexes_manager: 'IndexesManager') -> None: diff --git a/hathor/indexes/manager.py b/hathor/indexes/manager.py index e681f716b..d67aa3776 100644 --- a/hathor/indexes/manager.py +++ b/hathor/indexes/manager.py @@ -19,6 +19,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes.address_index import AddressIndex from hathor.indexes.base_index import BaseIndex from hathor.indexes.height_index import HeightIndex @@ -259,7 +260,7 @@ def del_tx(self, tx: BaseTransaction, *, remove_all: bool = False, relax_assert: class MemoryIndexesManager(IndexesManager): - def __init__(self) -> None: + def __init__(self, *, settings: HathorSettings | None = None) -> None: from hathor.indexes.memory_height_index import MemoryHeightIndex from hathor.indexes.memory_info_index import MemoryInfoIndex from hathor.indexes.memory_timestamp_index import MemoryTimestampIndex @@ -277,7 +278,7 @@ def __init__(self) -> None: self.addresses = None self.tokens = None self.utxo = None - self.height = MemoryHeightIndex() + self.height = MemoryHeightIndex(settings=settings) self.mempool_tips = None # XXX: this has to be at the end of __init__, after everything has been initialized @@ -347,7 +348,6 @@ def enable_utxo_index(self) -> None: self.utxo = RocksDBUtxoIndex(self._db) def enable_mempool_index(self) -> None: - from hathor.indexes.memory_mempool_tips_index import MemoryMempoolTipsIndex + from hathor.indexes.rocksdb_mempool_tips_index import RocksDBMempoolTipsIndex if self.mempool_tips is None: - # XXX: use of RocksDBMempoolTipsIndex is very slow and was suspended - self.mempool_tips = MemoryMempoolTipsIndex() + self.mempool_tips = RocksDBMempoolTipsIndex(self._db) diff --git a/hathor/indexes/memory_height_index.py b/hathor/indexes/memory_height_index.py index 50bf03004..18a0546ae 100644 --- a/hathor/indexes/memory_height_index.py +++ b/hathor/indexes/memory_height_index.py @@ -14,6 +14,7 @@ from typing import Optional +from hathor.conf.settings import HathorSettings from hathor.indexes.height_index import HeightIndex, HeightInfo, IndexEntry @@ -23,8 +24,8 @@ class MemoryHeightIndex(HeightIndex): _index: list[IndexEntry] - def __init__(self) -> None: - super().__init__() + def __init__(self, *, settings: HathorSettings | None = None) -> None: + super().__init__(settings=settings) self.force_clear() def get_db_name(self) -> Optional[str]: diff --git a/hathor/indexes/mempool_tips_index.py b/hathor/indexes/mempool_tips_index.py index 222cc8140..290b4f865 100644 --- a/hathor/indexes/mempool_tips_index.py +++ b/hathor/indexes/mempool_tips_index.py @@ -14,6 +14,7 @@ from abc import abstractmethod from collections.abc import Collection +from itertools import chain from typing import TYPE_CHECKING, Iterable, Iterator, Optional, cast import structlog @@ -27,22 +28,33 @@ from hathor.transaction.storage import TransactionStorage SCOPE = Scope( - include_blocks=True, + include_blocks=False, include_txs=True, include_voided=True, + topological_order=False, ) +def any_non_voided(tx_storage: 'TransactionStorage', hashes: Iterable[bytes]) -> bool: + """ + If there's any vertex with hash in hashes that is not voided, this function returns True, otherwise False. + + Notice that this means that an empty `hashes` also returns False. + """ + for tx_hash in hashes: + tx = tx_storage.get_transaction(tx_hash) + tx_meta = tx.get_metadata() + if not tx_meta.voided_by: + return True + return False + + class MempoolTipsIndex(BaseIndex): """Index to access the tips of the mempool transactions, which haven't been confirmed by a block.""" def get_scope(self) -> Scope: return SCOPE - def init_loop_step(self, tx: BaseTransaction) -> None: - self.update(tx) - - # originally tx_storage.update_mempool_tips @abstractmethod def update(self, tx: BaseTransaction, *, remove: Optional[bool] = None) -> None: """ @@ -52,7 +64,6 @@ def update(self, tx: BaseTransaction, *, remove: Optional[bool] = None) -> None: """ raise NotImplementedError - # originally tx_storage.iter_mempool_tips @abstractmethod def iter(self, tx_storage: 'TransactionStorage', max_timestamp: Optional[float] = None) -> Iterator[Transaction]: """ @@ -60,7 +71,6 @@ def iter(self, tx_storage: 'TransactionStorage', max_timestamp: Optional[float] """ raise NotImplementedError - # originally tx_storage.iter_mempool @abstractmethod def iter_all(self, tx_storage: 'TransactionStorage') -> Iterator[Transaction]: """ @@ -68,7 +78,6 @@ def iter_all(self, tx_storage: 'TransactionStorage') -> Iterator[Transaction]: """ raise NotImplementedError - # originally tx_storage.get_mempool_tips_index @abstractmethod def get(self) -> set[bytes]: """ @@ -85,6 +94,23 @@ class ByteCollectionMempoolTipsIndex(MempoolTipsIndex): log: 'structlog.stdlib.BoundLogger' _index: 'Collection[bytes]' + def init_loop_step(self, tx: BaseTransaction) -> None: + assert tx.hash is not None + assert tx.storage is not None + tx_meta = tx.get_metadata() + # do not include transactions that have been confirmed + if tx_meta.first_block: + return + tx_storage = tx.storage + # do not include transactions that have a non-voided child + if any_non_voided(tx_storage, tx_meta.children): + return + # do not include transactions that have a non-voided spent output + if any_non_voided(tx_storage, chain(*tx_meta.spent_outputs.values())): + return + # include them otherwise + self._add(tx.hash) + @abstractmethod def _discard(self, tx: bytes) -> None: raise NotImplementedError() diff --git a/hathor/manager.py b/hathor/manager.py index 354f7aaf1..9f1210d40 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -29,6 +29,7 @@ from hathor.checkpoint import Checkpoint from hathor.conf.settings import HathorSettings from hathor.consensus import ConsensusAlgorithm +from hathor.consensus.poa import PoaBlockProducer from hathor.daa import DifficultyAdjustmentAlgorithm from hathor.event.event_manager import EventManager from hathor.exception import ( @@ -56,6 +57,7 @@ from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.transaction_storage import TransactionStorage from hathor.transaction.storage.tx_allow_scope import TxAllowScope +from hathor.transaction.vertex_parser import VertexParser from hathor.types import Address, VertexId from hathor.util import EnvironmentInfo, LogDuration, Random, calculate_min_significant_weight, not_none from hathor.verification.verification_service import VerificationService @@ -104,6 +106,7 @@ def __init__( network: str, execution_manager: ExecutionManager, vertex_handler: VertexHandler, + vertex_parser: VertexParser, hostname: Optional[str] = None, wallet: Optional[BaseWallet] = None, capabilities: Optional[list[str]] = None, @@ -112,6 +115,7 @@ def __init__( environment_info: Optional[EnvironmentInfo] = None, full_verification: bool = False, enable_event_queue: bool = False, + poa_block_producer: PoaBlockProducer | None = None, ) -> None: """ :param reactor: Twisted reactor which handles the mainloop and the events. @@ -192,6 +196,7 @@ def __init__( self.connections = p2p_manager self.vertex_handler = vertex_handler + self.vertex_parser = vertex_parser self.metrics = Metrics( pubsub=self.pubsub, @@ -231,6 +236,8 @@ def __init__( # This is included in some logs to provide more context self.environment_info = environment_info + self.poa_block_producer = poa_block_producer + # Task that will count the total sync time self.lc_check_sync_state = LoopingCall(self.check_sync_state) self.lc_check_sync_state.clock = self.reactor @@ -329,6 +336,9 @@ def start(self) -> None: if self.stratum_factory: self.stratum_factory.start() + if self.poa_block_producer: + self.poa_block_producer.start() + # Start running self.tx_storage.start_running_manager(self._execution_manager) @@ -363,6 +373,9 @@ def stop(self) -> Deferred: if self._enable_event_queue: self._event_manager.stop() + if self.poa_block_producer: + self.poa_block_producer.stop() + self.tx_storage.flush() return defer.DeferredList(waits) @@ -862,7 +875,7 @@ def generate_mining_block(self, timestamp: Optional[int] = None, assert address is not None block = self.get_block_templates(parent_block_hash, timestamp).generate_mining_block( rng=self.rng, - merge_mined=merge_mined, + cls=MergeMinedBlock if merge_mined else Block, address=address or None, # XXX: because we allow b'' for explicit empty output script data=data, ) @@ -895,6 +908,9 @@ def push_tx(self, tx: Transaction, allow_non_standard_script: bool = False, max_output_script_size: int | None = None) -> None: """Used by all APIs that accept a new transaction (like push_tx) """ + if self.tx_storage.transaction_exists(tx.hash): + raise InvalidNewTransaction('Transaction already exists {}'.format(tx.hash_hex)) + if max_output_script_size is None: max_output_script_size = self._settings.PUSHTX_MAX_OUTPUT_SCRIPT_SIZE diff --git a/hathor/merged_mining/coordinator.py b/hathor/merged_mining/coordinator.py index 9a191a47b..f65a181da 100644 --- a/hathor/merged_mining/coordinator.py +++ b/hathor/merged_mining/coordinator.py @@ -45,7 +45,7 @@ ) from hathor.merged_mining.bitcoin_rpc import IBitcoinRPC from hathor.merged_mining.util import Periodic -from hathor.transaction import BitcoinAuxPow, MergeMinedBlock as HathorBlock +from hathor.transaction import BitcoinAuxPow, MergeMinedBlock, MergeMinedBlock as HathorBlock from hathor.transaction.exceptions import ScriptError, TxValidationError from hathor.util import MaxSizeOrderedDict, Random, ichunks @@ -1296,7 +1296,7 @@ async def _update_hathor_block(self, mining: IMiningChannel) -> None: block_template = block_templates.choose_random_template(self.rng) address_str = self.payback_address_hathor address = decode_address(address_str) if address_str is not None else None - block = block_template.generate_mining_block(self.rng, merge_mined=True, address=address) + block = block_template.generate_mining_block(self.rng, address=address, cls=MergeMinedBlock) height = block_template.height assert isinstance(block, HathorBlock) self.last_hathor_block_received = time.time() diff --git a/hathor/metrics.py b/hathor/metrics.py index 64ee2bd08..92b73d3ad 100644 --- a/hathor/metrics.py +++ b/hathor/metrics.py @@ -252,7 +252,7 @@ def collect_peer_connection_metrics(self) -> None: continue metric = PeerConnectionMetrics( - connection_string=connection.connection_string if connection.connection_string else "", + connection_string=str(connection.entrypoint) if connection.entrypoint else "", peer_id=connection.peer.id, network=connection.network, received_messages=connection.metrics.received_messages, diff --git a/hathor/mining/block_template.py b/hathor/mining/block_template.py index 54e690d95..c1ef07f60 100644 --- a/hathor/mining/block_template.py +++ b/hathor/mining/block_template.py @@ -16,12 +16,15 @@ Module for abstractions around generating mining templates. """ -from typing import Iterable, NamedTuple, Optional, Union +from typing import Iterable, NamedTuple, Optional, TypeVar, cast from hathor.transaction import BaseTransaction, Block, MergeMinedBlock +from hathor.transaction.poa import PoaBlock from hathor.transaction.storage import TransactionStorage from hathor.util import Random +T = TypeVar('T', bound=Block) + class BlockTemplate(NamedTuple): versions: set[int] @@ -47,10 +50,16 @@ def generate_minimally_valid_block(self) -> BaseTransaction: signal_bits=self.signal_bits, ) - def generate_mining_block(self, rng: Random, merge_mined: bool = False, address: Optional[bytes] = None, - timestamp: Optional[int] = None, data: Optional[bytes] = None, - storage: Optional[TransactionStorage] = None, include_metadata: bool = False - ) -> Union[Block, MergeMinedBlock]: + def generate_mining_block( + self, + rng: Random, + address: Optional[bytes] = None, + timestamp: Optional[int] = None, + data: Optional[bytes] = None, + storage: Optional[TransactionStorage] = None, + include_metadata: bool = False, + cls: type[T] | None = None, + ) -> T: """ Generates a block by filling the template with the given options and random parents (if multiple choices). Note that if a timestamp is given it will be coerced into the [timestamp_min, timestamp_max] range. @@ -60,11 +69,14 @@ def generate_mining_block(self, rng: Random, merge_mined: bool = False, address: from hathor.transaction.scripts import create_output_script parents = list(self.get_random_parents(rng)) - output_script = create_output_script(address) if address is not None else b'' base_timestamp = timestamp if timestamp is not None else self.timestamp_now block_timestamp = min(max(base_timestamp, self.timestamp_min), self.timestamp_max) - tx_outputs = [TxOutput(self.reward, output_script)] - cls: Union[type['Block'], type['MergeMinedBlock']] = MergeMinedBlock if merge_mined else Block + tx_outputs = [] + if self.reward: + output_script = create_output_script(address) if address is not None else b'' + tx_outputs = [TxOutput(self.reward, output_script)] + if cls is None: + cls = cast(type[T], Block) block = cls(outputs=tx_outputs, parents=parents, timestamp=block_timestamp, data=data or b'', storage=storage, weight=self.weight, signal_bits=self.signal_bits) if include_metadata: @@ -125,12 +137,23 @@ def choose_random_template(self, rng: Random) -> BlockTemplate: """ Randomly choose and return a template and use that for generating a block, see BlockTemplate""" return rng.choice(self) - def generate_mining_block(self, rng: Random, merge_mined: bool = False, address: Optional[bytes] = None, - timestamp: Optional[int] = None, data: Optional[bytes] = None, - storage: Optional[TransactionStorage] = None, include_metadata: bool = False, - signal_bits: int = 0) -> Union[Block, MergeMinedBlock]: + def generate_mining_block( + self, + rng: Random, + address: Optional[bytes] = None, + timestamp: Optional[int] = None, + data: Optional[bytes] = None, + storage: Optional[TransactionStorage] = None, + include_metadata: bool = False, + cls: type[T] | None = None, + ) -> Block | MergeMinedBlock | PoaBlock: """ Randomly choose a template and use that for generating a block, see BlockTemplate.generate_mining_block""" - return self.choose_random_template(rng).generate_mining_block(rng, merge_mined=merge_mined, address=address, - timestamp=timestamp, data=data, - storage=storage or self.storage, - include_metadata=include_metadata) + return self.choose_random_template(rng).generate_mining_block( + rng, + address=address, + timestamp=timestamp, + data=data, + storage=storage or self.storage, + include_metadata=include_metadata, + cls=cls, + ) diff --git a/hathor/mining/ws.py b/hathor/mining/ws.py index e4839f525..d8189a480 100644 --- a/hathor/mining/ws.py +++ b/hathor/mining/ws.py @@ -24,7 +24,6 @@ from hathor.manager import HathorManager from hathor.pubsub import EventArguments, HathorEvents -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.util import json_dumpb, json_loadb logger = get_logger() @@ -163,7 +162,10 @@ def do_mining_submit(self, hexdata: str, optimistic: bool = False) -> Union[bool if not self.factory.manager.can_start_mining(): self.log.warn('node syncing') return False - tx = tx_or_block_from_bytes(bytes.fromhex(hexdata), storage=self.factory.manager.tx_storage) + tx = self.factory.manager.vertex_parser.deserialize( + bytes.fromhex(hexdata), + storage=self.factory.manager.tx_storage + ) if not tx.is_block: self.log.warn('expected Block, received Transaction', data=hexdata) return False diff --git a/hathor/p2p/entrypoint.py b/hathor/p2p/entrypoint.py new file mode 100644 index 000000000..3340f784d --- /dev/null +++ b/hathor/p2p/entrypoint.py @@ -0,0 +1,219 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from enum import Enum +from urllib.parse import parse_qs, urlparse + +from twisted.internet.address import IPv4Address, IPv6Address +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.interfaces import IStreamClientEndpoint +from typing_extensions import Self + +from hathor.reactor import ReactorProtocol as Reactor +from hathor.types import Hash + + +class Protocol(Enum): + TCP = 'tcp' + + +class PeerId(Hash): + pass + + +@dataclass(frozen=True, slots=True) +class Entrypoint: + """Endpoint description (returned from DNS query, or received from the p2p network) may contain a peer-id.""" + + protocol: Protocol + host: str + port: int + peer_id: PeerId | None = None + + def __str__(self): + if self.peer_id is None: + return f'{self.protocol.value}://{self.host}:{self.port}' + else: + return f'{self.protocol.value}://{self.host}:{self.port}/?id={self.peer_id}' + + @classmethod + def parse(cls, description: str) -> Self: + """Parse endpoint description into an Entrypoint object. + + Examples: + + >>> str(Entrypoint.parse('tcp://127.0.0.1:40403/')) + 'tcp://127.0.0.1:40403' + + >>> id1 = 'c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + >>> Entrypoint.parse(f'tcp://127.0.0.1:40403/?id={id1}') + Entrypoint(protocol=, host='127.0.0.1', port=40403, \ +peer_id=PeerId('c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696')) + + >>> str(Entrypoint.parse(f'tcp://127.0.0.1:40403/?id={id1}')) + 'tcp://127.0.0.1:40403/?id=c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + + >>> Entrypoint.parse('tcp://127.0.0.1:40403') + Entrypoint(protocol=, host='127.0.0.1', port=40403, peer_id=None) + + >>> Entrypoint.parse('tcp://127.0.0.1:40403/') + Entrypoint(protocol=, host='127.0.0.1', port=40403, peer_id=None) + + >>> Entrypoint.parse('tcp://foo.bar.baz:40403/') + Entrypoint(protocol=, host='foo.bar.baz', port=40403, peer_id=None) + + >>> str(Entrypoint.parse('tcp://foo.bar.baz:40403/')) + 'tcp://foo.bar.baz:40403' + + >>> Entrypoint.parse('tcp://127.0.0.1:40403/?id=123') + Traceback (most recent call last): + ... + ValueError: non-hexadecimal number found in fromhex() arg at position 3 + + >>> Entrypoint.parse('tcp://127.0.0.1:4040f') + Traceback (most recent call last): + ... + ValueError: Port could not be cast to integer value as '4040f' + + >>> Entrypoint.parse('udp://127.0.0.1:40403/') + Traceback (most recent call last): + ... + ValueError: 'udp' is not a valid Protocol + + >>> Entrypoint.parse('tcp://127.0.0.1/') + Traceback (most recent call last): + ... + ValueError: expected a port + + >>> Entrypoint.parse('tcp://:40403/') + Traceback (most recent call last): + ... + ValueError: expected a host + + >>> Entrypoint.parse('tcp://127.0.0.1:40403/foo') + Traceback (most recent call last): + ... + ValueError: unexpected path: /foo + + >>> id2 = 'bc5119d47bb4ea7c19100bd97fb11f36970482108bd3d45ff101ee4f6bbec872' + >>> Entrypoint.parse(f'tcp://127.0.0.1:40403/?id={id1}&id={id2}') + Traceback (most recent call last): + ... + ValueError: unexpected id count: 2 + """ + url = urlparse(description) + protocol = Protocol(url.scheme) + host = url.hostname + if host is None: + raise ValueError('expected a host') + port = url.port + if port is None: + raise ValueError('expected a port') + if url.path not in {'', '/'}: + raise ValueError(f'unexpected path: {url.path}') + peer_id: PeerId | None = None + + if url.query: + query = parse_qs(url.query) + if 'id' in query: + ids = query['id'] + if len(ids) != 1: + raise ValueError(f'unexpected id count: {len(ids)}') + peer_id = PeerId(ids[0]) + + return cls(protocol, host, port, peer_id) + + @classmethod + def from_hostname_address(cls, hostname: str, address: IPv4Address | IPv6Address) -> Self: + return cls.parse(f'{address.type}://{hostname}:{address.port}') + + def to_client_endpoint(self, reactor: Reactor) -> IStreamClientEndpoint: + """This method generates a twisted client endpoint that has a .connect() method.""" + # XXX: currently we don't support IPv6, but when we do we have to decide between TCP4ClientEndpoint and + # TCP6ClientEndpoint, when the host is an IP address that is easy, but when it is a DNS hostname, we will not + # know which to use until we know which resource records it holds (A or AAAA) + return TCP4ClientEndpoint(reactor, self.host, self.port) + + def equals_ignore_peer_id(self, other: Self) -> bool: + """Compares `self` and `other` ignoring the `peer_id` fields of either. + + Examples: + + >>> ep1 = 'tcp://foo:111' + >>> ep2 = 'tcp://foo:111/?id=c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + >>> ep3 = 'tcp://foo:111/?id=bc5119d47bb4ea7c19100bd97fb11f36970482108bd3d45ff101ee4f6bbec872' + >>> ep4 = 'tcp://bar:111/?id=c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + >>> ep5 = 'tcp://foo:112/?id=c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + >>> Entrypoint.parse(ep1).equals_ignore_peer_id(Entrypoint.parse(ep2)) + True + >>> Entrypoint.parse(ep2).equals_ignore_peer_id(Entrypoint.parse(ep3)) + True + >>> Entrypoint.parse(ep1).equals_ignore_peer_id(Entrypoint.parse(ep4)) + False + >>> Entrypoint.parse(ep2).equals_ignore_peer_id(Entrypoint.parse(ep4)) + False + >>> Entrypoint.parse(ep2).equals_ignore_peer_id(Entrypoint.parse(ep5)) + False + """ + return (self.protocol, self.host, self.port) == (other.protocol, other.host, other.port) + + def peer_id_conflicts_with(self, other: Self) -> bool: + """Returns True if both self and other have a peer_id and they are different, returns False otherwise. + + This method ignores the host. Which is useful for catching the cases where both `self` and `other` have a + declared `peer_id` and they are not equal. + + >>> desc_no_pid = 'tcp://127.0.0.1:40403/' + >>> ep_no_pid = Entrypoint.parse(desc_no_pid) + >>> desc_pid1 = 'tcp://127.0.0.1:40403/?id=c0f19299c2a4dcbb6613a14011ff07b63d6cb809e4cee25e9c1ccccdd6628696' + >>> ep_pid1 = Entrypoint.parse(desc_pid1) + >>> desc_pid2 = 'tcp://127.0.0.1:40403/?id=bc5119d47bb4ea7c19100bd97fb11f36970482108bd3d45ff101ee4f6bbec872' + >>> ep_pid2 = Entrypoint.parse(desc_pid2) + >>> desc2_pid2 = 'tcp://foo.bar:40403/?id=bc5119d47bb4ea7c19100bd97fb11f36970482108bd3d45ff101ee4f6bbec872' + >>> ep2_pid2 = Entrypoint.parse(desc2_pid2) + >>> ep_no_pid.peer_id_conflicts_with(ep_no_pid) + False + >>> ep_no_pid.peer_id_conflicts_with(ep_pid1) + False + >>> ep_pid1.peer_id_conflicts_with(ep_no_pid) + False + >>> ep_pid1.peer_id_conflicts_with(ep_pid2) + True + >>> ep_pid1.peer_id_conflicts_with(ep2_pid2) + True + >>> ep_pid2.peer_id_conflicts_with(ep2_pid2) + False + """ + return self.peer_id is not None and other.peer_id is not None and self.peer_id != other.peer_id + + def is_localhost(self) -> bool: + """Used to determine if the entrypoint host is a localhost address. + + Examples: + + >>> Entrypoint.parse('tcp://127.0.0.1:444').is_localhost() + True + >>> Entrypoint.parse('tcp://localhost:444').is_localhost() + True + >>> Entrypoint.parse('tcp://8.8.8.8:444').is_localhost() + False + >>> Entrypoint.parse('tcp://foo.bar:444').is_localhost() + False + """ + if self.host == '127.0.0.1': + return True + if self.host == 'localhost': + return True + return False diff --git a/hathor/p2p/factory.py b/hathor/p2p/factory.py index da02328d3..a90cf2882 100644 --- a/hathor/p2p/factory.py +++ b/hathor/p2p/factory.py @@ -17,6 +17,7 @@ from twisted.internet import protocol from twisted.internet.interfaces import IAddress +from hathor.conf.settings import HathorSettings from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer_id import PeerId from hathor.p2p.protocol import HathorLineReceiver @@ -36,14 +37,16 @@ class HathorServerFactory(protocol.ServerFactory): protocol: type[MyServerProtocol] = MyServerProtocol def __init__( - self, - network: str, - my_peer: PeerId, - p2p_manager: ConnectionsManager, - *, - use_ssl: bool, + self, + network: str, + my_peer: PeerId, + p2p_manager: ConnectionsManager, + *, + settings: HathorSettings, + use_ssl: bool, ): super().__init__() + self._settings = settings self.network = network self.my_peer = my_peer self.p2p_manager = p2p_manager @@ -57,6 +60,7 @@ def buildProtocol(self, addr: IAddress) -> MyServerProtocol: p2p_manager=self.p2p_manager, use_ssl=self.use_ssl, inbound=True, + settings=self._settings ) p.factory = self return p @@ -69,14 +73,16 @@ class HathorClientFactory(protocol.ClientFactory): protocol: type[MyClientProtocol] = MyClientProtocol def __init__( - self, - network: str, - my_peer: PeerId, - p2p_manager: ConnectionsManager, - *, - use_ssl: bool, + self, + network: str, + my_peer: PeerId, + p2p_manager: ConnectionsManager, + *, + settings: HathorSettings, + use_ssl: bool, ): super().__init__() + self._settings = settings self.network = network self.my_peer = my_peer self.p2p_manager = p2p_manager @@ -90,6 +96,7 @@ def buildProtocol(self, addr: IAddress) -> MyClientProtocol: p2p_manager=self.p2p_manager, use_ssl=self.use_ssl, inbound=False, + settings=self._settings ) p.factory = self return p diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index d7e7f422b..b34254282 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -24,7 +24,8 @@ from twisted.python.failure import Failure from twisted.web.client import Agent -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.netfilter.factory import NetfilterFactory from hathor.p2p.peer_discovery import PeerDiscovery from hathor.p2p.peer_id import PeerId @@ -34,7 +35,7 @@ from hathor.p2p.states.ready import ReadyState from hathor.p2p.sync_factory import SyncAgentFactory from hathor.p2p.sync_version import SyncVersion -from hathor.p2p.utils import description_to_connection_string, parse_whitelist +from hathor.p2p.utils import parse_whitelist from hathor.pubsub import HathorEvents, PubSubManager from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction @@ -44,7 +45,6 @@ from hathor.manager import HathorManager logger = get_logger() -settings = get_global_settings() # The timeout in seconds for the whitelist GET request WHITELIST_REQUEST_TIMEOUT = 45 @@ -59,7 +59,7 @@ class _SyncRotateInfo(NamedTuple): class _ConnectingPeer(NamedTuple): - connection_string: str + entrypoint: Entrypoint endpoint_deferred: Deferred @@ -73,9 +73,6 @@ class PeerConnectionsMetrics(NamedTuple): class ConnectionsManager: """ It manages all peer-to-peer connections and events related to control messages. """ - MAX_ENABLED_SYNC = settings.MAX_ENABLED_SYNC - SYNC_UPDATE_INTERVAL = settings.SYNC_UPDATE_INTERVAL - PEER_DISCOVERY_INTERVAL = settings.PEER_DISCOVERY_INTERVAL class GlobalRateLimiter: SEND_TIPS = 'NodeSyncTimestamp.send_tips' @@ -91,18 +88,25 @@ class GlobalRateLimiter: rate_limiter: RateLimiter - def __init__(self, - reactor: Reactor, - network: str, - my_peer: PeerId, - pubsub: PubSubManager, - ssl: bool, - rng: Random, - whitelist_only: bool) -> None: + def __init__( + self, + settings: HathorSettings, + reactor: Reactor, + network: str, + my_peer: PeerId, + pubsub: PubSubManager, + ssl: bool, + rng: Random, + whitelist_only: bool, + ) -> None: self.log = logger.new() + self._settings = settings self.rng = rng self.manager = None - self._settings = get_global_settings() + + self.MAX_ENABLED_SYNC = settings.MAX_ENABLED_SYNC + self.SYNC_UPDATE_INTERVAL = settings.SYNC_UPDATE_INTERVAL + self.PEER_DISCOVERY_INTERVAL = settings.PEER_DISCOVERY_INTERVAL self.reactor = reactor self.my_peer = my_peer @@ -124,8 +128,12 @@ def __init__(self, # Factories. from hathor.p2p.factory import HathorClientFactory, HathorServerFactory self.use_ssl = ssl - self.server_factory = HathorServerFactory(self.network, self.my_peer, p2p_manager=self, use_ssl=self.use_ssl) - self.client_factory = HathorClientFactory(self.network, self.my_peer, p2p_manager=self, use_ssl=self.use_ssl) + self.server_factory = HathorServerFactory( + self.network, self.my_peer, p2p_manager=self, use_ssl=self.use_ssl, settings=self._settings + ) + self.client_factory = HathorClientFactory( + self.network, self.my_peer, p2p_manager=self, use_ssl=self.use_ssl, settings=self._settings + ) # Global maximum number of connections. self.max_connections: int = self._settings.PEER_MAX_CONNECTIONS @@ -360,8 +368,8 @@ def disconnect_all_peers(self, *, force: bool = False) -> None: def on_connection_failure(self, failure: Failure, peer: Optional[PeerId], endpoint: IStreamClientEndpoint) -> None: connecting_peer = self.connecting_peers[endpoint] - connection_string = connecting_peer.connection_string - self.log.warn('connection failure', endpoint=connection_string, failure=failure.getErrorMessage()) + entrypoint = connecting_peer.entrypoint + self.log.warn('connection failure', entrypoint=entrypoint, failure=failure.getErrorMessage()) self.connecting_peers.pop(endpoint) self.pubsub.publish( @@ -467,13 +475,13 @@ def iter_ready_connections(self) -> Iterable[HathorProtocol]: for conn in self.connected_peers.values(): yield conn - def iter_not_ready_endpoints(self) -> Iterable[str]: + def iter_not_ready_endpoints(self) -> Iterable[Entrypoint]: """Iterate over not-ready connections.""" for connecting_peer in self.connecting_peers.values(): - yield connecting_peer.connection_string + yield connecting_peer.entrypoint for protocol in self.handshaking_peers: - if protocol.connection_string is not None: - yield protocol.connection_string + if protocol.entrypoint is not None: + yield protocol.entrypoint else: self.log.warn('handshaking protocol has empty connection string', protocol=protocol) @@ -583,38 +591,50 @@ def connect_to_if_not_connected(self, peer: PeerId, now: int) -> None: if peer.can_retry(now): self.connect_to(self.rng.choice(peer.entrypoints), peer) - def _connect_to_callback(self, protocol: Union[HathorProtocol, TLSMemoryBIOProtocol], peer: Optional[PeerId], - endpoint: IStreamClientEndpoint, connection_string: str, - url_peer_id: Optional[str]) -> None: + def _connect_to_callback( + self, + protocol: Union[HathorProtocol, TLSMemoryBIOProtocol], + peer: Optional[PeerId], + endpoint: IStreamClientEndpoint, + entrypoint: Entrypoint, + ) -> None: """Called when we successfully connect to a peer.""" if isinstance(protocol, HathorProtocol): - protocol.on_outbound_connect(url_peer_id, connection_string) + protocol.on_outbound_connect(entrypoint) else: assert isinstance(protocol.wrappedProtocol, HathorProtocol) - protocol.wrappedProtocol.on_outbound_connect(url_peer_id, connection_string) + protocol.wrappedProtocol.on_outbound_connect(entrypoint) self.connecting_peers.pop(endpoint) - def connect_to(self, description: str, peer: Optional[PeerId] = None, use_ssl: Optional[bool] = None) -> None: + def connect_to( + self, + entrypoint: Entrypoint, + peer: Optional[PeerId] = None, + use_ssl: Optional[bool] = None, + ) -> None: """ Attempt to connect to a peer, even if a connection already exists. Usually you should call `connect_to_if_not_connected`. If `use_ssl` is True, then the connection will be wraped by a TLS. """ + if entrypoint.peer_id is not None and peer is not None and str(entrypoint.peer_id) != peer.id: + self.log.debug('skipping because the entrypoint peer_id does not match the actual peer_id', + entrypoint=entrypoint) + return + for connecting_peer in self.connecting_peers.values(): - if connecting_peer.connection_string == description: - self.log.debug('skipping because we are already connecting to this endpoint', endpoint=description) + if connecting_peer.entrypoint.equals_ignore_peer_id(entrypoint): + self.log.debug('skipping because we are already connecting to this endpoint', entrypoint=entrypoint) return + if self.localhost_only and not entrypoint.is_localhost(): + self.log.debug('skip because of simple localhost check', entrypoint=entrypoint) + return + if use_ssl is None: use_ssl = self.use_ssl - connection_string, peer_id = description_to_connection_string(description) - # When using twisted endpoints we can't have // in the connection string - endpoint_url = connection_string.replace('//', '') - endpoint = endpoints.clientFromString(self.reactor, endpoint_url) - if self.localhost_only: - if ('127.0.0.1' not in endpoint_url) and ('localhost' not in endpoint_url): - return + endpoint = entrypoint.to_client_endpoint(self.reactor) factory: IProtocolFactory if use_ssl: @@ -628,11 +648,11 @@ def connect_to(self, description: str, peer: Optional[PeerId] = None, use_ssl: O peer.increment_retry_attempt(now) deferred = endpoint.connect(factory) - self.connecting_peers[endpoint] = _ConnectingPeer(connection_string, deferred) + self.connecting_peers[endpoint] = _ConnectingPeer(entrypoint, deferred) - deferred.addCallback(self._connect_to_callback, peer, endpoint, connection_string, peer_id) + deferred.addCallback(self._connect_to_callback, peer, endpoint, entrypoint) deferred.addErrback(self.on_connection_failure, peer, endpoint) - self.log.info('connect to ', endpoint=description, peer=str(peer)) + self.log.info('connect to', entrypoint=str(entrypoint), peer=str(peer)) self.pubsub.publish( HathorEvents.NETWORK_PEER_CONNECTING, peer=peer, @@ -689,19 +709,14 @@ def update_hostname_entrypoints(self, *, old_hostname: str | None, new_hostname: assert self.manager is not None for address in self._listen_addresses: if old_hostname is not None: - old_address_str = self._get_hostname_address_str(old_hostname, address) - if old_address_str in self.my_peer.entrypoints: - self.my_peer.entrypoints.remove(old_address_str) - + old_entrypoint = Entrypoint.from_hostname_address(old_hostname, address) + if old_entrypoint in self.my_peer.entrypoints: + self.my_peer.entrypoints.remove(old_entrypoint) self._add_hostname_entrypoint(new_hostname, address) def _add_hostname_entrypoint(self, hostname: str, address: IPv4Address | IPv6Address) -> None: - hostname_address_str = self._get_hostname_address_str(hostname, address) - self.my_peer.entrypoints.append(hostname_address_str) - - @staticmethod - def _get_hostname_address_str(hostname: str, address: IPv4Address | IPv6Address) -> str: - return '{}://{}:{}'.format(address.type, hostname, address.port).lower() + hostname_entrypoint = Entrypoint.from_hostname_address(hostname, address) + self.my_peer.entrypoints.append(hostname_entrypoint) def get_connection_to_drop(self, protocol: HathorProtocol) -> HathorProtocol: """ When there are duplicate connections, determine which one should be dropped. diff --git a/hathor/p2p/peer_discovery/__init__.py b/hathor/p2p/peer_discovery/__init__.py new file mode 100644 index 000000000..d7167c36a --- /dev/null +++ b/hathor/p2p/peer_discovery/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .bootstrap import BootstrapPeerDiscovery +from .dns import DNSPeerDiscovery +from .peer_discovery import PeerDiscovery + +__all__ = [ + 'PeerDiscovery', + 'BootstrapPeerDiscovery', + 'DNSPeerDiscovery', +] diff --git a/hathor/p2p/peer_discovery/bootstrap.py b/hathor/p2p/peer_discovery/bootstrap.py new file mode 100644 index 000000000..a30970ae2 --- /dev/null +++ b/hathor/p2p/peer_discovery/bootstrap.py @@ -0,0 +1,42 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable + +from structlog import get_logger +from typing_extensions import override + +from hathor.p2p.entrypoint import Entrypoint + +from .peer_discovery import PeerDiscovery + +logger = get_logger() + + +class BootstrapPeerDiscovery(PeerDiscovery): + """ It implements a bootstrap peer discovery, which receives a static list of peers. + """ + + def __init__(self, entrypoints: list[Entrypoint]): + """ + :param descriptions: Descriptions of peers to connect to. + """ + super().__init__() + self.log = logger.new() + self.entrypoints = entrypoints + + @override + async def discover_and_connect(self, connect_to: Callable[[Entrypoint], None]) -> None: + for entrypoint in self.entrypoints: + connect_to(entrypoint) diff --git a/hathor/p2p/peer_discovery.py b/hathor/p2p/peer_discovery/dns.py similarity index 53% rename from hathor/p2p/peer_discovery.py rename to hathor/p2p/peer_discovery/dns.py index a202f6409..b946fc9eb 100644 --- a/hathor/p2p/peer_discovery.py +++ b/hathor/p2p/peer_discovery/dns.py @@ -1,4 +1,4 @@ -# Copyright 2021 Hathor Labs +# Copyright 2024 Hathor Labs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,48 +13,23 @@ # limitations under the License. import socket -from abc import ABC, abstractmethod -from typing import Callable +from collections.abc import Iterator +from itertools import chain +from typing import Callable, TypeAlias, cast from structlog import get_logger -from twisted.internet import defer +from twisted.internet.defer import Deferred, gatherResults from twisted.names.client import lookupAddress, lookupText from twisted.names.dns import Record_A, Record_TXT, RRHeader from typing_extensions import override -logger = get_logger() - - -class PeerDiscovery(ABC): - """ Base class to implement peer discovery strategies. - """ - - @abstractmethod - async def discover_and_connect(self, connect_to: Callable[[str], None]) -> None: - """ This method must discover the peers and call `connect_to` for each of them. +from hathor.p2p.entrypoint import Entrypoint, Protocol - :param connect_to: Function which will be called for each discovered peer. - :type connect_to: function - """ - raise NotImplementedError - - -class BootstrapPeerDiscovery(PeerDiscovery): - """ It implements a bootstrap peer discovery, which receives a static list of peers. - """ +from .peer_discovery import PeerDiscovery - def __init__(self, descriptions: list[str]): - """ - :param descriptions: Descriptions of peers to connect to. - """ - super().__init__() - self.log = logger.new() - self.descriptions = descriptions +logger = get_logger() - @override - async def discover_and_connect(self, connect_to: Callable[[str], None]) -> None: - for description in self.descriptions: - connect_to(description) +LookupResult: TypeAlias = tuple[list[RRHeader], list[RRHeader], list[RRHeader]] class DNSPeerDiscovery(PeerDiscovery): @@ -71,37 +46,42 @@ def __init__(self, hosts: list[str], default_port: int = 40403, test_mode: int = self.default_port = default_port self.test_mode = test_mode + def do_lookup_address(self, host: str) -> Deferred[LookupResult]: + return lookupAddress(host) + + def do_lookup_text(self, host: str) -> Deferred[LookupResult]: + return lookupText(host) + @override - async def discover_and_connect(self, connect_to: Callable[[str], None]) -> None: + async def discover_and_connect(self, connect_to: Callable[[Entrypoint], None]) -> None: """ Run DNS lookup for host and connect to it This is executed when starting the DNS Peer Discovery and first connecting to the network """ for host in self.hosts: - url_list = await self.dns_seed_lookup(host) - for url in url_list: - connect_to(url) + for entrypoint in (await self.dns_seed_lookup(host)): + connect_to(entrypoint) - async def dns_seed_lookup(self, host: str) -> list[str]: + async def dns_seed_lookup(self, host: str) -> set[Entrypoint]: """ Run a DNS lookup for TXT, A, and AAAA records and return a list of connection strings. """ if self.test_mode: # Useful for testing purposes, so we don't need to execute a DNS query - return ['tcp://127.0.0.1:40403'] + return {Entrypoint.parse('tcp://127.0.0.1:40403')} + + deferreds = [] - d1 = lookupText(host) + d1 = self.do_lookup_text(host) d1.addCallback(self.dns_seed_lookup_text) - d1.addErrback(self.errback), + d1.addErrback(self.errback) + deferreds.append(cast(Deferred[Iterator[Entrypoint]], d1)) # mypy doesn't know how addCallback affects d1 - d2 = lookupAddress(host) + d2 = self.do_lookup_address(host) d2.addCallback(self.dns_seed_lookup_address) - d2.addErrback(self.errback), + d2.addErrback(self.errback) + deferreds.append(cast(Deferred[Iterator[Entrypoint]], d2)) # mypy doesn't know how addCallback affects d2 - d = defer.gatherResults([d1, d2]) - results = await d - unique_urls: set[str] = set() - for urls in results: - unique_urls.update(urls) - return list(unique_urls) + results: list[Iterator[Entrypoint]] = await gatherResults(deferreds) + return set(chain(*results)) def errback(self, result): """ Return an empty list if any error occur. @@ -109,38 +89,35 @@ def errback(self, result): self.log.error('errback', result=result) return [] - def dns_seed_lookup_text( - self, results: tuple[list[RRHeader], list[RRHeader], list[RRHeader]] - ) -> list[str]: + def dns_seed_lookup_text(self, results: LookupResult) -> Iterator[Entrypoint]: """ Run a DNS lookup for TXT records to discover new peers. The `results` has three lists that contain answer records, authority records, and additional records. """ answers, _, _ = results - ret: list[str] = [] for record in answers: assert isinstance(record.payload, Record_TXT) for txt in record.payload.data: - txt = txt.decode('utf-8') - self.log.info('seed DNS TXT found', endpoint=txt) - ret.append(txt) - return ret - - def dns_seed_lookup_address( - self, results: tuple[list[RRHeader], list[RRHeader], list[RRHeader]] - ) -> list[str]: + raw_entrypoint = txt.decode('utf-8') + try: + entrypoint = Entrypoint.parse(raw_entrypoint) + except ValueError: + self.log.warning('could not parse entrypoint, skipping it', raw_entrypoint=raw_entrypoint) + continue + self.log.info('seed DNS TXT found', entrypoint=str(entrypoint)) + yield entrypoint + + def dns_seed_lookup_address(self, results: LookupResult) -> Iterator[Entrypoint]: """ Run a DNS lookup for A records to discover new peers. The `results` has three lists that contain answer records, authority records, and additional records. """ answers, _, _ = results - ret: list[str] = [] for record in answers: assert isinstance(record.payload, Record_A) address = record.payload.address assert address is not None host = socket.inet_ntoa(address) - txt = 'tcp://{}:{}'.format(host, self.default_port) - self.log.info('seed DNS A found', endpoint=txt) - ret.append(txt) - return ret + entrypoint = Entrypoint(Protocol.TCP, host, self.default_port) + self.log.info('seed DNS A found', entrypoint=str(entrypoint)) + yield entrypoint diff --git a/hathor/p2p/peer_discovery/peer_discovery.py b/hathor/p2p/peer_discovery/peer_discovery.py new file mode 100644 index 000000000..a6ff799ed --- /dev/null +++ b/hathor/p2p/peer_discovery/peer_discovery.py @@ -0,0 +1,32 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Callable + +from hathor.p2p.entrypoint import Entrypoint + + +class PeerDiscovery(ABC): + """ Base class to implement peer discovery strategies. + """ + + @abstractmethod + async def discover_and_connect(self, connect_to: Callable[[Entrypoint], None]) -> None: + """ This method must discover the peers and call `connect_to` for each of them. + + :param connect_to: Function which will be called for each discovered peer. + :type connect_to: function + """ + raise NotImplementedError diff --git a/hathor/p2p/peer_id.py b/hathor/p2p/peer_id.py index 1ab1ae58e..d2aef2634 100644 --- a/hathor/p2p/peer_id.py +++ b/hathor/p2p/peer_id.py @@ -31,7 +31,8 @@ from hathor.conf.get_settings import get_global_settings from hathor.daa import DifficultyAdjustmentAlgorithm -from hathor.p2p.utils import connection_string_to_host, discover_dns, generate_certificate +from hathor.p2p.entrypoint import Entrypoint +from hathor.p2p.utils import discover_dns, generate_certificate from hathor.util import not_none if TYPE_CHECKING: @@ -59,7 +60,7 @@ class PeerId: """ id: Optional[str] - entrypoints: list[str] + entrypoints: list[Entrypoint] private_key: Optional[rsa.RSAPrivateKeyWithSerialization] public_key: Optional[rsa.RSAPublicKey] certificate: Optional[x509.Certificate] @@ -90,8 +91,14 @@ def __init__(self, auto_generate_keys: bool = True) -> None: self.generate_keys() def __str__(self): - return ('PeerId(id=%s, entrypoints=%s, retry_timestamp=%d, retry_interval=%d)' % (self.id, self.entrypoints, - self.retry_timestamp, self.retry_interval)) + return ( + f'PeerId(id={self.id}, entrypoints={self.entrypoints_as_str()}, retry_timestamp={self.retry_timestamp}, ' + f'retry_interval={self.retry_interval})' + ) + + def entrypoints_as_str(self) -> list[str]: + """Return a list of entrypoints serialized as str""" + return list(map(str, self.entrypoints)) def merge(self, other: 'PeerId') -> None: """ Merge two PeerId objects, checking that they have the same @@ -200,7 +207,11 @@ def create_from_json(cls, data: dict[str, Any]) -> 'PeerId': obj.private_key = private_key if 'entrypoints' in data: - obj.entrypoints = data['entrypoints'] + for entrypoint_string in data['entrypoints']: + entrypoint = Entrypoint.parse(entrypoint_string) + if entrypoint.peer_id is not None: + raise ValueError('do not add id= to peer.json entrypoints') + obj.entrypoints.append(entrypoint) # TODO(epnichols): call obj.validate()? return obj @@ -243,7 +254,7 @@ def to_json(self, include_private_key: bool = False) -> dict[str, Any]: result = { 'id': self.id, 'pubKey': base64.b64encode(public_der).decode('utf-8'), - 'entrypoints': self.entrypoints, + 'entrypoints': self.entrypoints_as_str(), } if include_private_key: assert self.private_key is not None @@ -352,23 +363,24 @@ async def validate_entrypoint(self, protocol: 'HathorProtocol') -> bool: # Entrypoint validation with connection string and connection host # Entrypoints have the format tcp://IP|name:port for entrypoint in self.entrypoints: - if protocol.connection_string: + if protocol.entrypoint is not None: # Connection string has the format tcp://IP:port # So we must consider that the entrypoint could be in name format - if protocol.connection_string == entrypoint: + if protocol.entrypoint.equals_ignore_peer_id(entrypoint): + # XXX: wrong peer-id should not make it into self.entrypoints + assert not protocol.entrypoint.peer_id_conflicts_with(entrypoint), 'wrong peer-id was added before' # Found the entrypoint found_entrypoint = True break - host = connection_string_to_host(entrypoint) # TODO: don't use `daa.TEST_MODE` for this test_mode = not_none(DifficultyAdjustmentAlgorithm.singleton).TEST_MODE - result = await discover_dns(host, test_mode) - if protocol.connection_string in result: + result = await discover_dns(entrypoint.host, test_mode) + if protocol.entrypoint in result: # Found the entrypoint found_entrypoint = True break else: - # When the peer is the server part of the connection we don't have the full connection_string + # When the peer is the server part of the connection we don't have the full entrypoint description # So we can only validate the host from the protocol assert protocol.transport is not None connection_remote = protocol.transport.getPeer() @@ -377,13 +389,12 @@ async def validate_entrypoint(self, protocol: 'HathorProtocol') -> bool: continue # Connection host has only the IP # So we must consider that the entrypoint could be in name format and we just validate the host - host = connection_string_to_host(entrypoint) - if connection_host == host: + if connection_host == entrypoint.host: found_entrypoint = True break test_mode = not_none(DifficultyAdjustmentAlgorithm.singleton).TEST_MODE - result = await discover_dns(host, test_mode) - if connection_host in [connection_string_to_host(x) for x in result]: + result = await discover_dns(entrypoint.host, test_mode) + if connection_host in [entrypoint.host for entrypoint in result]: # Found the entrypoint found_entrypoint = True break diff --git a/hathor/p2p/protocol.py b/hathor/p2p/protocol.py index e4cff0d5a..99c63b29e 100644 --- a/hathor/p2p/protocol.py +++ b/hathor/p2p/protocol.py @@ -23,7 +23,8 @@ from twisted.protocols.basic import LineReceiver from twisted.python.failure import Failure -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.messages import ProtocolMessages from hathor.p2p.peer_id import PeerId from hathor.p2p.rate_limiter import RateLimiter @@ -82,8 +83,7 @@ class WarningFlags(str, Enum): state: Optional[BaseState] connection_time: float _state_instances: dict[PeerState, BaseState] - connection_string: Optional[str] - expected_peer_id: Optional[str] + entrypoint: Optional[Entrypoint] warning_flags: set[str] aborting: bool diff_timestamp: Optional[int] @@ -91,9 +91,17 @@ class WarningFlags(str, Enum): sync_version: Optional[SyncVersion] # version chosen to be used on this connection capabilities: set[str] # capabilities received from the peer in HelloState - def __init__(self, network: str, my_peer: PeerId, p2p_manager: 'ConnectionsManager', - *, use_ssl: bool, inbound: bool) -> None: - self._settings = get_global_settings() + def __init__( + self, + network: str, + my_peer: PeerId, + p2p_manager: 'ConnectionsManager', + *, + settings: HathorSettings, + use_ssl: bool, + inbound: bool, + ) -> None: + self._settings = settings self.network = network self.my_peer = my_peer self.connections = p2p_manager @@ -138,7 +146,7 @@ def __init__(self, network: str, my_peer: PeerId, p2p_manager: 'ConnectionsManag # Connection string of the peer # Used to validate if entrypoints has this string - self.connection_string: Optional[str] = None + self.entrypoint: Optional[Entrypoint] = None # Peer id sent in the connection url that is expected to connect (optional) self.expected_peer_id: Optional[str] = None @@ -164,7 +172,7 @@ def change_state(self, state_enum: PeerState) -> None: """Called to change the state of the connection.""" if state_enum not in self._state_instances: state_cls = state_enum.value - instance = state_cls(self) + instance = state_cls(self, self._settings) instance.state_name = state_enum.name self._state_instances[state_enum] = instance new_state = self._state_instances[state_enum] @@ -243,17 +251,10 @@ def on_connect(self) -> None: if self.connections: self.connections.on_peer_connect(self) - def on_outbound_connect(self, url_peer_id: Optional[str], connection_string: str) -> None: + def on_outbound_connect(self, entrypoint: Entrypoint) -> None: """Called when we successfully establish an outbound connection to a peer.""" - if url_peer_id: - # Set in protocol the peer id extracted from the URL that must be validated - self.expected_peer_id = url_peer_id - else: - # Add warning flag - self.warning_flags.add(self.WarningFlags.NO_PEER_ID_URL) - - # Setting connection string in protocol, so we can validate it matches the entrypoints data - self.connection_string = connection_string + # Save the used entrypoint in protocol so we can validate that it matches the entrypoints data + self.entrypoint = entrypoint def on_peer_ready(self) -> None: assert self.connections is not None diff --git a/hathor/p2p/resources/add_peers.py b/hathor/p2p/resources/add_peers.py index d5579d6b9..75a39b901 100644 --- a/hathor/p2p/resources/add_peers.py +++ b/hathor/p2p/resources/add_peers.py @@ -14,8 +14,13 @@ from json import JSONDecodeError +from twisted.internet.defer import Deferred +from twisted.web.http import Request + from hathor.api_util import Resource, render_options, set_cors from hathor.cli.openapi_files.register import register_resource +from hathor.manager import HathorManager +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.peer_discovery import BootstrapPeerDiscovery from hathor.util import json_dumpb, json_loadb @@ -28,57 +33,65 @@ class AddPeersResource(Resource): """ isLeaf = True - def __init__(self, manager): + def __init__(self, manager: HathorManager) -> None: self.manager = manager - def render_POST(self, request): + def render_POST(self, request: Request) -> bytes: """ Add p2p peers It expects a list of peers, in the format protocol://host:port (tcp://172.121.212.12:40403) """ request.setHeader(b'content-type', b'application/json; charset=utf-8') set_cors(request, 'POST') + assert request.content is not None raw_data = request.content.read() if raw_data is None: return json_dumpb({'success': False, 'message': 'No post data'}) try: - peers = json_loadb(raw_data) + raw_entrypoints = json_loadb(raw_data) except (JSONDecodeError, AttributeError): return json_dumpb({'success': False, 'message': 'Invalid format for post data'}) - if not isinstance(peers, list): + if not isinstance(raw_entrypoints, list): return json_dumpb({ 'success': False, 'message': 'Invalid format for post data. It was expected a list of strings.' }) - known_peers = self.manager.connections.peer_storage.values() + try: + entrypoints = list(map(Entrypoint.parse, raw_entrypoints)) + except ValueError: + return json_dumpb({ + 'success': False, + 'message': 'Malformed entrypoint found.' + }) - def already_connected(connection_string: str) -> bool: - # determines if given connection string is already among connected or connecting peers - endpoint_url = connection_string.replace('//', '') + known_peers = self.manager.connections.peer_storage.values() + def already_connected(entrypoint: Entrypoint) -> bool: # ignore peers that we're already trying to connect - if endpoint_url in self.manager.connections.iter_not_ready_endpoints(): + if entrypoint in self.manager.connections.iter_not_ready_endpoints(): return True # remove peers we already know about for peer in known_peers: - if connection_string in peer.entrypoints: + if entrypoint in peer.entrypoints: return True return False - filtered_peers = [connection_string for connection_string in peers if not already_connected(connection_string)] + filtered_peers = [entrypoint for entrypoint in entrypoints if not already_connected(entrypoint)] pd = BootstrapPeerDiscovery(filtered_peers) - pd.discover_and_connect(self.manager.connections.connect_to) + # this fires and forget the coroutine, which is compatible with the original behavior + coro = pd.discover_and_connect(self.manager.connections.connect_to) + Deferred.fromCoroutine(coro) - ret = {'success': True, 'peers': filtered_peers} + ret = {'success': True, 'peers': [str(p) for p in filtered_peers]} return json_dumpb(ret) - def render_OPTIONS(self, request): + def render_OPTIONS(self, request: Request) -> int: return render_options(request) diff --git a/hathor/p2p/resources/status.py b/hathor/p2p/resources/status.py index 544484f46..225665930 100644 --- a/hathor/p2p/resources/status.py +++ b/hathor/p2p/resources/status.py @@ -83,7 +83,7 @@ def render_GET(self, request): for peer in self.manager.connections.peer_storage.values(): known_peers.append({ 'id': peer.id, - 'entrypoints': peer.entrypoints, + 'entrypoints': peer.entrypoints_as_str(), 'last_seen': now - peer.last_seen, 'flags': [flag.value for flag in peer.flags], }) @@ -107,7 +107,7 @@ def render_GET(self, request): 'state': self.manager.state.value, 'network': self.manager.network, 'uptime': now - self.manager.start_time, - 'entrypoints': self.manager.connections.my_peer.entrypoints, + 'entrypoints': self.manager.connections.my_peer.entrypoints_as_str(), }, 'peers_whitelist': self.manager.peers_whitelist, 'known_peers': known_peers, diff --git a/hathor/p2p/states/base.py b/hathor/p2p/states/base.py index abbc17dd0..f08401cc0 100644 --- a/hathor/p2p/states/base.py +++ b/hathor/p2p/states/base.py @@ -18,6 +18,7 @@ from structlog import get_logger from twisted.internet.defer import Deferred +from hathor.conf.settings import HathorSettings from hathor.p2p.messages import ProtocolMessages if TYPE_CHECKING: @@ -33,8 +34,9 @@ class BaseState: Callable[[str], None] | Callable[[str], Deferred[None]] | Callable[[str], Coroutine[Deferred[None], Any, None]] ] - def __init__(self, protocol: 'HathorProtocol'): + def __init__(self, protocol: 'HathorProtocol', settings: HathorSettings): self.log = logger.new(**protocol.get_logger_context()) + self._settings = settings self.protocol = protocol self.cmd_map = { ProtocolMessages.ERROR: self.handle_error, diff --git a/hathor/p2p/states/hello.py b/hathor/p2p/states/hello.py index 9472f140c..b7cb42dce 100644 --- a/hathor/p2p/states/hello.py +++ b/hathor/p2p/states/hello.py @@ -18,6 +18,7 @@ import hathor from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.exception import HathorError from hathor.p2p.messages import ProtocolMessages from hathor.p2p.states.base import BaseState @@ -32,9 +33,8 @@ class HelloState(BaseState): - def __init__(self, protocol: 'HathorProtocol') -> None: - super().__init__(protocol) - self._settings = get_global_settings() + def __init__(self, protocol: 'HathorProtocol', settings: HathorSettings) -> None: + super().__init__(protocol, settings) self.log = logger.new(**protocol.get_logger_context()) self.cmd_map.update({ ProtocolMessages.HELLO: self.handle_hello, @@ -56,7 +56,7 @@ def _get_hello_data(self) -> dict[str, Any]: 'remote_address': format_address(remote), 'genesis_short_hash': get_genesis_short_hash(), 'timestamp': protocol.node.reactor.seconds(), - 'settings_dict': get_settings_hello_dict(), + 'settings_dict': get_settings_hello_dict(self._settings), 'capabilities': protocol.node.capabilities, } if self.protocol.node.has_sync_version_capability(): @@ -130,7 +130,10 @@ def handle_hello(self, payload: str) -> None: protocol.sync_version = max(common_sync_versions) if data['app'] != self._app(): - self.log.warn('different versions', theirs=data['app'], ours=self._app()) + remote_app = data['app'].encode().hex() + our_app = self._app().encode().hex() + # XXX: this used to be a warning, but it shouldn't be since it's perfectly normal + self.log.debug('different versions', theirs=remote_app, ours=our_app) if data['network'] != protocol.network: protocol.send_error_and_close_connection('Wrong network.') @@ -147,7 +150,7 @@ def handle_hello(self, payload: str) -> None: if 'settings_dict' in data: # If settings_dict is sent we must validate it - settings_dict = get_settings_hello_dict() + settings_dict = get_settings_hello_dict(self._settings) if data['settings_dict'] != settings_dict: protocol.send_error_and_close_connection( 'Settings values are different. {}'.format(json_dumps(settings_dict)) diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index b2e1f0a50..8d68b669b 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -16,7 +16,7 @@ from structlog import get_logger -from hathor.conf import HathorSettings +from hathor.conf.settings import HathorSettings from hathor.p2p.messages import ProtocolMessages from hathor.p2p.peer_id import PeerId from hathor.p2p.states.base import BaseState @@ -27,12 +27,10 @@ logger = get_logger() -settings = HathorSettings() - class PeerIdState(BaseState): - def __init__(self, protocol: 'HathorProtocol') -> None: - super().__init__(protocol) + def __init__(self, protocol: 'HathorProtocol', settings: HathorSettings) -> None: + super().__init__(protocol, settings) self.log = logger.new(remote=protocol.get_short_remote()) self.cmd_map.update({ ProtocolMessages.PEER_ID: self.handle_peer_id, @@ -72,7 +70,7 @@ def send_peer_id(self) -> None: hello = { 'id': my_peer.id, 'pubKey': my_peer.get_public_key(), - 'entrypoints': my_peer.entrypoints, + 'entrypoints': my_peer.entrypoints_as_str(), } self.send_message(ProtocolMessages.PEER_ID, json_dumps(hello)) @@ -100,7 +98,7 @@ async def handle_peer_id(self, payload: str) -> None: # is it on the whitelist? if peer.id and self._should_block_peer(peer.id): - if settings.WHITELIST_WARN_BLOCKED_PEERS: + if self._settings.WHITELIST_WARN_BLOCKED_PEERS: protocol.send_error_and_close_connection(f'Blocked (by {peer.id}). Get in touch with Hathor team.') else: protocol.send_error_and_close_connection('Connection rejected.') @@ -152,12 +150,12 @@ def _should_block_peer(self, peer_id: str) -> bool: return False # when ENABLE_PEER_WHITELIST is set, we check if we're on sync-v1 to block non-whitelisted peers - if settings.ENABLE_PEER_WHITELIST: + if self._settings.ENABLE_PEER_WHITELIST: assert self.protocol.sync_version is not None if not peer_is_whitelisted: if self.protocol.sync_version.is_v1(): return True - elif settings.USE_PEER_WHITELIST_ON_SYNC_V2: + elif self._settings.USE_PEER_WHITELIST_ON_SYNC_V2: return True # otherwise we block non-whitelisted peers when on "whitelist-only mode" diff --git a/hathor/p2p/states/ready.py b/hathor/p2p/states/ready.py index f500aadf1..35802b877 100644 --- a/hathor/p2p/states/ready.py +++ b/hathor/p2p/states/ready.py @@ -18,7 +18,7 @@ from structlog import get_logger from twisted.internet.task import LoopingCall -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.indexes.height_index import HeightInfo from hathor.p2p.messages import ProtocolMessages from hathor.p2p.peer_id import PeerId @@ -35,9 +35,8 @@ class ReadyState(BaseState): - def __init__(self, protocol: 'HathorProtocol') -> None: - super().__init__(protocol) - self._settings = get_global_settings() + def __init__(self, protocol: 'HathorProtocol', settings: HathorSettings) -> None: + super().__init__(protocol, settings) self.log = logger.new(**self.protocol.get_logger_context()) @@ -167,7 +166,7 @@ def send_peers(self, peer_list: Iterable['PeerId']) -> None: if peer.entrypoints: data.append({ 'id': peer.id, - 'entrypoints': peer.entrypoints, + 'entrypoints': peer.entrypoints_as_str(), }) self.send_message(ProtocolMessages.PEERS, json_dumps(data)) self.log.debug('send peers', peers=data) diff --git a/hathor/p2p/sync_v1/agent.py b/hathor/p2p/sync_v1/agent.py index cf395e4e3..72ef6b6d0 100644 --- a/hathor/p2p/sync_v1/agent.py +++ b/hathor/p2p/sync_v1/agent.py @@ -28,8 +28,8 @@ from hathor.p2p.sync_v1.downloader import Downloader from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.vertex_parser import VertexParser from hathor.util import json_dumps, json_loads logger = get_logger() @@ -60,7 +60,14 @@ class NodeSyncTimestamp(SyncAgent): MAX_HASHES: int = 40 - def __init__(self, protocol: 'HathorProtocol', downloader: Downloader, reactor: Reactor) -> None: + def __init__( + self, + protocol: 'HathorProtocol', + downloader: Downloader, + reactor: Reactor, + *, + vertex_parser: VertexParser, + ) -> None: """ :param protocol: Protocol of the connection. :type protocol: HathorProtocol @@ -69,6 +76,7 @@ def __init__(self, protocol: 'HathorProtocol', downloader: Downloader, reactor: :type reactor: Reactor """ self._settings = get_global_settings() + self.vertex_parser = vertex_parser self.protocol = protocol self.manager = protocol.node self.downloader = downloader @@ -597,7 +605,7 @@ def handle_data(self, payload: str) -> None: data = base64.b64decode(payload) try: - tx = tx_or_block_from_bytes(data) + tx = self.vertex_parser.deserialize(data) except struct.error: # Invalid data for tx decode return diff --git a/hathor/p2p/sync_v1/factory.py b/hathor/p2p/sync_v1/factory.py index d6fa55deb..2a205d728 100644 --- a/hathor/p2p/sync_v1/factory.py +++ b/hathor/p2p/sync_v1/factory.py @@ -20,14 +20,16 @@ from hathor.p2p.sync_v1.agent import NodeSyncTimestamp from hathor.p2p.sync_v1.downloader import Downloader from hathor.reactor import ReactorProtocol as Reactor +from hathor.transaction.vertex_parser import VertexParser if TYPE_CHECKING: from hathor.p2p.protocol import HathorProtocol class SyncV11Factory(SyncAgentFactory): - def __init__(self, connections: ConnectionsManager): + def __init__(self, connections: ConnectionsManager, *, vertex_parser: VertexParser): self.connections = connections + self.vertex_parser = vertex_parser self._downloader: Optional[Downloader] = None def get_downloader(self) -> Downloader: @@ -37,4 +39,9 @@ def get_downloader(self) -> Downloader: return self._downloader def create_sync_agent(self, protocol: 'HathorProtocol', reactor: Reactor) -> SyncAgent: - return NodeSyncTimestamp(protocol, downloader=self.get_downloader(), reactor=reactor) + return NodeSyncTimestamp( + protocol, + downloader=self.get_downloader(), + reactor=reactor, + vertex_parser=self.vertex_parser + ) diff --git a/hathor/p2p/sync_v2/agent.py b/hathor/p2p/sync_v2/agent.py index 780e84f41..2a71a4e4b 100644 --- a/hathor/p2p/sync_v2/agent.py +++ b/hathor/p2p/sync_v2/agent.py @@ -24,7 +24,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from twisted.internet.task import LoopingCall, deferLater -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.p2p.messages import ProtocolMessages from hathor.p2p.sync_agent import SyncAgent from hathor.p2p.sync_v2.blockchain_streaming_client import BlockchainStreamingClient, StreamingError @@ -39,8 +39,8 @@ from hathor.p2p.sync_v2.transaction_streaming_client import TransactionStreamingClient from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction, Block, Transaction -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.storage.exceptions import TransactionDoesNotExist +from hathor.transaction.vertex_parser import VertexParser from hathor.types import VertexId from hathor.util import collect_n, not_none @@ -84,7 +84,14 @@ class NodeBlockSync(SyncAgent): """ name: str = 'node-block-sync' - def __init__(self, protocol: 'HathorProtocol', reactor: Reactor) -> None: + def __init__( + self, + settings: HathorSettings, + protocol: 'HathorProtocol', + reactor: Reactor, + *, + vertex_parser: VertexParser, + ) -> None: """ :param protocol: Protocol of the connection. :type protocol: HathorProtocol @@ -92,7 +99,8 @@ def __init__(self, protocol: 'HathorProtocol', reactor: Reactor) -> None: :param reactor: Reactor to schedule later calls. (default=twisted.internet.reactor) :type reactor: Reactor """ - self._settings = get_global_settings() + self._settings = settings + self.vertex_parser = vertex_parser self.protocol = protocol self.manager = protocol.node self.tx_storage: 'TransactionStorage' = protocol.node.tx_storage @@ -768,7 +776,7 @@ def handle_blocks(self, payload: str) -> None: assert self.protocol.connections is not None blk_bytes = base64.b64decode(payload) - blk = tx_or_block_from_bytes(blk_bytes) + blk = self.vertex_parser.deserialize(blk_bytes) if not isinstance(blk, Block): # Not a block. Punish peer? return @@ -1018,7 +1026,7 @@ def handle_transaction(self, payload: str) -> None: # tx_bytes = bytes.fromhex(payload) tx_bytes = base64.b64decode(payload) - tx = tx_or_block_from_bytes(tx_bytes) + tx = self.vertex_parser.deserialize(tx_bytes) if not isinstance(tx, Transaction): self.log.warn('not a transaction', hash=tx.hash_hex) # Not a transaction. Punish peer? @@ -1127,7 +1135,7 @@ def handle_data(self, payload: str) -> None: data = base64.b64decode(part2) try: - tx = tx_or_block_from_bytes(data) + tx = self.vertex_parser.deserialize(data) except struct.error: # Invalid data for tx decode return diff --git a/hathor/p2p/sync_v2/factory.py b/hathor/p2p/sync_v2/factory.py index 71f17dd87..65d42d622 100644 --- a/hathor/p2p/sync_v2/factory.py +++ b/hathor/p2p/sync_v2/factory.py @@ -14,19 +14,23 @@ from typing import TYPE_CHECKING +from hathor.conf.settings import HathorSettings from hathor.p2p.manager import ConnectionsManager from hathor.p2p.sync_agent import SyncAgent from hathor.p2p.sync_factory import SyncAgentFactory from hathor.p2p.sync_v2.agent import NodeBlockSync from hathor.reactor import ReactorProtocol as Reactor +from hathor.transaction.vertex_parser import VertexParser if TYPE_CHECKING: from hathor.p2p.protocol import HathorProtocol class SyncV2Factory(SyncAgentFactory): - def __init__(self, connections: ConnectionsManager): + def __init__(self, settings: HathorSettings, connections: ConnectionsManager, *, vertex_parser: VertexParser): + self._settings = settings self.connections = connections + self.vertex_parser = vertex_parser def create_sync_agent(self, protocol: 'HathorProtocol', reactor: Reactor) -> SyncAgent: - return NodeBlockSync(protocol, reactor=reactor) + return NodeBlockSync(self._settings, protocol, reactor=reactor, vertex_parser=self.vertex_parser) diff --git a/hathor/p2p/utils.py b/hathor/p2p/utils.py index 4e2935a2e..0dfe9ebc1 100644 --- a/hathor/p2p/utils.py +++ b/hathor/p2p/utils.py @@ -15,7 +15,6 @@ import datetime import re from typing import Any, Optional -from urllib.parse import parse_qs, urlparse import requests from cryptography import x509 @@ -28,7 +27,9 @@ from twisted.internet.interfaces import IAddress from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.indexes.height_index import HeightInfo +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.peer_discovery import DNSPeerDiscovery from hathor.transaction.genesis import get_representation_for_all_genesis @@ -52,25 +53,6 @@ def discover_ip_ipify(timeout: float | None = None) -> Optional[str]: return None -def description_to_connection_string(description: str) -> tuple[str, Optional[str]]: - """ The description returned from DNS query may contain a peer-id parameter - This method splits this description into the connection URL and the peer-id (in case it exists) - Expected description is something like: tcp://127.0.0.1:40403/?id=123 - The expected returned tuple in this case would be ('tcp://127.0.0.1:40403', '123') - """ - result = urlparse(description) - - url = "{}://{}".format(result.scheme, result.netloc) - peer_id = None - - if result.query: - query_result = parse_qs(result.query) - if 'id' in query_result: - peer_id = query_result['id'][0] - - return url, peer_id - - def get_genesis_short_hash() -> str: """ Return the first 7 chars of the GENESIS_HASH used for validation that the genesis are the same """ @@ -78,10 +60,9 @@ def get_genesis_short_hash() -> str: return get_representation_for_all_genesis(settings).hex()[:7] -def get_settings_hello_dict() -> dict[str, Any]: +def get_settings_hello_dict(settings: HathorSettings) -> dict[str, Any]: """ Return a dict of settings values that must be validated in the hello state """ - settings = get_global_settings() settings_dict = {} for key in settings.P2P_SETTINGS_HASH_FIELDS: value = getattr(settings, key) @@ -89,17 +70,14 @@ def get_settings_hello_dict() -> dict[str, Any]: if type(value) is bytes: value = value.hex() settings_dict[key] = value - return settings_dict + if consensus_hash := settings.CONSENSUS_ALGORITHM.get_peer_hello_hash(): + settings_dict['CONSENSUS_ALGORITHM'] = consensus_hash -def connection_string_to_host(connection_string: str) -> str: - """ From a connection string I return the host - tcp://127.0.0.1:40403 -> 127.0.0.1 - """ - return urlparse(connection_string).netloc.split(':')[0] + return settings_dict -async def discover_dns(host: str, test_mode: int = 0) -> list[str]: +async def discover_dns(host: str, test_mode: int = 0) -> list[Entrypoint]: """ Start a DNS peer discovery object and execute a search for the host Returns the DNS string from the requested host @@ -107,7 +85,7 @@ async def discover_dns(host: str, test_mode: int = 0) -> list[str]: """ discovery = DNSPeerDiscovery([], test_mode=test_mode) result = await discovery.dns_seed_lookup(host) - return result + return list(result) def generate_certificate(private_key: RSAPrivateKey, ca_file: str, ca_pkey_file: str) -> Certificate: diff --git a/hathor/profiler/cpu.py b/hathor/profiler/cpu.py index fde33ed7f..63064df7e 100644 --- a/hathor/profiler/cpu.py +++ b/hathor/profiler/cpu.py @@ -73,7 +73,7 @@ def __init__(self, *, update_interval: float = 3.0, expiry: float = 15.0): self.proc_list: list[tuple[Key, ProcItem]] = [] # Timer to call `self.update()` periodically. - self.lc_update = LoopingCall(self.update) + self.lc_update: LoopingCall | None = None # Interval to update the list of processes. self.update_interval = update_interval @@ -102,6 +102,7 @@ def start(self) -> None: return self.reset() self.enabled = True + self.lc_update = LoopingCall(self.update) self.lc_update.start(self.update_interval) def stop(self) -> None: @@ -109,6 +110,7 @@ def stop(self) -> None: if not self.enabled: return self.enabled = False + assert self.lc_update is not None self.lc_update.stop() def get_proc_list(self) -> list[tuple[Key, ProcItem]]: diff --git a/hathor/pubsub.py b/hathor/pubsub.py index 0a3168aa7..6a6b28f74 100644 --- a/hathor/pubsub.py +++ b/hathor/pubsub.py @@ -21,6 +21,7 @@ from twisted.python.threadable import isInIOThread from hathor.reactor import ReactorProtocol as Reactor +from hathor.types import VertexId from hathor.utils.zope import verified_cast if TYPE_CHECKING: @@ -145,6 +146,7 @@ class EventArguments: # XXX: add these as needed, these attributes don't always exist, but when they do these are their types tx: 'BaseTransaction' + vertex_id: VertexId reorg_size: int old_best_block: 'Block' new_best_block: 'Block' diff --git a/hathor/reactor/reactor.py b/hathor/reactor/reactor.py index e94dc3a97..b1ff7e4d7 100644 --- a/hathor/reactor/reactor.py +++ b/hathor/reactor/reactor.py @@ -70,7 +70,9 @@ def initialize_global_reactor(*, use_asyncio_reactor: bool = False) -> ReactorPr msg = ( "There's a Twisted reactor installed already. It's probably the default one, installed indirectly by " "one of our imports. This can happen, for example, if we import from the hathor module in " - "entrypoint-level, like in CLI tools other than `RunNode`." + "entrypoint-level, like in CLI tools other than `RunNode`. Debug it by setting a breakpoint in " + "`installReactor()` in the `twisted/internet/main.py` file." + ) raise Exception(msg) from e diff --git a/hathor/stratum/stratum.py b/hathor/stratum/stratum.py index 78a9f29ae..92111a431 100644 --- a/hathor/stratum/stratum.py +++ b/hathor/stratum/stratum.py @@ -418,11 +418,13 @@ def handle_request(self, method: str, params: Optional[Union[list, dict]], msgid if not self.manager.can_start_mining(): return self.send_error(NODE_SYNCING, msgid) - if method in ['mining.subscribe', 'subscribe']: + if not isinstance(params, dict): + self.log.error(f'expected dict params, received: {params}') params = cast(dict, params) + + if method in ['mining.subscribe', 'subscribe']: return self.handle_subscribe(params, msgid) if method in ['mining.submit', 'submit']: - params = cast(dict, params) return self.handle_submit(params, msgid) self.send_error(METHOD_NOT_FOUND, msgid, data={'method': method, 'supported_methods': ['submit', 'subscribe']}) diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index 56898d6f9..607ddb539 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import base64 import datetime import hashlib @@ -38,6 +40,7 @@ if TYPE_CHECKING: from _hashlib import HASH + from hathor.conf.settings import HathorSettings from hathor.transaction.storage import TransactionStorage # noqa: F401 logger = get_logger() @@ -86,6 +89,7 @@ class TxVersion(IntEnum): REGULAR_TRANSACTION = 1 TOKEN_CREATION_TRANSACTION = 2 MERGE_MINED_BLOCK = 3 + POA_BLOCK = 5 @classmethod def _missing_(cls, value: Any) -> None: @@ -97,6 +101,7 @@ def _missing_(cls, value: Any) -> None: def get_cls(self) -> type['BaseTransaction']: from hathor.transaction.block import Block from hathor.transaction.merge_mined_block import MergeMinedBlock + from hathor.transaction.poa import PoaBlock from hathor.transaction.token_creation_tx import TokenCreationTransaction from hathor.transaction.transaction import Transaction @@ -105,6 +110,7 @@ def get_cls(self) -> type['BaseTransaction']: TxVersion.REGULAR_TRANSACTION: Transaction, TxVersion.TOKEN_CREATION_TRANSACTION: TokenCreationTransaction, TxVersion.MERGE_MINED_BLOCK: MergeMinedBlock, + TxVersion.POA_BLOCK: PoaBlock } cls = cls_map.get(self) @@ -135,17 +141,20 @@ class BaseTransaction(ABC): # bits reserved for future use, depending on the configuration. signal_bits: int - def __init__(self, - nonce: int = 0, - timestamp: Optional[int] = None, - signal_bits: int = 0, - version: TxVersion = TxVersion.REGULAR_BLOCK, - weight: float = 0, - inputs: Optional[list['TxInput']] = None, - outputs: Optional[list['TxOutput']] = None, - parents: Optional[list[VertexId]] = None, - hash: Optional[VertexId] = None, - storage: Optional['TransactionStorage'] = None) -> None: + def __init__( + self, + nonce: int = 0, + timestamp: Optional[int] = None, + signal_bits: int = 0, + version: TxVersion = TxVersion.REGULAR_BLOCK, + weight: float = 0, + inputs: Optional[list['TxInput']] = None, + outputs: Optional[list['TxOutput']] = None, + parents: Optional[list[VertexId]] = None, + hash: Optional[VertexId] = None, + storage: Optional['TransactionStorage'] = None, + settings: HathorSettings | None = None, + ) -> None: """ Nonce: nonce used for the proof-of-work Timestamp: moment of creation @@ -158,7 +167,7 @@ def __init__(self, assert signal_bits <= _ONE_BYTE, f'signal_bits {hex(signal_bits)} must not be larger than one byte' assert version <= _ONE_BYTE, f'version {hex(version)} must not be larger than one byte' - self._settings = get_global_settings() + self._settings = settings or get_global_settings() self.nonce = nonce self.timestamp = timestamp or int(time.time()) self.signal_bits = signal_bits @@ -627,6 +636,7 @@ def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True) min_height = 0 if self.is_genesis else None metadata = TransactionMetadata( + settings=self._settings, hash=self._hash, accumulated_weight=self.weight, height=height, @@ -1130,17 +1140,3 @@ def output_value_to_bytes(number: int) -> bytes: return (-number).to_bytes(8, byteorder='big', signed=True) else: return number.to_bytes(4, byteorder='big', signed=True) # `signed` makes no difference, but oh well - - -def tx_or_block_from_bytes(data: bytes, - storage: Optional['TransactionStorage'] = None) -> BaseTransaction: - """ Creates the correct tx subclass from a sequence of bytes - """ - # version field takes up the second byte only - version = data[1] - try: - tx_version = TxVersion(version) - cls = tx_version.get_cls() - return cls.create_from_struct(data, storage=storage) - except ValueError: - raise StructError('Invalid bytes to create transaction subclass.') diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 80f9ee67d..22e6d61ac 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -12,16 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import base64 from itertools import starmap, zip_longest from operator import add from struct import pack -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Iterator, Optional + +from typing_extensions import Self from hathor.checkpoint import Checkpoint from hathor.feature_activation.feature import Feature from hathor.feature_activation.model.feature_state import FeatureState -from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, TxOutput, TxVersion from hathor.transaction.exceptions import CheckpointError from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len @@ -29,10 +32,9 @@ from hathor.utils.int import get_bit_list if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings from hathor.transaction.storage import TransactionStorage # noqa: F401 -cpu = get_cpu_profiler() - # Signal bits (B), version (B), outputs len (B) _FUNDS_FORMAT_STRING = '!BBB' @@ -43,19 +45,32 @@ class Block(BaseTransaction): SERIALIZATION_NONCE_SIZE = 16 - def __init__(self, - nonce: int = 0, - timestamp: Optional[int] = None, - signal_bits: int = 0, - version: TxVersion = TxVersion.REGULAR_BLOCK, - weight: float = 0, - outputs: Optional[list[TxOutput]] = None, - parents: Optional[list[bytes]] = None, - hash: Optional[bytes] = None, - data: bytes = b'', - storage: Optional['TransactionStorage'] = None) -> None: - super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight, - outputs=outputs or [], parents=parents or [], hash=hash, storage=storage) + def __init__( + self, + nonce: int = 0, + timestamp: Optional[int] = None, + signal_bits: int = 0, + version: TxVersion = TxVersion.REGULAR_BLOCK, + weight: float = 0, + outputs: Optional[list[TxOutput]] = None, + parents: Optional[list[bytes]] = None, + hash: Optional[bytes] = None, + data: bytes = b'', + storage: Optional['TransactionStorage'] = None, + settings: HathorSettings | None = None, + ) -> None: + super().__init__( + nonce=nonce, + timestamp=timestamp, + signal_bits=signal_bits, + version=version, + weight=weight, + outputs=outputs or [], + parents=parents or [], + hash=hash, + storage=storage, + settings=settings, + ) self.data = data def _get_formatted_fields_dict(self, short: bool = True) -> dict[str, str]: @@ -76,7 +91,7 @@ def is_transaction(self) -> bool: @classmethod def create_from_struct(cls, struct_bytes: bytes, storage: Optional['TransactionStorage'] = None, - *, verbose: VerboseCallback = None) -> 'Block': + *, verbose: VerboseCallback = None) -> Self: blc = cls() buf = blc.get_fields_from_struct(struct_bytes, verbose=verbose) @@ -401,3 +416,14 @@ def get_feature_activation_bit_value(self, bit: int) -> int: bit_list = self._get_feature_activation_bit_list() return bit_list[bit] + + def iter_transactions_in_this_block(self) -> Iterator[BaseTransaction]: + """Return an iterator of the transactions that have this block as meta.first_block.""" + from hathor.transaction.storage.traversal import BFSOrderWalk + bfs = BFSOrderWalk(self.storage, is_dag_verifications=True, is_dag_funds=True, is_left_to_right=False) + for tx in bfs.run(self, skip_root=True): + tx_meta = tx.get_metadata() + if tx_meta.first_block != self.hash: + bfs.skip_neighbors(tx) + continue + yield tx diff --git a/hathor/transaction/exceptions.py b/hathor/transaction/exceptions.py index 25e61596c..2d1bfbda8 100644 --- a/hathor/transaction/exceptions.py +++ b/hathor/transaction/exceptions.py @@ -110,6 +110,10 @@ class WeightError(TxValidationError): """Transaction not using correct weight""" +class PoaValidationError(TxValidationError): + """Block using invalid PoA signature""" + + class InvalidBlockReward(TxValidationError): """Wrong amount of issued tokens""" @@ -134,6 +138,10 @@ class RewardLocked(TxValidationError): """Block reward cannot be spent yet, needs more confirmations""" +class InvalidVersionError(TxValidationError): + """Vertex version is invalid.""" + + class BlockWithInputs(BlockError): """Block has inputs""" diff --git a/hathor/transaction/genesis.py b/hathor/transaction/genesis.py index 0d567515d..3ec107aac 100644 --- a/hathor/transaction/genesis.py +++ b/hathor/transaction/genesis.py @@ -14,7 +14,12 @@ from typing import TYPE_CHECKING +import base58 + from hathor.conf.settings import HathorSettings +from hathor.mining.cpu_mining_service import CpuMiningService +from hathor.transaction import Block, Transaction, TxOutput +from hathor.transaction.scripts import P2PKH if TYPE_CHECKING: from hathor.transaction.storage import TransactionStorage # noqa: F401 @@ -41,3 +46,43 @@ def get_representation_for_all_genesis(settings: HathorSettings) -> bytes: def is_genesis(hash_bytes: bytes, *, settings: HathorSettings) -> bool: """Check whether hash is from a genesis transaction.""" return hash_bytes in get_all_genesis_hashes(settings) + + +def generate_new_genesis( + *, + tokens: int, + address: str, + block_timestamp: int, + min_block_weight: float, + min_tx_weight: float, +) -> tuple[Block, Transaction, Transaction]: + """ + Create new genesis block and transactions. This is a convenience method to be used when creating side-dags, + and maybe in some tests. It should never be used in runtime. + """ + output_script = P2PKH.create_output_script(address=base58.b58decode(address)) + mining_service = CpuMiningService() + + block = Block( + timestamp=block_timestamp, + weight=min_block_weight, + outputs=[TxOutput(tokens, output_script)], + ) + mining_service.start_mining(block) + block.update_hash() + + tx1 = Transaction( + timestamp=block_timestamp + 1, + weight=min_tx_weight, + ) + mining_service.start_mining(tx1) + tx1.update_hash() + + tx2 = Transaction( + timestamp=block_timestamp + 2, + weight=min_tx_weight, + ) + mining_service.start_mining(tx2) + tx2.update_hash() + + return block, tx1, tx2 diff --git a/hathor/transaction/merge_mined_block.py b/hathor/transaction/merge_mined_block.py index a0664d3ae..863909882 100644 --- a/hathor/transaction/merge_mined_block.py +++ b/hathor/transaction/merge_mined_block.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from typing import TYPE_CHECKING, Any, Optional from hathor.transaction.aux_pow import BitcoinAuxPow @@ -20,24 +22,39 @@ from hathor.transaction.util import VerboseCallback if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings from hathor.transaction.storage import TransactionStorage # noqa: F401 class MergeMinedBlock(Block): - def __init__(self, - nonce: int = 0, - timestamp: Optional[int] = None, - signal_bits: int = 0, - version: TxVersion = TxVersion.MERGE_MINED_BLOCK, - weight: float = 0, - outputs: Optional[list[TxOutput]] = None, - parents: Optional[list[bytes]] = None, - hash: Optional[bytes] = None, - data: bytes = b'', - aux_pow: Optional[BitcoinAuxPow] = None, - storage: Optional['TransactionStorage'] = None) -> None: - super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight, - data=data, outputs=outputs or [], parents=parents or [], hash=hash, storage=storage) + def __init__( + self, + nonce: int = 0, + timestamp: Optional[int] = None, + signal_bits: int = 0, + version: TxVersion = TxVersion.MERGE_MINED_BLOCK, + weight: float = 0, + outputs: Optional[list[TxOutput]] = None, + parents: Optional[list[bytes]] = None, + hash: Optional[bytes] = None, + data: bytes = b'', + aux_pow: Optional[BitcoinAuxPow] = None, + storage: Optional['TransactionStorage'] = None, + settings: HathorSettings | None = None, + ) -> None: + super().__init__( + nonce=nonce, + timestamp=timestamp, + signal_bits=signal_bits, + version=version, + weight=weight, + data=data, + outputs=outputs or [], + parents=parents or [], + hash=hash, + storage=storage, + settings=settings + ) self.aux_pow = aux_pow def _get_formatted_fields_dict(self, short: bool = True) -> dict[str, str]: diff --git a/hathor/transaction/poa/__init__.py b/hathor/transaction/poa/__init__.py new file mode 100644 index 000000000..0719c5324 --- /dev/null +++ b/hathor/transaction/poa/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.transaction.poa.poa_block import PoaBlock + +__all__ = [ + 'PoaBlock', +] diff --git a/hathor/transaction/poa/poa_block.py b/hathor/transaction/poa/poa_block.py new file mode 100644 index 000000000..8d7f9ad7c --- /dev/null +++ b/hathor/transaction/poa/poa_block.py @@ -0,0 +1,103 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from typing_extensions import override + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings +from hathor.transaction import Block, TxOutput, TxVersion +from hathor.transaction.storage import TransactionStorage +from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len + +# Size limit in bytes for signature field +_MAX_POA_SIGNATURE_LEN: int = 100 + + +class PoaBlock(Block): + """A Proof-of-Authority block.""" + + def __init__( + self, + timestamp: int | None = None, + signal_bits: int = 0, + weight: float = 0, + outputs: list[TxOutput] | None = None, + parents: list[bytes] | None = None, + hash: bytes | None = None, + data: bytes = b'', + storage: TransactionStorage | None = None, + signer_id: bytes = b'', + signature: bytes = b'', + settings: HathorSettings | None = None, + ) -> None: + assert not outputs, 'PoaBlocks must not have outputs' + super().__init__( + nonce=0, + timestamp=timestamp, + signal_bits=signal_bits, + version=TxVersion.POA_BLOCK, + weight=weight, + outputs=[], + parents=parents or [], + hash=hash, + data=data, + storage=storage, + settings=settings, + ) + self.signer_id = signer_id + self.signature = signature + + @override + def get_graph_fields_from_struct(self, buf: bytes, *, verbose: VerboseCallback = None) -> bytes: + buf = super().get_graph_fields_from_struct(buf, verbose=verbose) + + self.signer_id, buf = unpack_len(poa.SIGNER_ID_LEN, buf) + if verbose: + verbose('signer_id', self.signer_id.hex()) + + (signature_len,), buf = unpack('!B', buf) + if verbose: + verbose('signature_len', signature_len) + + if signature_len > _MAX_POA_SIGNATURE_LEN: + raise ValueError(f'invalid signature length: {signature_len}') + + self.signature, buf = unpack_len(signature_len, buf) + if verbose: + verbose('signature', self.signature.hex()) + + return buf + + @override + def get_graph_struct(self) -> bytes: + assert len(self.signer_id) == poa.SIGNER_ID_LEN + struct_bytes_without_poa = super().get_graph_struct() + signature_len = int_to_bytes(len(self.signature), 1) + return struct_bytes_without_poa + self.signer_id + signature_len + self.signature + + @override + def to_json(self, decode_script: bool = False, include_metadata: bool = False) -> dict[str, Any]: + poa_settings = self._settings.CONSENSUS_ALGORITHM + assert isinstance(poa_settings, PoaSettings) + json = super().to_json(decode_script=decode_script, include_metadata=include_metadata) + signature_validation = poa.verify_poa_signature(poa_settings, self) + + if isinstance(signature_validation, poa.ValidSignature): + json['signer'] = signature_validation.public_key.hex() + + json['signer_id'] = self.signer_id.hex() + return json diff --git a/hathor/transaction/resources/block_at_height.py b/hathor/transaction/resources/block_at_height.py index d7b83af1e..9d4be036f 100644 --- a/hathor/transaction/resources/block_at_height.py +++ b/hathor/transaction/resources/block_at_height.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from hathor.api_util import Resource, get_args, get_missing_params_msg, parse_args, parse_int, set_cors +from hathor.api_util import Resource, set_cors from hathor.cli.openapi_files.register import register_resource from hathor.util import json_dumpb +from hathor.utils.api import ErrorResponse, QueryParams if TYPE_CHECKING: from twisted.web.http import Request @@ -48,38 +49,52 @@ def render_GET(self, request: 'Request') -> bytes: request.setHeader(b'content-type', b'application/json; charset=utf-8') set_cors(request, 'GET') - # Height parameter is required - parsed = parse_args(get_args(request), ['height']) - if not parsed['success']: - return get_missing_params_msg(parsed['missing']) + params = BlockAtHeightParams.from_request(request) + if isinstance(params, ErrorResponse): + return params.json_dumpb() - args = parsed['args'] + # Get hash of the block with the height + block_hash = self.manager.tx_storage.indexes.height.get(params.height) - # Height parameter must be an integer - try: - height = parse_int(args['height']) - except ValueError as e: + # If there is no block in the index with this height, block_hash will be None + if block_hash is None: return json_dumpb({ 'success': False, - 'message': f'Failed to parse \'height\': {e}' + 'message': 'No block with height {}.'.format(params.height) }) - # Get hash of the block with the height - block_hash = self.manager.tx_storage.indexes.height.get(height) + block = self.manager.tx_storage.get_block(block_hash) + data = {'success': True, 'block': block.to_json_extended()} - # If there is no block in the index with this height, block_hash will be None - if block_hash is None: + if params.include_transactions is None: + pass + + elif params.include_transactions == 'txid': + tx_ids: list[str] = [] + for tx in block.iter_transactions_in_this_block(): + tx_ids.append(tx.hash.hex()) + data['tx_ids'] = tx_ids + + elif params.include_transactions == 'full': + tx_list: list[Any] = [] + for tx in block.iter_transactions_in_this_block(): + tx_list.append(tx.to_json_extended()) + data['transactions'] = tx_list + + else: return json_dumpb({ 'success': False, - 'message': 'No block with height {}.'.format(height) + 'message': 'Invalid include_transactions. Choices are: txid or full.' }) - block = self.manager.tx_storage.get_transaction(block_hash) - - data = {'success': True, 'block': block.to_json_extended()} return json_dumpb(data) +class BlockAtHeightParams(QueryParams): + height: int + include_transactions: str | None + + BlockAtHeightResource.openapi = { '/block_at_height': { 'x-visibility': 'public', @@ -114,6 +129,19 @@ def render_GET(self, request: 'Request') -> bytes: 'type': 'int' } }, + { + 'name': 'include_transactions', + 'in': 'query', + 'description': 'Add transactions confirmed by this block.', + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [ + 'txid', + 'full', + ], + } + }, ], 'responses': { '200': { diff --git a/hathor/transaction/resources/decode_tx.py b/hathor/transaction/resources/decode_tx.py index 64274040c..39f32eec6 100644 --- a/hathor/transaction/resources/decode_tx.py +++ b/hathor/transaction/resources/decode_tx.py @@ -16,7 +16,6 @@ from hathor.api_util import Resource, get_args, get_missing_params_msg, parse_args, set_cors from hathor.cli.openapi_files.register import register_resource -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.resources.transaction import get_tx_extra_data from hathor.util import json_dumpb @@ -51,7 +50,7 @@ def render_GET(self, request): try: tx_bytes = bytes.fromhex(parsed['args']['hex_tx']) - tx = tx_or_block_from_bytes(tx_bytes) + tx = self.manager.vertex_parser.deserialize(tx_bytes) tx.storage = self.manager.tx_storage data = get_tx_extra_data(tx) except ValueError: diff --git a/hathor/transaction/resources/mining.py b/hathor/transaction/resources/mining.py index eaa80abfd..cda18f3ff 100644 --- a/hathor/transaction/resources/mining.py +++ b/hathor/transaction/resources/mining.py @@ -20,7 +20,6 @@ from hathor.cli.openapi_files.register import register_resource from hathor.crypto.util import decode_address from hathor.exception import HathorError -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.util import api_catch_exceptions, json_dumpb, json_loadb logger = get_logger() @@ -114,7 +113,7 @@ def render_POST(self, request): data = json_loadb(request.content.read()) - tx = tx_or_block_from_bytes(bytes.fromhex(data['hexdata']), storage=self.manager.tx_storage) + tx = self.manager.vertex_parser.deserialize(bytes.fromhex(data['hexdata']), storage=self.manager.tx_storage) if not tx.is_block: self.log.debug('expected Block, received Transaction', data=data) diff --git a/hathor/transaction/resources/push_tx.py b/hathor/transaction/resources/push_tx.py index c550231f2..b2b376b82 100644 --- a/hathor/transaction/resources/push_tx.py +++ b/hathor/transaction/resources/push_tx.py @@ -24,7 +24,6 @@ from hathor.conf.get_settings import get_global_settings from hathor.exception import InvalidNewTransaction from hathor.transaction import Transaction -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.exceptions import TxValidationError from hathor.util import json_dumpb, json_loadb @@ -70,7 +69,7 @@ def _get_client_ip(self, request: 'Request') -> str: def handle_push_tx(self, params: dict[str, Any], client_addr: str) -> dict[str, Any]: try: tx_bytes = bytes.fromhex(params['hex_tx']) - tx = tx_or_block_from_bytes(tx_bytes) + tx = self.manager.vertex_parser.deserialize(tx_bytes) except ValueError: return {'success': False, 'message': 'Invalid hexadecimal data', 'can_force': False} except struct.error: diff --git a/hathor/transaction/resources/transaction.py b/hathor/transaction/resources/transaction.py index 816185438..39a677784 100644 --- a/hathor/transaction/resources/transaction.py +++ b/hathor/transaction/resources/transaction.py @@ -14,6 +14,9 @@ from typing import Any +from structlog import get_logger +from twisted.web.http import Request + from hathor.api_util import ( Resource, get_args, @@ -31,6 +34,8 @@ GET_LIST_ARGS = ['count', 'type'] +logger = get_logger() + def update_serialized_tokens_array(tx: BaseTransaction, serialized: dict[str, Any]) -> None: """ A token creation tx to_json does not add its hash to the array of tokens @@ -146,9 +151,10 @@ class TransactionResource(Resource): def __init__(self, manager): # Important to have the manager so we can know the tx_storage + self._log = logger.new() self.manager = manager - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: """ Get request /transaction/ that returns list of tx or a single one If receive 'id' (hash) as GET parameter we return the tx with this hash @@ -174,13 +180,12 @@ def render_GET(self, request): return data - def get_one_tx(self, request): + def get_one_tx(self, request: Request) -> bytes: """ Get 'id' (hash) from request.args Returns the tx with this hash or {'success': False} if hash is invalid or tx does not exist """ - if not self.manager.tx_storage.indexes.tokens: - request.setResponseCode(503) - return json_dumpb({'success': False}) + if error_message := self._validate_index(request): + return error_message raw_args = get_args(request) requested_hash = raw_args[b'id'][0].decode('utf-8') @@ -195,6 +200,18 @@ def get_one_tx(self, request): return json_dumpb(data) + def _validate_index(self, request: Request) -> bytes | None: + """Return None if validation is successful (tokens index is enabled), and an error message otherwise.""" + if self.manager.tx_storage.indexes.tokens: + return None + + self._log.warn( + 'trying to reach transaction endpoint, but tokens index is disabled.\n' + 'use `--wallet-index` to enable it' + ) + request.setResponseCode(503) + return json_dumpb({'success': False, 'message': 'wallet index is disabled'}) + def get_list_tx(self, request): """ Get parameter from request.args and return list of blocks/txs diff --git a/hathor/transaction/storage/cache_storage.py b/hathor/transaction/storage/cache_storage.py index f8f058e5f..63b9af6b3 100644 --- a/hathor/transaction/storage/cache_storage.py +++ b/hathor/transaction/storage/cache_storage.py @@ -17,6 +17,7 @@ from twisted.internet import threads +from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction @@ -32,8 +33,17 @@ class TransactionCacheStorage(BaseTransactionStorage): cache: OrderedDict[bytes, BaseTransaction] dirty_txs: set[bytes] - def __init__(self, store: 'BaseTransactionStorage', reactor: Reactor, interval: int = 5, - capacity: int = 10000, *, indexes: Optional[IndexesManager], _clone_if_needed: bool = False): + def __init__( + self, + store: 'BaseTransactionStorage', + reactor: Reactor, + interval: int = 5, + capacity: int = 10000, + *, + settings: HathorSettings, + indexes: Optional[IndexesManager], + _clone_if_needed: bool = False, + ) -> None: """ :param store: a subclass of BaseTransactionStorage :type store: :py:class:`hathor.transaction.storage.BaseTransactionStorage` @@ -68,7 +78,7 @@ def __init__(self, store: 'BaseTransactionStorage', reactor: Reactor, interval: # we need to use only one weakref dict, so we must first initialize super, and then # attribute the same weakref for both. - super().__init__(indexes=indexes) + super().__init__(indexes=indexes, settings=settings) self._tx_weakref = store._tx_weakref # XXX: just to make sure this isn't being used anywhere, setters/getters should be used instead del self._allow_scope diff --git a/hathor/transaction/storage/memory_storage.py b/hathor/transaction/storage/memory_storage.py index 25dba96ee..efb47b1ba 100644 --- a/hathor/transaction/storage/memory_storage.py +++ b/hathor/transaction/storage/memory_storage.py @@ -14,6 +14,7 @@ from typing import Any, Iterator, Optional, TypeVar +from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.migrations import MigrationState @@ -25,7 +26,13 @@ class TransactionMemoryStorage(BaseTransactionStorage): - def __init__(self, indexes: Optional[IndexesManager] = None, *, _clone_if_needed: bool = False) -> None: + def __init__( + self, + indexes: Optional[IndexesManager] = None, + *, + settings: HathorSettings, + _clone_if_needed: bool = False, + ) -> None: """ :param _clone_if_needed: *private parameter*, defaults to True, controls whether to clone transaction/blocks/metadata when returning those objects. @@ -36,7 +43,7 @@ def __init__(self, indexes: Optional[IndexesManager] = None, *, _clone_if_needed # Store custom key/value attributes self.attributes: dict[str, Any] = {} self._clone_if_needed = _clone_if_needed - super().__init__(indexes=indexes) + super().__init__(indexes=indexes, settings=settings) def _check_and_set_network(self) -> None: # XXX: does not apply to memory storage, can safely be ignored diff --git a/hathor/transaction/storage/rocksdb_storage.py b/hathor/transaction/storage/rocksdb_storage.py index ec74e6227..50c7be615 100644 --- a/hathor/transaction/storage/rocksdb_storage.py +++ b/hathor/transaction/storage/rocksdb_storage.py @@ -16,11 +16,13 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.indexes import IndexesManager from hathor.storage import RocksDBStorage from hathor.transaction.storage.exceptions import TransactionDoesNotExist from hathor.transaction.storage.migrations import MigrationState from hathor.transaction.storage.transaction_storage import BaseTransactionStorage +from hathor.transaction.vertex_parser import VertexParser from hathor.util import json_dumpb, json_loadb if TYPE_CHECKING: @@ -43,7 +45,14 @@ class TransactionRocksDBStorage(BaseTransactionStorage): It uses Protobuf serialization internally. """ - def __init__(self, rocksdb_storage: RocksDBStorage, indexes: Optional[IndexesManager] = None): + def __init__( + self, + rocksdb_storage: RocksDBStorage, + indexes: Optional[IndexesManager] = None, + *, + settings: HathorSettings, + vertex_parser: VertexParser, + ) -> None: self._cf_tx = rocksdb_storage.get_or_create_column_family(_CF_NAME_TX) self._cf_meta = rocksdb_storage.get_or_create_column_family(_CF_NAME_META) self._cf_attr = rocksdb_storage.get_or_create_column_family(_CF_NAME_ATTR) @@ -51,13 +60,13 @@ def __init__(self, rocksdb_storage: RocksDBStorage, indexes: Optional[IndexesMan self._rocksdb_storage = rocksdb_storage self._db = rocksdb_storage.get_db() - super().__init__(indexes=indexes) + self.vertex_parser = vertex_parser + super().__init__(indexes=indexes, settings=settings) def _load_from_bytes(self, tx_data: bytes, meta_data: bytes) -> 'BaseTransaction': - from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.transaction_metadata import TransactionMetadata - tx = tx_or_block_from_bytes(tx_data) + tx = self.vertex_parser.deserialize(tx_data) tx._metadata = TransactionMetadata.create_from_json(json_loadb(meta_data)) tx.storage = self return tx diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 9b90af63f..85a978b2c 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -14,7 +14,7 @@ import hashlib from abc import ABC, abstractmethod, abstractproperty -from collections import defaultdict, deque +from collections import deque from contextlib import AbstractContextManager from threading import Lock from typing import Any, Iterator, NamedTuple, Optional, cast @@ -23,7 +23,7 @@ from intervaltree.interval import Interval from structlog import get_logger -from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.execution_manager import ExecutionManager from hathor.indexes import IndexesManager from hathor.indexes.height_index import HeightInfo @@ -104,8 +104,8 @@ class TransactionStorage(ABC): _migrations: list[BaseMigration] - def __init__(self) -> None: - self._settings = get_global_settings() + def __init__(self, *, settings: HathorSettings) -> None: + self._settings = settings # Weakref is used to guarantee that there is only one instance of each transaction in memory. self._tx_weakref: WeakValueDictionary[bytes, BaseTransaction] = WeakValueDictionary() self._tx_weakref_disabled: bool = False @@ -478,47 +478,6 @@ def remove_transaction(self, tx: BaseTransaction) -> None: if self.indexes is not None: self.del_from_indexes(tx, remove_all=True, relax_assert=True) - def remove_transactions(self, txs: list[BaseTransaction]) -> None: - """Will remove all the transactions on the list from the database. - - Special notes: - - - will refuse and raise an error when removing all transactions would leave dangling transactions, that is, - transactions without existing parent. That is, it expects the `txs` list to include all children of deleted - txs, from both the confirmation and funds DAGs - - inputs's spent_outputs should not have any of the transactions being removed as spending transactions, - this method will update and save those transaction's metadata - - parent's children metadata will be updated to reflect the removals - - all indexes will be updated - """ - parents_to_update: dict[bytes, list[bytes]] = defaultdict(list) - dangling_children: set[bytes] = set() - txset = {tx.hash for tx in txs} - for tx in txs: - tx_meta = tx.get_metadata() - assert not tx_meta.validation.is_checkpoint() - for parent in set(tx.parents) - txset: - parents_to_update[parent].append(tx.hash) - dangling_children.update(set(tx_meta.children) - txset) - for spending_txs in tx_meta.spent_outputs.values(): - dangling_children.update(set(spending_txs) - txset) - for tx_input in tx.inputs: - spent_tx = tx.get_spent_tx(tx_input) - spent_tx_meta = spent_tx.get_metadata() - if tx.hash in spent_tx_meta.spent_outputs[tx_input.index]: - spent_tx_meta.spent_outputs[tx_input.index].remove(tx.hash) - self.save_transaction(spent_tx, only_metadata=True) - assert not dangling_children, 'It is an error to try to remove transactions that would leave a gap in the DAG' - for parent_hash, children_to_remove in parents_to_update.items(): - parent_tx = self.get_transaction(parent_hash) - parent_meta = parent_tx.get_metadata() - for child in children_to_remove: - parent_meta.children.remove(child) - self.save_transaction(parent_tx, only_metadata=True) - for tx in txs: - self.log.debug('remove transaction', tx=tx.hash_hex) - self.remove_transaction(tx) - @abstractmethod def transaction_exists(self, hash_bytes: bytes) -> bool: """Returns `True` if transaction with hash `hash_bytes` exists. @@ -1111,6 +1070,7 @@ def compute_transactions_that_became_invalid(self, new_best_height: int) -> list def _construct_genesis_block(self) -> Block: """Return the genesis block.""" block = Block( + settings=self._settings, storage=self, nonce=self._settings.GENESIS_BLOCK_NONCE, timestamp=self._settings.GENESIS_BLOCK_TIMESTAMP, @@ -1127,6 +1087,7 @@ def _construct_genesis_block(self) -> Block: def _construct_genesis_tx1(self) -> Transaction: """Return the genesis tx1.""" tx1 = Transaction( + settings=self._settings, storage=self, nonce=self._settings.GENESIS_TX1_NONCE, timestamp=self._settings.GENESIS_TX1_TIMESTAMP, @@ -1140,6 +1101,7 @@ def _construct_genesis_tx1(self) -> Transaction: def _construct_genesis_tx2(self) -> Transaction: """Return the genesis tx2.""" tx2 = Transaction( + settings=self._settings, storage=self, nonce=self._settings.GENESIS_TX2_NONCE, timestamp=self._settings.GENESIS_TX2_TIMESTAMP, @@ -1165,8 +1127,14 @@ def get_block(self, block_id: VertexId) -> Block: class BaseTransactionStorage(TransactionStorage): indexes: Optional[IndexesManager] - def __init__(self, indexes: Optional[IndexesManager] = None, pubsub: Optional[Any] = None) -> None: - super().__init__() + def __init__( + self, + indexes: Optional[IndexesManager] = None, + pubsub: Optional[Any] = None, + *, + settings: HathorSettings, + ) -> None: + super().__init__(settings=settings) # Pubsub is used to publish tx voided and winner but it's optional self.pubsub = pubsub diff --git a/hathor/transaction/token_creation_tx.py b/hathor/transaction/token_creation_tx.py index 61a676b2a..629050197 100644 --- a/hathor/transaction/token_creation_tx.py +++ b/hathor/transaction/token_creation_tx.py @@ -17,6 +17,7 @@ from typing_extensions import override +from hathor.conf.settings import HathorSettings from hathor.transaction.base_transaction import TxInput, TxOutput, TxVersion from hathor.transaction.storage import TransactionStorage # noqa: F401 from hathor.transaction.transaction import TokenInfo, Transaction @@ -35,21 +36,35 @@ class TokenCreationTransaction(Transaction): - def __init__(self, - nonce: int = 0, - timestamp: Optional[int] = None, - signal_bits: int = 0, - version: TxVersion = TxVersion.TOKEN_CREATION_TRANSACTION, - weight: float = 0, - inputs: Optional[list[TxInput]] = None, - outputs: Optional[list[TxOutput]] = None, - parents: Optional[list[bytes]] = None, - hash: Optional[bytes] = None, - token_name: str = '', - token_symbol: str = '', - storage: Optional['TransactionStorage'] = None) -> None: - super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight, - inputs=inputs, outputs=outputs or [], parents=parents or [], hash=hash, storage=storage) + def __init__( + self, + nonce: int = 0, + timestamp: Optional[int] = None, + signal_bits: int = 0, + version: TxVersion = TxVersion.TOKEN_CREATION_TRANSACTION, + weight: float = 0, + inputs: Optional[list[TxInput]] = None, + outputs: Optional[list[TxOutput]] = None, + parents: Optional[list[bytes]] = None, + hash: Optional[bytes] = None, + token_name: str = '', + token_symbol: str = '', + storage: Optional['TransactionStorage'] = None, + settings: HathorSettings | None = None, + ) -> None: + super().__init__( + nonce=nonce, + timestamp=timestamp, + signal_bits=signal_bits, + version=version, + weight=weight, + inputs=inputs, + outputs=outputs or [], + parents=parents or [], + hash=hash, + storage=storage, + settings=settings, + ) self.token_name = token_name self.token_symbol = token_symbol # for this special tx, its own hash is used as the created token uid. We're artificially diff --git a/hathor/transaction/transaction.py b/hathor/transaction/transaction.py index a9d9fec5a..a787339d8 100644 --- a/hathor/transaction/transaction.py +++ b/hathor/transaction/transaction.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import hashlib from itertools import chain from struct import pack @@ -19,7 +21,6 @@ from hathor.checkpoint import Checkpoint from hathor.exception import InvalidNewTransaction -from hathor.profiler import get_cpu_profiler from hathor.reward_lock import iter_spent_rewards from hathor.transaction import BaseTransaction, TxInput, TxOutput, TxVersion from hathor.transaction.base_transaction import TX_HASH_SIZE @@ -29,10 +30,9 @@ from hathor.util import not_none if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings from hathor.transaction.storage import TransactionStorage # noqa: F401 -cpu = get_cpu_profiler() - # Signal bits (B), version (B), token uids len (B) and inputs len (B), outputs len (B). _FUNDS_FORMAT_STRING = '!BBBBB' @@ -55,24 +55,38 @@ class Transaction(BaseTransaction): SERIALIZATION_NONCE_SIZE = 4 - def __init__(self, - nonce: int = 0, - timestamp: Optional[int] = None, - signal_bits: int = 0, - version: TxVersion = TxVersion.REGULAR_TRANSACTION, - weight: float = 0, - inputs: Optional[list[TxInput]] = None, - outputs: Optional[list[TxOutput]] = None, - parents: Optional[list[VertexId]] = None, - tokens: Optional[list[TokenUid]] = None, - hash: Optional[VertexId] = None, - storage: Optional['TransactionStorage'] = None) -> None: + def __init__( + self, + nonce: int = 0, + timestamp: Optional[int] = None, + signal_bits: int = 0, + version: TxVersion = TxVersion.REGULAR_TRANSACTION, + weight: float = 0, + inputs: Optional[list[TxInput]] = None, + outputs: Optional[list[TxOutput]] = None, + parents: Optional[list[VertexId]] = None, + tokens: Optional[list[TokenUid]] = None, + hash: Optional[VertexId] = None, + storage: Optional['TransactionStorage'] = None, + settings: HathorSettings | None = None, + ) -> None: """ Creating new init just to make sure inputs will always be empty array Inputs: all inputs that are being used (empty in case of a block) """ - super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight, - inputs=inputs or [], outputs=outputs or [], parents=parents or [], hash=hash, storage=storage) + super().__init__( + nonce=nonce, + timestamp=timestamp, + signal_bits=signal_bits, + version=version, + weight=weight, + inputs=inputs or [], + outputs=outputs or [], + parents=parents or [], + hash=hash, + storage=storage, + settings=settings + ) self.tokens = tokens or [] self._sighash_cache: Optional[bytes] = None self._sighash_data_cache: Optional[bytes] = None diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 17ed326a1..999691638 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from collections import defaultdict from typing import TYPE_CHECKING, Any, Optional @@ -24,6 +26,7 @@ if TYPE_CHECKING: from weakref import ReferenceType # noqa: F401 + from hathor.conf.settings import HathorSettings from hathor.transaction import BaseTransaction from hathor.transaction.storage import TransactionStorage @@ -71,7 +74,8 @@ def __init__( score: float = 0, height: Optional[int] = None, min_height: Optional[int] = None, - feature_activation_bit_counts: Optional[list[int]] = None + feature_activation_bit_counts: Optional[list[int]] = None, + settings: HathorSettings | None = None, ) -> None: from hathor.transaction.genesis import is_genesis @@ -129,7 +133,7 @@ def __init__( self.feature_activation_bit_counts = feature_activation_bit_counts - settings = get_global_settings() + settings = settings or get_global_settings() # Genesis specific: if hash is not None and is_genesis(hash, settings=settings): diff --git a/hathor/transaction/util.py b/hathor/transaction/util.py index a7970359a..d1bec3832 100644 --- a/hathor/transaction/util.py +++ b/hathor/transaction/util.py @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import re import struct from math import ceil, floor -from typing import Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional -from hathor.conf.get_settings import get_global_settings +if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings VerboseCallback = Optional[Callable[[str, Any], None]] @@ -48,13 +51,11 @@ def unpack_len(n: int, buf: bytes) -> tuple[bytes, bytes]: return buf[:n], buf[n:] -def get_deposit_amount(mint_amount: int) -> int: - settings = get_global_settings() +def get_deposit_amount(settings: HathorSettings, mint_amount: int) -> int: return ceil(abs(settings.TOKEN_DEPOSIT_PERCENTAGE * mint_amount)) -def get_withdraw_amount(melt_amount: int) -> int: - settings = get_global_settings() +def get_withdraw_amount(settings: HathorSettings, melt_amount: int) -> int: return floor(abs(settings.TOKEN_DEPOSIT_PERCENTAGE * melt_amount)) diff --git a/hathor/transaction/vertex_parser.py b/hathor/transaction/vertex_parser.py new file mode 100644 index 000000000..03979123f --- /dev/null +++ b/hathor/transaction/vertex_parser.py @@ -0,0 +1,46 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from struct import error as StructError +from typing import TYPE_CHECKING + +from hathor.conf.settings import HathorSettings + +if TYPE_CHECKING: + from hathor.transaction import BaseTransaction + from hathor.transaction.storage import TransactionStorage + + +class VertexParser: + __slots__ = ('_settings',) + + def __init__(self, *, settings: HathorSettings) -> None: + self._settings = settings + + def deserialize(self, data: bytes, storage: TransactionStorage | None = None) -> BaseTransaction: + """ Creates the correct tx subclass from a sequence of bytes + """ + # version field takes up the second byte only + from hathor.transaction import TxVersion + version = data[1] + try: + tx_version = TxVersion(version) + if not self._settings.CONSENSUS_ALGORITHM.is_vertex_version_valid(tx_version, include_genesis=True): + raise StructError(f"invalid vertex version: {tx_version}") + cls = tx_version.get_cls() + return cls.create_from_struct(data, storage=storage) + except ValueError: + raise StructError('Invalid bytes to create transaction subclass.') diff --git a/hathor/types.py b/hathor/types.py index 40ad6dead..7dfa808aa 100644 --- a/hathor/types.py +++ b/hathor/types.py @@ -24,3 +24,87 @@ Timestamp: TypeAlias = int # NewType('Timestamp', int) TokenUid: TypeAlias = VertexId # NewType('TokenUid', VertexId) Amount: TypeAlias = int # NewType('Amount', int) + + +class Hash: + r""" A type for easily representing a 32-byte hash, it is not meant to be used directly. + + Instead it is meant to be used to make new-types that also happen to be a hash. This class will provide convenient + methods for parsing and representing it. + + Examples: + + >>> x = Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc') + >>> bytes(x) + b'\x00\x00\x06\xcb\x938[\x8b\x87\xa5E\xa1\xcb\xb6\x19~l\xaf\xf6\x00\xc1,\xc1/\xc5BP\xd3\x9c\x80\x88\xfc' + + >>> Hash(b'\x00\x00\x06\xcb\x938[\x8b\x87\xa5E\xa1\xcb\xb6\x19~l\xaf\xf6\x00\xc1,\xc1/\xc5BP\xd3\x9c\x80\x88\xfc') + Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc') + + >>> str(x) + '000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc' + + >>> repr(x) + "Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc')" + + >>> {x} + {Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc')} + + >>> class Foo(Hash): + ... pass + >>> y = Foo('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc') + >>> repr(y) + "Foo('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc')" + + >>> x == y + True + + >>> {x: 123}[y] + 123 + + >>> Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088fc34') + Traceback (most recent call last): + ... + ValueError: expected 32 bytes, got 33 bytes + + >>> Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088') + Traceback (most recent call last): + ... + ValueError: expected 32 bytes, got 31 bytes + + >>> Hash('000006cb93385b8b87a545a1cbb6197e6caff600c12cc12fc54250d39c8088f') + Traceback (most recent call last): + ... + ValueError: non-hexadecimal number found in fromhex() arg at position 63 + + >>> Hash(123) + Traceback (most recent call last): + ... + TypeError: expected a bytes or str instance, got a instead + """ + __slots__ = ('_inner',) + _inner: bytes + + def __init__(self, inner: bytes | str) -> None: + if isinstance(inner, str): + inner = bytes.fromhex(inner) + if not isinstance(inner, bytes): + raise TypeError(f'expected a bytes or str instance, got a {repr(type(inner))} instead') + if len(inner) != 32: + raise ValueError(f'expected 32 bytes, got {len(inner)} bytes') + self._inner = inner + + def __bytes__(self): + return self._inner + + def __str__(self): + return self._inner.hex() + + def __repr__(self): + return f"{type(self).__name__}('{self}')" + + def __hash__(self): + return hash(self._inner) + + def __eq__(self, other): + return self._inner.__eq__(other._inner) diff --git a/hathor/verification/block_verifier.py b/hathor/verification/block_verifier.py index 2110bbd91..2935e24b4 100644 --- a/hathor/verification/block_verifier.py +++ b/hathor/verification/block_verifier.py @@ -51,6 +51,7 @@ def verify_height(self, block: Block) -> None: def verify_weight(self, block: Block) -> None: """Validate minimum block difficulty.""" + assert self._settings.CONSENSUS_ALGORITHM.is_pow() assert block.storage is not None min_block_weight = self._daa.calculate_block_difficulty(block, block.storage.get_parent_block) if block.weight < min_block_weight - self._settings.WEIGHT_TOL: diff --git a/hathor/verification/poa_block_verifier.py b/hathor/verification/poa_block_verifier.py new file mode 100644 index 000000000..b1befc24a --- /dev/null +++ b/hathor/verification/poa_block_verifier.py @@ -0,0 +1,51 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings +from hathor.transaction.exceptions import PoaValidationError +from hathor.transaction.poa import PoaBlock + + +class PoaBlockVerifier: + __slots__ = ('_settings',) + + def __init__(self, *, settings: HathorSettings): + self._settings = settings + + def verify_poa(self, block: PoaBlock) -> None: + """Validate the Proof-of-Authority.""" + poa_settings = self._settings.CONSENSUS_ALGORITHM + assert isinstance(poa_settings, PoaSettings) + + parent_block = block.get_block_parent() + if block.timestamp < parent_block.timestamp + self._settings.AVG_TIME_BETWEEN_BLOCKS: + raise PoaValidationError( + f'blocks must have at least {self._settings.AVG_TIME_BETWEEN_BLOCKS} seconds between them' + ) + + # validate block rewards + if block.outputs: + raise PoaValidationError('blocks must not have rewards in a PoA network') + + # validate that the signature is valid + signature_validation = poa.verify_poa_signature(poa_settings, block) + if not isinstance(signature_validation, poa.ValidSignature): + raise PoaValidationError('invalid PoA signature') + + # validate block weight is in turn + expected_weight = poa.calculate_weight(poa_settings, block, signature_validation.signer_index) + if block.weight != expected_weight: + raise PoaValidationError(f'block weight is {block.weight}, expected {expected_weight}') diff --git a/hathor/verification/transaction_verifier.py b/hathor/verification/transaction_verifier.py index f55e0239c..05f5f3189 100644 --- a/hathor/verification/transaction_verifier.py +++ b/hathor/verification/transaction_verifier.py @@ -63,6 +63,7 @@ def verify_parents_basic(self, tx: Transaction) -> None: def verify_weight(self, tx: Transaction) -> None: """Validate minimum tx difficulty.""" + assert self._settings.CONSENSUS_ALGORITHM.is_pow() min_tx_weight = self._daa.minimum_tx_weight(tx) max_tx_weight = min_tx_weight + self._settings.MAX_TX_WEIGHT_DIFF if tx.weight < min_tx_weight - self._settings.WEIGHT_TOL: @@ -231,13 +232,13 @@ def verify_sum(self, token_dict: dict[TokenUid, TokenInfo]) -> None: if not token_info.can_melt: raise InputOutputMismatch('{} {} tokens melted, but there is no melt authority input'.format( token_info.amount, token_uid.hex())) - withdraw += get_withdraw_amount(token_info.amount) + withdraw += get_withdraw_amount(self._settings, token_info.amount) else: # tokens have been minted if not token_info.can_mint: raise InputOutputMismatch('{} {} tokens minted, but there is no mint authority input'.format( (-1) * token_info.amount, token_uid.hex())) - deposit += get_deposit_amount(token_info.amount) + deposit += get_deposit_amount(self._settings, token_info.amount) # check whether the deposit/withdraw amount is correct htr_expected_amount = withdraw - deposit diff --git a/hathor/verification/verification_service.py b/hathor/verification/verification_service.py index efa18c6f6..3f75c0b9b 100644 --- a/hathor/verification/verification_service.py +++ b/hathor/verification/verification_service.py @@ -14,8 +14,10 @@ from typing_extensions import assert_never +from hathor.conf.settings import HathorSettings from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, Block, MergeMinedBlock, Transaction, TxVersion +from hathor.transaction.poa import PoaBlock from hathor.transaction.token_creation_tx import TokenCreationTransaction from hathor.transaction.transaction import TokenInfo from hathor.transaction.validation_state import ValidationState @@ -26,9 +28,10 @@ class VerificationService: - __slots__ = ('verifiers', ) + __slots__ = ('_settings', 'verifiers') - def __init__(self, *, verifiers: VertexVerifiers) -> None: + def __init__(self, *, settings: HathorSettings, verifiers: VertexVerifiers) -> None: + self._settings = settings self.verifiers = verifiers def validate_basic(self, vertex: BaseTransaction, *, skip_block_weight_verification: bool = False) -> bool: @@ -82,6 +85,8 @@ def verify_basic(self, vertex: BaseTransaction, *, skip_block_weight_verificatio """Basic verifications (the ones without access to dependencies: parents+inputs). Raises on error. Used by `self.validate_basic`. Should not modify the validation state.""" + self.verifiers.vertex.verify_version(vertex) + # We assert with type() instead of isinstance() because each subclass has a specific branch. match vertex.version: case TxVersion.REGULAR_BLOCK: @@ -90,6 +95,9 @@ def verify_basic(self, vertex: BaseTransaction, *, skip_block_weight_verificatio case TxVersion.MERGE_MINED_BLOCK: assert type(vertex) is MergeMinedBlock self._verify_basic_merge_mined_block(vertex, skip_weight_verification=skip_block_weight_verification) + case TxVersion.POA_BLOCK: + assert type(vertex) is PoaBlock + self._verify_basic_poa_block(vertex) case TxVersion.REGULAR_TRANSACTION: assert type(vertex) is Transaction self._verify_basic_tx(vertex) @@ -108,13 +116,18 @@ def _verify_basic_block(self, block: Block, *, skip_weight_verification: bool) - def _verify_basic_merge_mined_block(self, block: MergeMinedBlock, *, skip_weight_verification: bool) -> None: self._verify_basic_block(block, skip_weight_verification=skip_weight_verification) + def _verify_basic_poa_block(self, block: PoaBlock) -> None: + self.verifiers.poa_block.verify_poa(block) + self.verifiers.block.verify_reward(block) + def _verify_basic_tx(self, tx: Transaction) -> None: """Partially run validations, the ones that need parents/inputs are skipped.""" if tx.is_genesis: # TODO do genesis validation? return self.verifiers.tx.verify_parents_basic(tx) - self.verifiers.tx.verify_weight(tx) + if self._settings.CONSENSUS_ALGORITHM.is_pow(): + self.verifiers.tx.verify_weight(tx) self.verify_without_storage(tx) def _verify_basic_token_creation_tx(self, tx: TokenCreationTransaction) -> None: @@ -132,6 +145,9 @@ def verify(self, vertex: BaseTransaction, *, reject_locked_reward: bool = True) case TxVersion.MERGE_MINED_BLOCK: assert type(vertex) is MergeMinedBlock self._verify_merge_mined_block(vertex) + case TxVersion.POA_BLOCK: + assert type(vertex) is PoaBlock + self._verify_poa_block(vertex) case TxVersion.REGULAR_TRANSACTION: assert type(vertex) is Transaction self._verify_tx(vertex, reject_locked_reward=reject_locked_reward) @@ -168,6 +184,9 @@ def _verify_block(self, block: Block) -> None: def _verify_merge_mined_block(self, block: MergeMinedBlock) -> None: self._verify_block(block) + def _verify_poa_block(self, block: PoaBlock) -> None: + self._verify_block(block) + @cpu.profiler(key=lambda _, tx: 'tx-verify!{}'.format(tx.hash.hex())) def _verify_tx( self, @@ -217,6 +236,9 @@ def verify_without_storage(self, vertex: BaseTransaction) -> None: case TxVersion.MERGE_MINED_BLOCK: assert type(vertex) is MergeMinedBlock self._verify_without_storage_merge_mined_block(vertex) + case TxVersion.POA_BLOCK: + assert type(vertex) is PoaBlock + self._verify_without_storage_poa_block(vertex) case TxVersion.REGULAR_TRANSACTION: assert type(vertex) is Transaction self._verify_without_storage_tx(vertex) @@ -226,24 +248,31 @@ def verify_without_storage(self, vertex: BaseTransaction) -> None: case _: assert_never(vertex.version) - def _verify_without_storage_block(self, block: Block) -> None: - """ Run all verifications that do not need a storage. - """ - self.verifiers.vertex.verify_pow(block) + def _verify_without_storage_base_block(self, block: Block) -> None: self.verifiers.block.verify_no_inputs(block) self.verifiers.vertex.verify_outputs(block) self.verifiers.block.verify_output_token_indexes(block) self.verifiers.block.verify_data(block) self.verifiers.vertex.verify_sigops_output(block) + def _verify_without_storage_block(self, block: Block) -> None: + """ Run all verifications that do not need a storage. + """ + self.verifiers.vertex.verify_pow(block) + self._verify_without_storage_base_block(block) + def _verify_without_storage_merge_mined_block(self, block: MergeMinedBlock) -> None: self.verifiers.merge_mined_block.verify_aux_pow(block) self._verify_without_storage_block(block) + def _verify_without_storage_poa_block(self, block: PoaBlock) -> None: + self._verify_without_storage_base_block(block) + def _verify_without_storage_tx(self, tx: Transaction) -> None: """ Run all verifications that do not need a storage. """ - self.verifiers.vertex.verify_pow(tx) + if self._settings.CONSENSUS_ALGORITHM.is_pow(): + self.verifiers.vertex.verify_pow(tx) self.verifiers.tx.verify_number_of_inputs(tx) self.verifiers.vertex.verify_outputs(tx) self.verifiers.tx.verify_output_token_indexes(tx) diff --git a/hathor/verification/vertex_verifier.py b/hathor/verification/vertex_verifier.py index d3ef72046..0e4282410 100644 --- a/hathor/verification/vertex_verifier.py +++ b/hathor/verification/vertex_verifier.py @@ -22,6 +22,7 @@ InvalidOutputScriptSize, InvalidOutputValue, InvalidToken, + InvalidVersionError, ParentDoesNotExist, PowError, TimestampError, @@ -44,6 +45,11 @@ class VertexVerifier: def __init__(self, *, settings: HathorSettings) -> None: self._settings = settings + def verify_version(self, vertex: BaseTransaction) -> None: + """Verify that the vertex version is valid.""" + if not self._settings.CONSENSUS_ALGORITHM.is_vertex_version_valid(vertex.version): + raise InvalidVersionError(f"invalid vertex version: {vertex.version}") + def verify_parents(self, vertex: BaseTransaction) -> None: """All parents must exist and their timestamps must be smaller than ours. @@ -126,6 +132,7 @@ def verify_pow(self, vertex: BaseTransaction, *, override_weight: Optional[float :raises PowError: when the hash is equal or greater than the target """ + assert self._settings.CONSENSUS_ALGORITHM.is_pow() numeric_hash = int(vertex.hash_hex, vertex.HEX_BASE) minimum_target = vertex.get_target(override_weight) if numeric_hash >= minimum_target: diff --git a/hathor/verification/vertex_verifiers.py b/hathor/verification/vertex_verifiers.py index 31e3fe190..1a9b56b21 100644 --- a/hathor/verification/vertex_verifiers.py +++ b/hathor/verification/vertex_verifiers.py @@ -19,6 +19,7 @@ from hathor.feature_activation.feature_service import FeatureService from hathor.verification.block_verifier import BlockVerifier from hathor.verification.merge_mined_block_verifier import MergeMinedBlockVerifier +from hathor.verification.poa_block_verifier import PoaBlockVerifier from hathor.verification.token_creation_transaction_verifier import TokenCreationTransactionVerifier from hathor.verification.transaction_verifier import TransactionVerifier from hathor.verification.vertex_verifier import VertexVerifier @@ -29,6 +30,7 @@ class VertexVerifiers(NamedTuple): vertex: VertexVerifier block: BlockVerifier merge_mined_block: MergeMinedBlockVerifier + poa_block: PoaBlockVerifier tx: TransactionVerifier token_creation_tx: TokenCreationTransactionVerifier @@ -67,6 +69,7 @@ def create( """ block_verifier = BlockVerifier(settings=settings, daa=daa, feature_service=feature_service) merge_mined_block_verifier = MergeMinedBlockVerifier(settings=settings, feature_service=feature_service) + poa_block_verifier = PoaBlockVerifier(settings=settings) tx_verifier = TransactionVerifier(settings=settings, daa=daa) token_creation_tx_verifier = TokenCreationTransactionVerifier(settings=settings) @@ -74,6 +77,7 @@ def create( vertex=vertex_verifier, block=block_verifier, merge_mined_block=merge_mined_block_verifier, + poa_block=poa_block_verifier, tx=tx_verifier, token_creation_tx=token_creation_tx_verifier, ) diff --git a/hathor/version.py b/hathor/version.py index ba87fa88a..7b4749024 100644 --- a/hathor/version.py +++ b/hathor/version.py @@ -19,7 +19,7 @@ from structlog import get_logger -BASE_VERSION = '0.60.1' +BASE_VERSION = '0.62.0' DEFAULT_VERSION_SUFFIX = "local" BUILD_VERSION_FILE_PATH = "./BUILD_VERSION" diff --git a/hathor/version_resource.py b/hathor/version_resource.py index ec64c3114..57d2801f2 100644 --- a/hathor/version_resource.py +++ b/hathor/version_resource.py @@ -51,6 +51,14 @@ def render_GET(self, request): 'reward_spend_min_blocks': self._settings.REWARD_SPEND_MIN_BLOCKS, 'max_number_inputs': self._settings.MAX_NUM_INPUTS, 'max_number_outputs': self._settings.MAX_NUM_OUTPUTS, + 'decimal_places': self._settings.DECIMAL_PLACES, + 'genesis_block_hash': self._settings.GENESIS_BLOCK_HASH.hex(), + 'genesis_tx1_hash': self._settings.GENESIS_TX1_HASH.hex(), + 'genesis_tx2_hash': self._settings.GENESIS_TX2_HASH.hex(), + 'native_token': dict( + name=self._settings.NATIVE_TOKEN_NAME, + symbol=self._settings.NATIVE_TOKEN_SYMBOL, + ), } return json_dumpb(data) diff --git a/hathor/vertex_handler/vertex_handler.py b/hathor/vertex_handler/vertex_handler.py index 0ef601aa0..903af0d31 100644 --- a/hathor/vertex_handler/vertex_handler.py +++ b/hathor/vertex_handler/vertex_handler.py @@ -19,7 +19,6 @@ from hathor.conf.settings import HathorSettings from hathor.consensus import ConsensusAlgorithm from hathor.exception import HathorError, InvalidNewTransaction -from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import FeatureService from hathor.p2p.manager import ConnectionsManager from hathor.pubsub import HathorEvents, PubSubManager @@ -209,7 +208,6 @@ def _post_consensus( self._wallet.on_new_tx(vertex) self._log_new_object(vertex, 'new {}', quiet=quiet) - self._log_feature_states(vertex) if propagate_to_peers: # Propagate to our peers. @@ -231,7 +229,13 @@ def _log_new_object(self, tx: BaseTransaction, message_fmt: str, *, quiet: bool) if tx.is_block: message = message_fmt.format('block') if isinstance(tx, Block): - kwargs['height'] = tx.get_height() + feature_descriptions = self._feature_service.get_bits_description(block=tx) + feature_states = { + feature.value: description.state.value + for feature, description in feature_descriptions.items() + } + kwargs['_height'] = tx.get_height() + kwargs['feature_states'] = feature_states else: message = message_fmt.format('tx') if not quiet: @@ -239,35 +243,3 @@ def _log_new_object(self, tx: BaseTransaction, message_fmt: str, *, quiet: bool) else: log_func = self._log.debug log_func(message, **kwargs) - - def _log_feature_states(self, vertex: BaseTransaction) -> None: - """Log features states for a block. Used as part of the Feature Activation Phased Testing.""" - if not isinstance(vertex, Block): - return - - feature_descriptions = self._feature_service.get_bits_description(block=vertex) - state_by_feature = { - feature.value: description.state.value - for feature, description in feature_descriptions.items() - } - - self._log.info( - 'New block accepted with feature activation states', - block_hash=vertex.hash_hex, - block_height=vertex.get_height(), - features_states=state_by_feature - ) - - features = [Feature.NOP_FEATURE_1, Feature.NOP_FEATURE_2] - for feature in features: - self._log_if_feature_is_active(vertex, feature) - - def _log_if_feature_is_active(self, block: Block, feature: Feature) -> None: - """Log if a feature is ACTIVE for a block. Used as part of the Feature Activation Phased Testing.""" - if self._feature_service.is_feature_active(block=block, feature=feature): - self._log.info( - 'Feature is ACTIVE for block', - feature=feature.value, - block_hash=block.hash_hex, - block_height=block.get_height() - ) diff --git a/hathor/wallet/hd_wallet.py b/hathor/wallet/hd_wallet.py index 3773ed4df..3c64d0c1f 100644 --- a/hathor/wallet/hd_wallet.py +++ b/hathor/wallet/hd_wallet.py @@ -173,6 +173,10 @@ def generate_new_key(self, index): new_key = self.chain_key.subkey(index) self._key_generated(new_key, index) + def get_xpub(self) -> str: + """Return wallet xpub after derivation.""" + return self.chain_key.as_text(as_private=False) + def _key_generated(self, key, index): """ Add generated key to self.keys and set last_generated_index diff --git a/hathor/wallet/resources/thin_wallet/address_history.py b/hathor/wallet/resources/thin_wallet/address_history.py index 4f5608871..db0e2c221 100644 --- a/hathor/wallet/resources/thin_wallet/address_history.py +++ b/hathor/wallet/resources/thin_wallet/address_history.py @@ -15,6 +15,7 @@ from json import JSONDecodeError from typing import Any, Optional +from structlog import get_logger from twisted.web.http import Request from hathor.api_util import Resource, get_args, get_missing_params_msg, set_cors @@ -25,6 +26,8 @@ from hathor.util import json_dumpb, json_loadb from hathor.wallet.exceptions import InvalidAddress +logger = get_logger() + @register_resource class AddressHistoryResource(Resource): @@ -36,6 +39,7 @@ class AddressHistoryResource(Resource): def __init__(self, manager): self.manager = manager + self._log = logger.new() settings = get_global_settings() # XXX: copy the parameters that are needed so tests can more easily tweak them self.max_tx_addresses_history = settings.MAX_TX_ADDRESSES_HISTORY @@ -52,9 +56,8 @@ def render_POST(self, request: Request) -> bytes: request.setHeader(b'content-type', b'application/json; charset=utf-8') set_cors(request, 'POST') - if not self.manager.tx_storage.indexes.addresses: - request.setResponseCode(503) - return json_dumpb({'success': False}) + if error_message := self._validate_index(request): + return error_message assert request.content is not None raw_body = request.content.read() or b'' @@ -117,11 +120,8 @@ def render_GET(self, request: Request) -> bytes: request.setHeader(b'content-type', b'application/json; charset=utf-8') set_cors(request, 'GET') - addresses_index = self.manager.tx_storage.indexes.addresses - - if not addresses_index: - request.setResponseCode(503) - return json_dumpb({'success': False}) + if error_message := self._validate_index(request): + return error_message raw_args = get_args(request) @@ -137,6 +137,18 @@ def render_GET(self, request: Request) -> bytes: return self.get_address_history([address.decode('utf-8') for address in addresses], ref_hash) + def _validate_index(self, request: Request) -> bytes | None: + """Return None if validation is successful (addresses index is enabled), and an error message otherwise.""" + if self.manager.tx_storage.indexes.addresses: + return None + + self._log.warn( + 'trying to reach address history endpoint, but addresses index is disabled.\n' + 'use `--wallet-index` to enable it' + ) + request.setResponseCode(503) + return json_dumpb({'success': False, 'message': 'wallet index is disabled'}) + def get_address_history(self, addresses: list[str], ref_hash: Optional[str]) -> bytes: ref_hash_bytes = None if ref_hash: diff --git a/hathor/wallet/resources/thin_wallet/send_tokens.py b/hathor/wallet/resources/thin_wallet/send_tokens.py index 2ffb10dfc..16f21d004 100644 --- a/hathor/wallet/resources/thin_wallet/send_tokens.py +++ b/hathor/wallet/resources/thin_wallet/send_tokens.py @@ -29,7 +29,6 @@ from hathor.exception import InvalidNewTransaction from hathor.reactor import get_global_reactor from hathor.transaction import Transaction -from hathor.transaction.base_transaction import tx_or_block_from_bytes from hathor.transaction.exceptions import TxValidationError from hathor.util import json_dumpb, json_loadb @@ -110,7 +109,7 @@ def render_POST(self, request: Request) -> Any: ) try: - tx = tx_or_block_from_bytes(bytes.fromhex(tx_hex)) + tx = self.manager.vertex_parser.deserialize(bytes.fromhex(tx_hex)) except (ValueError, struct.error): # ValueError: invalid hex # struct.error: invalid transaction data diff --git a/hathor/websocket/exception.py b/hathor/websocket/exception.py new file mode 100644 index 000000000..20f83a0bf --- /dev/null +++ b/hathor/websocket/exception.py @@ -0,0 +1,27 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hathor.exception import HathorError + + +class InvalidXPub(HathorError): + """Raised when an invalid xpub is provided.""" + + +class LimitExceeded(HathorError): + """Raised when a limit is exceeded.""" + + +class InvalidAddress(HathorError): + """Raised when an invalid address is provided.""" diff --git a/hathor/websocket/factory.py b/hathor/websocket/factory.py index 2c7aa2d16..b96dcbdff 100644 --- a/hathor/websocket/factory.py +++ b/hathor/websocket/factory.py @@ -13,7 +13,7 @@ # limitations under the License. from collections import defaultdict, deque -from typing import Any, Optional, Union +from typing import Any, Optional from autobahn.exception import Disconnected from autobahn.twisted.websocket import WebSocketServerFactory @@ -22,11 +22,12 @@ from hathor.conf import HathorSettings from hathor.indexes import AddressIndex +from hathor.manager import HathorManager from hathor.metrics import Metrics from hathor.p2p.rate_limiter import RateLimiter from hathor.pubsub import EventArguments, HathorEvents from hathor.reactor import get_global_reactor -from hathor.util import json_dumpb, json_loadb, json_loads +from hathor.util import json_dumpb from hathor.websocket.protocol import HathorAdminWebsocketProtocol settings = HathorSettings() @@ -83,18 +84,25 @@ class HathorAdminWebsocketFactory(WebSocketServerFactory): max_subs_addrs_empty: Optional[int] = settings.WS_MAX_SUBS_ADDRS_EMPTY def buildProtocol(self, addr): - return self.protocol(self) + return self.protocol(self, is_history_streaming_enabled=self.is_history_streaming_enabled) - def __init__(self, metrics: Optional[Metrics] = None, address_index: Optional[AddressIndex] = None): + def __init__(self, + manager: HathorManager, + metrics: Optional[Metrics] = None, + address_index: Optional[AddressIndex] = None): """ :param metrics: If not given, a new one is created. :type metrics: :py:class:`hathor.metrics.Metrics` """ + self.manager = manager self.reactor = get_global_reactor() # Opened websocket connections so I can broadcast messages later # It contains only connections that have finished handshaking. self.connections: set[HathorAdminWebsocketProtocol] = set() + # Enable/disable history streaming over the websocket connection. + self.is_history_streaming_enabled: bool = True + # Websocket connection for each address self.address_connections: defaultdict[str, set[HathorAdminWebsocketProtocol]] = defaultdict(set) super().__init__() @@ -129,6 +137,12 @@ def stop(self): self._lc_send_metrics.stop() self.is_running = False + def disable_history_streaming(self) -> None: + """Disable history streaming for all connections.""" + self.is_history_streaming_enabled = False + for conn in self.connections: + self.disable_history_streaming() + def _setup_rate_limit(self): """ Set the limit of the RateLimiter and start the buffer deques with BUFFER_SIZE """ @@ -300,44 +314,33 @@ def process_deque(self, data_type): data_type=data_type) break - def handle_message(self, connection: HathorAdminWebsocketProtocol, data: Union[bytes, str]) -> None: - """ General message handler, detects type and deletages to specific handler.""" - if isinstance(data, bytes): - message = json_loadb(data) - else: - message = json_loads(data) - # we only handle ping messages for now - if message['type'] == 'ping': - self._handle_ping(connection, message) - elif message['type'] == 'subscribe_address': - self._handle_subscribe_address(connection, message) - elif message['type'] == 'unsubscribe_address': - self._handle_unsubscribe_address(connection, message) - - def _handle_ping(self, connection: HathorAdminWebsocketProtocol, message: dict[Any, Any]) -> None: - """ Handler for ping message, should respond with a simple {"type": "pong"}""" - payload = json_dumpb({'type': 'pong'}) - connection.sendMessage(payload, False) - def _handle_subscribe_address(self, connection: HathorAdminWebsocketProtocol, message: dict[Any, Any]) -> None: """ Handler for subscription to an address, consideirs subscription limits.""" - addr: str = message['address'] + address: str = message['address'] + success, errmsg = self.subscribe_address(connection, address) + response = { + 'type': 'subscribe_address', + 'address': address, + 'success': success, + } + if not success: + response['message'] = errmsg + connection.sendMessage(json_dumpb(response), False) + + def subscribe_address(self, connection: HathorAdminWebsocketProtocol, address: str) -> tuple[bool, str]: + """Subscribe an address to send real time updates to a websocket connection.""" subs: set[str] = connection.subscribed_to if self.max_subs_addrs_conn is not None and len(subs) >= self.max_subs_addrs_conn: - payload = json_dumpb({'message': 'Reached maximum number of subscribed ' - f'addresses ({self.max_subs_addrs_conn}).', - 'type': 'subscribe_address', 'success': False}) + return False, f'Reached maximum number of subscribed addresses ({self.max_subs_addrs_conn}).' + elif self.max_subs_addrs_empty is not None and ( self.address_index and _count_empty(subs, self.address_index) >= self.max_subs_addrs_empty ): - payload = json_dumpb({'message': 'Reached maximum number of subscribed ' - f'addresses without output ({self.max_subs_addrs_empty}).', - 'type': 'subscribe_address', 'success': False}) - else: - self.address_connections[addr].add(connection) - connection.subscribed_to.add(addr) - payload = json_dumpb({'type': 'subscribe_address', 'success': True}) - connection.sendMessage(payload, False) + return False, f'Reached maximum number of subscribed empty addresses ({self.max_subs_addrs_empty}).' + + self.address_connections[address].add(connection) + connection.subscribed_to.add(address) + return True, '' def _handle_unsubscribe_address(self, connection: HathorAdminWebsocketProtocol, message: dict[Any, Any]) -> None: """ Handler for unsubscribing from an address, also removes address connection set if it ends up empty.""" diff --git a/hathor/websocket/iterators.py b/hathor/websocket/iterators.py new file mode 100644 index 000000000..41f6e7298 --- /dev/null +++ b/hathor/websocket/iterators.py @@ -0,0 +1,151 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import deque +from collections.abc import AsyncIterable +from dataclasses import dataclass +from typing import AsyncIterator, Iterator, TypeAlias + +from twisted.internet.defer import Deferred + +from hathor.manager import HathorManager +from hathor.transaction import BaseTransaction +from hathor.types import AddressB58 +from hathor.websocket.exception import InvalidAddress, InvalidXPub, LimitExceeded + + +@dataclass(frozen=True, slots=True) +class AddressItem: + index: int + address: AddressB58 + + +@dataclass(frozen=True, slots=True) +class VertexItem: + vertex: BaseTransaction + + +class ManualAddressSequencer(AsyncIterable[AddressItem]): + """An async iterable that yields addresses from a list. More addresses + can be added while the iterator is being consumed. + """ + + ADDRESS_SIZE: int = 34 + MAX_PENDING_ADDRESSES_SIZE: int = 5_000 + + def __init__(self) -> None: + self.max_pending_addresses_size: int = self.MAX_PENDING_ADDRESSES_SIZE + self.pending_addresses: deque[AddressItem] = deque() + self.await_items: Deferred | None = None + + # Flag to mark when all addresses have been received so the iterator + # can stop yielding after the pending list of addresses is empty. + self._stop = False + + def _resume_iter(self) -> None: + """Resume yield addresses.""" + if self.await_items is None: + return + if not self.await_items.called: + self.await_items.callback(None) + + def add_addresses(self, addresses: list[AddressItem], last: bool) -> None: + """Add more addresses to be yielded. If `last` is true, the iterator + will stop when the pending list of items gets empty.""" + if len(self.pending_addresses) + len(addresses) > self.max_pending_addresses_size: + raise LimitExceeded + + # Validate addresses. + for item in addresses: + if len(item.address) != self.ADDRESS_SIZE: + raise InvalidAddress(item) + + self.pending_addresses.extend(addresses) + if last: + self._stop = True + self._resume_iter() + + def __aiter__(self) -> AsyncIterator[AddressItem]: + """Return an async iterator.""" + return self._async_iter() + + async def _async_iter(self) -> AsyncIterator[AddressItem]: + """Internal method that implements the async iterator.""" + while True: + while self.pending_addresses: + item = self.pending_addresses.popleft() + yield item + + if self._stop: + break + + self.await_items = Deferred() + await self.await_items + + +def iter_xpub_addresses(xpub_str: str, *, first_index: int = 0) -> Iterator[AddressItem]: + """An iterator that yields addresses derived from an xpub.""" + from pycoin.networks.registry import network_for_netcode + + from hathor.wallet.hd_wallet import _register_pycoin_networks + _register_pycoin_networks() + network = network_for_netcode('htr') + + xpub = network.parse.bip32(xpub_str) + if xpub is None: + raise InvalidXPub(xpub_str) + + idx = first_index + while True: + key = xpub.subkey(idx) + yield AddressItem(idx, AddressB58(key.address())) + idx += 1 + + +async def aiter_xpub_addresses(xpub: str, *, first_index: int = 0) -> AsyncIterator[AddressItem]: + """An async iterator that yields addresses derived from an xpub.""" + it = iter_xpub_addresses(xpub, first_index=first_index) + for item in it: + yield item + + +AddressSearch: TypeAlias = AsyncIterator[AddressItem | VertexItem] + + +async def gap_limit_search( + manager: HathorManager, + address_iter: AsyncIterable[AddressItem], + gap_limit: int +) -> AddressSearch: + """An async iterator that yields addresses and vertices, stopping when the gap limit is reached. + """ + assert manager.tx_storage.indexes is not None + assert manager.tx_storage.indexes.addresses is not None + addresses_index = manager.tx_storage.indexes.addresses + empty_addresses_counter = 0 + async for item in address_iter: + yield item # AddressItem + + vertex_counter = 0 + for vertex_id in addresses_index.get_sorted_from_address(item.address): + tx = manager.tx_storage.get_transaction(vertex_id) + yield VertexItem(tx) + vertex_counter += 1 + + if vertex_counter == 0: + empty_addresses_counter += 1 + if empty_addresses_counter >= gap_limit: + break + else: + empty_addresses_counter = 0 diff --git a/hathor/websocket/messages.py b/hathor/websocket/messages.py new file mode 100644 index 000000000..f8d2e6c9a --- /dev/null +++ b/hathor/websocket/messages.py @@ -0,0 +1,62 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from pydantic import Field + +from hathor.utils.pydantic import BaseModel + + +class WebSocketMessage(BaseModel): + pass + + +class CapabilitiesMessage(WebSocketMessage): + type: str = Field('capabilities', const=True) + capabilities: list[str] + + +class StreamBase(WebSocketMessage): + pass + + +class StreamErrorMessage(StreamBase): + type: str = Field('stream:history:error', const=True) + id: str + errmsg: str + + +class StreamBeginMessage(StreamBase): + type: str = Field('stream:history:begin', const=True) + id: str + + +class StreamEndMessage(StreamBase): + type: str = Field('stream:history:end', const=True) + id: str + + +class StreamVertexMessage(StreamBase): + type: str = Field('stream:history:vertex', const=True) + id: str + data: dict[str, Any] + + +class StreamAddressMessage(StreamBase): + type: str = Field('stream:history:address', const=True) + id: str + index: int + address: str + subscribed: bool diff --git a/hathor/websocket/protocol.py b/hathor/websocket/protocol.py index 5429b9506..1b3bde0bb 100644 --- a/hathor/websocket/protocol.py +++ b/hathor/websocket/protocol.py @@ -12,11 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Union from autobahn.twisted.websocket import WebSocketServerProtocol from structlog import get_logger +from hathor.p2p.utils import format_address +from hathor.util import json_dumpb, json_loadb, json_loads +from hathor.websocket.exception import InvalidAddress, InvalidXPub, LimitExceeded +from hathor.websocket.iterators import ( + AddressItem, + AddressSearch, + ManualAddressSequencer, + aiter_xpub_addresses, + gap_limit_search, +) +from hathor.websocket.messages import CapabilitiesMessage, StreamErrorMessage, WebSocketMessage +from hathor.websocket.streamer import HistoryStreamer + if TYPE_CHECKING: from hathor.websocket.factory import HathorAdminWebsocketFactory @@ -30,23 +43,269 @@ class HathorAdminWebsocketProtocol(WebSocketServerProtocol): can send the data update to the clients """ - def __init__(self, factory: 'HathorAdminWebsocketFactory') -> None: + MAX_GAP_LIMIT: int = 10_000 + HISTORY_STREAMING_CAPABILITY: str = 'history-streaming' + + def __init__(self, + factory: 'HathorAdminWebsocketFactory', + is_history_streaming_enabled: bool) -> None: self.log = logger.new() self.factory = factory - self.subscribed_to: set[str] = set() super().__init__() + self.subscribed_to: set[str] = set() + + # Enable/disable history streaming for this connection. + self.is_history_streaming_enabled = is_history_streaming_enabled + self._history_streamer: HistoryStreamer | None = None + self._manual_address_iter: ManualAddressSequencer | None = None + + def get_capabilities(self) -> list[str]: + """Get a list of websocket capabilities.""" + capabilities = [] + if self.is_history_streaming_enabled: + capabilities.append(self.HISTORY_STREAMING_CAPABILITY) + return capabilities + + def send_capabilities(self) -> None: + """Send a capabilities message.""" + self.send_message(CapabilitiesMessage(capabilities=self.get_capabilities())) + + def disable_history_streaming(self) -> None: + """Disable history streaming in this connection.""" + self.is_history_streaming_enabled = False + if self._history_streamer: + self._history_streamer.stop(success=False) + self.log.info('websocket history streaming disabled') + + def get_short_remote(self) -> str: + """Get remote for logging.""" + assert self.transport is not None + return format_address(self.transport.getPeer()) + def onConnect(self, request): - self.log.info('connection opened, starting handshake...', request=request) + """Called by the websocket protocol when the connection is opened but it is still pending handshaking.""" + self.log = logger.new(remote=self.get_short_remote()) + self.log.info('websocket connection opened, starting handshake...') def onOpen(self) -> None: + """Called by the websocket protocol when the connection is established.""" self.factory.on_client_open(self) - self.log.info('connection established') + self.log.info('websocket connection established') + self.send_capabilities() def onClose(self, wasClean, code, reason): + """Called by the websocket protocol when the connection is closed.""" self.factory.on_client_close(self) - self.log.info('connection closed', reason=reason) + self.log.info('websocket connection closed', reason=reason) def onMessage(self, payload: Union[bytes, str], isBinary: bool) -> None: + """Called by the websocket protocol when a new message is received.""" self.log.debug('new message', payload=payload.hex() if isinstance(payload, bytes) else payload) - self.factory.handle_message(self, payload) + if isinstance(payload, bytes): + message = json_loadb(payload) + else: + message = json_loads(payload) + + _type = message.get('type') + + if _type == 'ping': + self._handle_ping(message) + elif _type == 'subscribe_address': + self.factory._handle_subscribe_address(self, message) + elif _type == 'unsubscribe_address': + self.factory._handle_unsubscribe_address(self, message) + elif _type == 'request:history:xpub': + self._open_history_xpub_streamer(message) + elif _type == 'request:history:manual': + self._handle_history_manual_streamer(message) + elif _type == 'request:history:stop': + self._stop_streamer(message) + + def _handle_ping(self, message: dict[Any, Any]) -> None: + """Handle ping message, should respond with a simple {"type": "pong"}""" + payload = json_dumpb({'type': 'pong'}) + self.sendMessage(payload, False) + + def fail_if_history_streaming_is_disabled(self) -> bool: + """Return false if the history streamer is enabled. Otherwise, it sends an + error message and returns true.""" + if self.is_history_streaming_enabled: + return False + + self.send_message(StreamErrorMessage( + id='', + errmsg='Streaming history is disabled.' + )) + return True + + def _create_streamer(self, stream_id: str, search: AddressSearch) -> None: + """Create the streamer and handle its callbacks.""" + self._history_streamer = HistoryStreamer(protocol=self, stream_id=stream_id, search=search) + deferred = self._history_streamer.start() + deferred.addBoth(self._streamer_callback) + return + + def _open_history_xpub_streamer(self, message: dict[Any, Any]) -> None: + """Handle request to stream transactions using an xpub.""" + if self.fail_if_history_streaming_is_disabled(): + return + + stream_id = message['id'] + + if self._history_streamer is not None: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='Streaming is already opened.' + )) + return + + xpub = message['xpub'] + gap_limit = message.get('gap-limit', 20) + first_index = message.get('first-index', 0) + if gap_limit > self.MAX_GAP_LIMIT: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg=f'GAP limit is too big. Maximum: {self.MAX_GAP_LIMIT}' + )) + return + + try: + address_iter = aiter_xpub_addresses(xpub, first_index=first_index) + except InvalidXPub: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg=f'Invalid XPub: {xpub}' + )) + return + + search = gap_limit_search(self.factory.manager, address_iter, gap_limit) + self._create_streamer(stream_id, search) + self.log.info('opening a websocket xpub streaming', + stream_id=stream_id, + xpub=xpub, + gap_limit=gap_limit, + first_index=first_index) + + def _handle_history_manual_streamer(self, message: dict[Any, Any]) -> None: + """Handle request to stream transactions using a list of addresses.""" + if self.fail_if_history_streaming_is_disabled(): + return + + stream_id = message['id'] + addresses: list[AddressItem] = [AddressItem(idx, address) for idx, address in message.get('addresses', [])] + first = message.get('first', False) + last = message.get('last', False) + + if self._history_streamer is not None: + if first or self._history_streamer.stream_id != stream_id: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='Streaming is already opened.' + )) + return + + if not self._add_addresses_to_manual_iter(stream_id, addresses, last): + return + + self.log.info('Adding addresses to a websocket manual streaming', + stream_id=stream_id, + addresses=addresses, + last=last) + return + + gap_limit = message.get('gap-limit', 20) + if gap_limit > self.MAX_GAP_LIMIT: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg=f'GAP limit is too big. Maximum: {self.MAX_GAP_LIMIT}' + )) + return + + if not first: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='Streaming not found. You must send first=true in your first message.' + )) + return + + address_iter = ManualAddressSequencer() + self._manual_address_iter = address_iter + if not self._add_addresses_to_manual_iter(stream_id, addresses, last): + self._manual_address_iter = None + return + + search = gap_limit_search(self.factory.manager, address_iter, gap_limit) + self._create_streamer(stream_id, search) + self.log.info('opening a websocket manual streaming', + stream_id=stream_id, + addresses=addresses, + gap_limit=gap_limit, + last=last) + + def _streamer_callback(self, success: bool) -> None: + """Callback used to identify when the streamer has ended.""" + assert self._history_streamer is not None + self.log.info('websocket xpub streaming has been finished', + stream_id=self._history_streamer.stream_id, + success=success, + sent_addresses=self._history_streamer.stats_sent_addresses, + sent_vertices=self._history_streamer.stats_sent_vertices) + self._history_streamer = None + self._manual_address_iter = None + + def _stop_streamer(self, message: dict[Any, Any]) -> None: + """Handle request to stop the current streamer.""" + stream_id: str = message.get('id', '') + + if self._history_streamer is None: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='No streaming opened.' + )) + return + + assert self._history_streamer is not None + + if self._history_streamer.stream_id != stream_id: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='Current stream has a different id.' + )) + return + + self._history_streamer.stop(success=False) + self.log.info('stopping a websocket xpub streaming', stream_id=stream_id) + + def send_message(self, message: WebSocketMessage) -> None: + """Send a typed message.""" + payload = message.json_dumpb() + self.sendMessage(payload) + + def subscribe_address(self, address: str) -> tuple[bool, str]: + """Subscribe to receive real-time messages for all vertices related to an address.""" + return self.factory.subscribe_address(self, address) + + def _add_addresses_to_manual_iter(self, stream_id: str, addresses: list[AddressItem], last: bool) -> bool: + """Add addresses to manual address iter and returns true if it succeeds.""" + assert self._manual_address_iter is not None + try: + self._manual_address_iter.add_addresses(addresses, last) + except LimitExceeded: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg='List of addresses is too long.' + )) + return False + except InvalidAddress as exc: + self.send_message(StreamErrorMessage( + id=stream_id, + errmsg=f'Invalid address: {exc}' + )) + return False + + self.log.info('Adding addresses to a websocket manual streaming', + stream_id=stream_id, + addresses=addresses, + last=last) + return True diff --git a/hathor/websocket/streamer.py b/hathor/websocket/streamer.py new file mode 100644 index 000000000..f116fc36e --- /dev/null +++ b/hathor/websocket/streamer.py @@ -0,0 +1,185 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IPushProducer +from twisted.internet.task import deferLater +from zope.interface import implementer + +from hathor.websocket.iterators import AddressItem, AddressSearch, VertexItem +from hathor.websocket.messages import ( + StreamAddressMessage, + StreamBase, + StreamBeginMessage, + StreamEndMessage, + StreamErrorMessage, + StreamVertexMessage, +) + +if TYPE_CHECKING: + from hathor.websocket.protocol import HathorAdminWebsocketProtocol + + +@implementer(IPushProducer) +class HistoryStreamer: + """A producer that pushes addresses and transactions to a websocket connection. + Each pushed address is automatically subscribed for real-time updates. + + Streaming messages: + + 1. `stream:history:begin`: mark the beginning of a streaming. + 2. `stream:history:address`: mark the beginning of a new address. + 3. `stream:history:vertex`: vertex information in JSON format. + 4. `stream:history:vertex`: vertex information in JSON format. + 5. `stream:history:vertex`: vertex information in JSON format. + 6. `stream:history:vertex`: vertex information in JSON format. + 7. `stream:history:address`: mark the beginning of another address, so the previous address has been finished. + 8. `stream:history:address`: mark the beginning of another address, so the previous address has been finished. + 9. `stream:history:address`: mark the beginning of another address, so the previous address has been finished. + 10. `stream:history:end`: mark the end of the streaming. + + Notice that the streaming might send two or more `address` messages in a row if there are empty addresses. + """ + + STATS_LOG_INTERVAL = 10_000 + + def __init__(self, + *, + protocol: 'HathorAdminWebsocketProtocol', + stream_id: str, + search: AddressSearch) -> None: + self.protocol = protocol + self.stream_id = stream_id + self.search_iter = aiter(search) + + self.reactor = self.protocol.factory.manager.reactor + + self.max_seconds_locking_event_loop = 1 + + self.stats_log_interval = self.STATS_LOG_INTERVAL + self.stats_total_messages: int = 0 + self.stats_sent_addresses: int = 0 + self.stats_sent_vertices: int = 0 + + self._paused = False + self._stop = False + + def start(self) -> Deferred[bool]: + """Start streaming items.""" + self.send_message(StreamBeginMessage(id=self.stream_id)) + + # The websocket connection somehow instantiates an twisted.web.http.HTTPChannel object + # which register a producer. It seems the HTTPChannel is not used anymore after switching + # to websocket but it keep registered. So we have to unregister before registering a new + # producer. + if self.protocol.transport.producer: + self.protocol.unregisterProducer() + + self.protocol.registerProducer(self, True) + self.deferred: Deferred[bool] = Deferred() + self.resumeProducing() + return self.deferred + + def stop(self, success: bool) -> None: + """Stop streaming items.""" + self._stop = True + self.protocol.unregisterProducer() + self.deferred.callback(success) + + def pauseProducing(self) -> None: + """Pause streaming. Called by twisted.""" + self._paused = True + + def stopProducing(self) -> None: + """Stop streaming. Called by twisted.""" + self._stop = True + self.stop(False) + + def resumeProducing(self) -> None: + """Resume streaming. Called by twisted.""" + self._paused = False + self._run() + + def _run(self) -> None: + """Run the streaming main loop.""" + coro = self._async_run() + Deferred.fromCoroutine(coro) + + async def _async_run(self): + """Internal method that runs the streaming main loop.""" + t0 = self.reactor.seconds() + + async for item in self.search_iter: + # The methods `pauseProducing()` and `stopProducing()` might be called during the + # call to `self.protocol.sendMessage()`. So both `_paused` and `_stop` might change + # during the loop. + if self._paused or self._stop: + break + + match item: + case AddressItem(): + subscribed, errmsg = self.protocol.subscribe_address(item.address) + + if not subscribed: + self.send_message(StreamErrorMessage( + id=self.stream_id, + errmsg=f'Address subscription failed: {errmsg}' + )) + self.stop(False) + return + + self.stats_sent_addresses += 1 + self.send_message(StreamAddressMessage( + id=self.stream_id, + index=item.index, + address=item.address, + subscribed=subscribed, + )) + + case VertexItem(): + self.stats_sent_vertices += 1 + self.send_message(StreamVertexMessage( + id=self.stream_id, + data=item.vertex.to_json_extended(), + )) + + case _: + assert False + + self.stats_total_messages += 1 + if self.stats_total_messages % self.stats_log_interval == 0: + self.protocol.log.info('websocket streaming statistics', + total_messages=self.stats_total_messages, + sent_vertices=self.stats_sent_vertices, + sent_addresses=self.stats_sent_addresses) + + dt = self.reactor.seconds() - t0 + if dt > self.max_seconds_locking_event_loop: + # Let the event loop run at least once. + await deferLater(self.reactor, 0, lambda: None) + t0 = self.reactor.seconds() + + else: + if self._stop: + # If the streamer has been stopped, there is nothing else to do. + return + self.send_message(StreamEndMessage(id=self.stream_id)) + self.stop(True) + + def send_message(self, message: StreamBase) -> None: + """Send a message to the websocket connection.""" + payload = message.json_dumpb() + self.protocol.sendMessage(payload) diff --git a/pyproject.toml b/pyproject.toml index 4ebd77226..d255c061a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ [tool.poetry] name = "hathor" -version = "0.60.1" +version = "0.62.0" description = "Hathor Network full-node" authors = ["Hathor Team "] license = "Apache-2.0" @@ -130,6 +130,7 @@ module = [ 'sortedcontainers', 'structlog_sentry', 'structlog_sentry', + 'psutil', ] ignore_missing_imports = true diff --git a/tests/cli/test_side_dag.py b/tests/cli/test_side_dag.py new file mode 100644 index 000000000..0d4caf4a5 --- /dev/null +++ b/tests/cli/test_side_dag.py @@ -0,0 +1,48 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from hathor.cli.side_dag import _partition_argv + + +@pytest.mark.parametrize( + ['argv', 'expected_hathor_node_argv', 'expected_side_dag_argv'], + [ + ( + ['--testnet', '--side-dag-testnet'], + ['--testnet'], + ['--testnet'], + ), + ( + ['--testnet', '--some-config', 'config', '--side-dag-some-other-config', 'other-config'], + ['--testnet', '--some-config', 'config'], + ['--some-other-config', 'other-config'], + ), + ( + ['--side-dag-A', 'A', '--side-dag-B', '--B', 'B', '--side-dag-C', 'C'], + ['--B', 'B'], + ['--A', 'A', '--B', '--C', 'C'], + ), + ] +) +def test_partition_argv( + argv: list[str], + expected_hathor_node_argv: list[str], + expected_side_dag_argv: list[str] +) -> None: + hathor_node_argv, side_dag_argv = _partition_argv(argv) + + assert hathor_node_argv == expected_hathor_node_argv + assert side_dag_argv == expected_side_dag_argv diff --git a/tests/consensus/test_consensus.py b/tests/consensus/test_consensus.py index 797b88a1f..7a4eef49d 100644 --- a/tests/consensus/test_consensus.py +++ b/tests/consensus/test_consensus.py @@ -13,7 +13,7 @@ class BaseConsensusTestCase(unittest.TestCase): def setUp(self) -> None: super().setUp() - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/event/test_event_simulation_scenarios.py b/tests/event/test_event_simulation_scenarios.py index d3189093c..a65ff3aed 100644 --- a/tests/event/test_event_simulation_scenarios.py +++ b/tests/event/test_event_simulation_scenarios.py @@ -23,6 +23,7 @@ TxInput, TxMetadata, TxOutput, + VertexIdData, ) from hathor.event.model.event_type import EventType from hathor.event.websocket.request import StartStreamRequest @@ -277,6 +278,79 @@ def test_unvoided_transaction(self) -> None: expected = _remove_timestamp(expected) assert responses == expected, f'expected: {expected}\n\nactual: {responses}' + def test_invalid_mempool(self) -> None: + stream_id = self.manager._event_manager._stream_id + assert stream_id is not None + Scenario.INVALID_MEMPOOL_TRANSACTION.simulate(self.simulator, self.manager) + self._start_stream() + + responses = self._get_success_responses() + + expected = [ + # LOAD_STATED + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=0, timestamp=1578878880.0, type=EventType.LOAD_STARTED, data=EmptyData(), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One NEW_VERTEX_ACCEPTED for each genesis (1 block and 2 txs) + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=1, timestamp=1578878880.0, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', nonce=0, timestamp=1572636343, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=100000000000, token_data=0, script='dqkU/QUFm2AGJJVDuC82h2oXxz/SJnuIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HVayMofEDh4XGsaQJeRJKhutYxYodYNop6', timelock=None))], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=2.0, first_block=None, height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=2, timestamp=1578878880.0, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', nonce=6, timestamp=1572636344, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=2.0, first_block=None, height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=3, timestamp=1578878880.0, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', nonce=2, timestamp=1572636345, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=2.0, first_block=None, height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # LOAD_FINISHED + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=4, timestamp=1578878880.0, type=EventType.LOAD_FINISHED, data=EmptyData(), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for a new block (below), and one VERTEX_METADATA_CHANGED for each genesis tx (2), adding the new block as their child # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=5, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', nonce=0, timestamp=1578878910, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f'], twins=[], accumulated_weight=2.0, score=4.0, first_block=None, height=1, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=6, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', nonce=2, timestamp=1572636345, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', 'fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', 'eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', 'f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], twins=[], accumulated_weight=2.0, score=2.0, first_block='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=7, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', nonce=6, timestamp=1572636344, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', 'fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', 'eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', 'f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], twins=[], accumulated_weight=2.0, score=2.0, first_block='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One NEW_VERTEX_ACCEPTED for a new block + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=8, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', nonce=0, timestamp=1578878910, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f'], twins=[], accumulated_weight=2.0, score=4.0, first_block=None, height=1, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED and one NEW_VERTEX_ACCEPTED for 10 new blocks + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=9, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', nonce=0, timestamp=1578878911, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUXRFxfhIYOXURHjiAlx9XPuMh7E2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HF1E8Aibb17Rha6r1cM1oCp74DRmYqP61V', timelock=None))], parents=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8'], twins=[], accumulated_weight=2.0, score=4.321928094887363, first_block=None, height=2, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=10, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', nonce=0, timestamp=1578878911, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUXRFxfhIYOXURHjiAlx9XPuMh7E2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HF1E8Aibb17Rha6r1cM1oCp74DRmYqP61V', timelock=None))], parents=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8'], twins=[], accumulated_weight=2.0, score=4.321928094887363, first_block=None, height=2, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=11, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', nonce=0, timestamp=1578878912, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUu9S/kjy3HbglEu3bA4JargdORiiIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HPeHcEFtRZvMBijqFwccicDMkN17hoNq21', timelock=None))], parents=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393'], twins=[], accumulated_weight=2.0, score=4.584962500721156, first_block=None, height=3, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=12, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', nonce=0, timestamp=1578878912, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUu9S/kjy3HbglEu3bA4JargdORiiIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HPeHcEFtRZvMBijqFwccicDMkN17hoNq21', timelock=None))], parents=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393'], twins=[], accumulated_weight=2.0, score=4.584962500721156, first_block=None, height=3, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=13, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', nonce=0, timestamp=1578878913, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUzskI6jayLvTobJDhpVZiuMu7zt+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HRNWR1HpdAiDx7va9VkNUuqqSo2MGW5iE6', timelock=None))], parents=['32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49'], twins=[], accumulated_weight=2.0, score=4.807354922057604, first_block=None, height=4, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=14, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', nonce=0, timestamp=1578878913, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUzskI6jayLvTobJDhpVZiuMu7zt+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HRNWR1HpdAiDx7va9VkNUuqqSo2MGW5iE6', timelock=None))], parents=['32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49'], twins=[], accumulated_weight=2.0, score=4.807354922057604, first_block=None, height=4, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=15, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', nonce=0, timestamp=1578878914, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkU7B7Cf/pnj2DglfhnqyiRzxNg+K2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HU3chqobPRBt8pjYXt4WahKERjV8UMCWbd', timelock=None))], parents=['896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3'], twins=[], accumulated_weight=2.0, score=5.0, first_block=None, height=5, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=16, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', nonce=0, timestamp=1578878914, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkU7B7Cf/pnj2DglfhnqyiRzxNg+K2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HU3chqobPRBt8pjYXt4WahKERjV8UMCWbd', timelock=None))], parents=['896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3'], twins=[], accumulated_weight=2.0, score=5.0, first_block=None, height=5, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=17, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', nonce=0, timestamp=1578878915, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUZmTJ0of2Ce9iuycIVpFCVU08WmKIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HFrY3outhFVXGLEvaVKVFkd2nB1ihumXCr', timelock=None))], parents=['0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e'], twins=[], accumulated_weight=2.0, score=5.169925001442312, first_block=None, height=6, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=18, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', nonce=0, timestamp=1578878915, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUZmTJ0of2Ce9iuycIVpFCVU08WmKIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HFrY3outhFVXGLEvaVKVFkd2nB1ihumXCr', timelock=None))], parents=['0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e'], twins=[], accumulated_weight=2.0, score=5.169925001442312, first_block=None, height=6, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=19, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', nonce=0, timestamp=1578878916, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPNN8M/qangqd2wYSzu0u+3OmwDmIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC4kH6pnYBofzTSFWRpA71Po7geNURh5p2', timelock=None))], parents=['97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d'], twins=[], accumulated_weight=2.0, score=5.321928094887363, first_block=None, height=7, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=20, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', nonce=0, timestamp=1578878916, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPNN8M/qangqd2wYSzu0u+3OmwDmIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC4kH6pnYBofzTSFWRpA71Po7geNURh5p2', timelock=None))], parents=['97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d'], twins=[], accumulated_weight=2.0, score=5.321928094887363, first_block=None, height=7, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=21, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', nonce=0, timestamp=1578878917, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUxbNqvpWbgNtk9km/VuYhzHHMp76IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HQYUSF8ytNmm92GYMCS8XPYkt3JeKkBDyj', timelock=None))], parents=['6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6'], twins=[], accumulated_weight=2.0, score=5.459431618637297, first_block=None, height=8, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=22, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', nonce=0, timestamp=1578878917, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUxbNqvpWbgNtk9km/VuYhzHHMp76IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HQYUSF8ytNmm92GYMCS8XPYkt3JeKkBDyj', timelock=None))], parents=['6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6'], twins=[], accumulated_weight=2.0, score=5.459431618637297, first_block=None, height=8, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=23, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', nonce=0, timestamp=1578878918, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkU48C0XcFpiaWq2gwTICyEVdvJXcCIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HTHNdEhmQeECj5brwUzHK4Sq3fFrFiEvaK', timelock=None))], parents=['fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7'], twins=[], accumulated_weight=2.0, score=5.584962500721156, first_block=None, height=9, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=24, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', nonce=0, timestamp=1578878918, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkU48C0XcFpiaWq2gwTICyEVdvJXcCIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HTHNdEhmQeECj5brwUzHK4Sq3fFrFiEvaK', timelock=None))], parents=['fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7'], twins=[], accumulated_weight=2.0, score=5.584962500721156, first_block=None, height=9, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=25, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', nonce=0, timestamp=1578878919, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUmQRjqRyxq26raJZnhnpRJsrS9n2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HLUD2fi9udkg3ysPKdGvbWDyHFWdXBY1i1', timelock=None))], parents=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], twins=[], accumulated_weight=2.0, score=5.700439718141092, first_block=None, height=10, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=26, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', nonce=0, timestamp=1578878919, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUmQRjqRyxq26raJZnhnpRJsrS9n2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HLUD2fi9udkg3ysPKdGvbWDyHFWdXBY1i1', timelock=None))], parents=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], twins=[], accumulated_weight=2.0, score=5.700439718141092, first_block=None, height=10, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=27, timestamp=1578878910.25, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', nonce=0, timestamp=1578878920, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUYFHjcujZZHs0JWZkriEbn5jTv/aIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HFJRMUG7GTjdqG5f6e5tqnrnquBMFCvvs2', timelock=None))], parents=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=5.807354922057604, first_block=None, height=11, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=28, timestamp=1578878910.25, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', nonce=0, timestamp=1578878920, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUYFHjcujZZHs0JWZkriEbn5jTv/aIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HFJRMUG7GTjdqG5f6e5tqnrnquBMFCvvs2', timelock=None))], parents=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=5.807354922057604, first_block=None, height=11, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for a new tx (below), and one VERTEX_METADATA_CHANGED for a block, adding the new tx as spending their output # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=29, timestamp=1578878970.5, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650', nonce=0, timestamp=1578878970, signal_bits=0, version=1, weight=18.656776158409354, inputs=[TxInput(tx_id='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', index=0, spent_output=TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None)))], outputs=[TxOutput(value=5400, token_data=0, script='dqkUutgaVG8W5OnzgAEVUqB4XgmDgm2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HPZ4x7a2NXdrMa5ksPfeGMZmjhJHTjDZ9Q', timelock=None)), TxOutput(value=1000, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=18.656776158409354, score=0.0, first_block=None, height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=30, timestamp=1578878970.5, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', nonce=0, timestamp=1578878910, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', spent_outputs=[SpentOutput(index=0, tx_ids=['5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650'])], conflict_with=[], voided_by=[], received_by=[], children=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f'], twins=[], accumulated_weight=2.0, score=4.0, first_block=None, height=1, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One NEW_VERTEX_ACCEPTED for a new tx + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=31, timestamp=1578878970.5, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650', nonce=0, timestamp=1578878970, signal_bits=0, version=1, weight=18.656776158409354, inputs=[TxInput(tx_id='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', index=0, spent_output=TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None)))], outputs=[TxOutput(value=5400, token_data=0, script='dqkUutgaVG8W5OnzgAEVUqB4XgmDgm2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HPZ4x7a2NXdrMa5ksPfeGMZmjhJHTjDZ9Q', timelock=None)), TxOutput(value=1000, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=18.656776158409354, score=0.0, first_block=None, height=0, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # REORG_STARTED caused by a block with lower height but higher weight (below) + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=32, timestamp=0, type=EventType.REORG_STARTED, data=ReorgData(reorg_size=2, previous_best_block='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', new_best_block='2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94', common_block='eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6'), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for each block that was voided by the reorg + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=33, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', nonce=0, timestamp=1578878920, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUYFHjcujZZHs0JWZkriEbn5jTv/aIrA==', decoded=DecodedTxOutput(type='P2PKH', address='HFJRMUG7GTjdqG5f6e5tqnrnquBMFCvvs2', timelock=None))], parents=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', spent_outputs=[], conflict_with=[], voided_by=['f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], received_by=[], children=[], twins=[], accumulated_weight=2.0, score=5.807354922057604, first_block=None, height=11, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=34, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', nonce=0, timestamp=1578878919, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUmQRjqRyxq26raJZnhnpRJsrS9n2IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HLUD2fi9udkg3ysPKdGvbWDyHFWdXBY1i1', timelock=None))], parents=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', spent_outputs=[], conflict_with=[], voided_by=['1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7'], received_by=[], children=['f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb'], twins=[], accumulated_weight=2.0, score=5.700439718141092, first_block=None, height=10, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for the new block + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=35, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94', nonce=0, timestamp=1578879030, signal_bits=0, version=0, weight=10.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='', decoded=None)], parents=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=10.0, score=10.066089190457772, first_block=None, height=10, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for the block that had its output unspent, since the previous tx was removed + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=36, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', nonce=0, timestamp=1578878910, signal_bits=0, version=0, weight=2.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='dqkUPXOcGnrN0ZB2WrnPVcjdCCcacL+IrA==', decoded=DecodedTxOutput(type='P2PKH', address='HC846khX278aM1utqAgPzkKAxBTfftaRDm', timelock=None))], parents=['339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', spent_outputs=[SpentOutput(index=0, tx_ids=[])], conflict_with=[], voided_by=[], received_by=[], children=['8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f'], twins=[], accumulated_weight=2.0, score=4.0, first_block=None, height=1, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_METADATA_CHANGED for each parent of the tx that was removed + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=37, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', nonce=2, timestamp=1572636345, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', 'fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', 'eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', 'f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', '2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94'], twins=[], accumulated_weight=2.0, score=2.0, first_block='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', height=0, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=38, timestamp=0, type=EventType.VERTEX_METADATA_CHANGED, data=TxData(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', nonce=6, timestamp=1572636344, signal_bits=0, version=1, weight=2.0, inputs=[], outputs=[], parents=[], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=['9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', '8ab45f3b35f8dc437fb4a246d9b7dd3d3d5cfb7270e516076718a7a94598cf2f', '32fea29451e575e9e001f55878f4df61a2f6cf0212c4b9cbfb8125691d5377a8', '896593a8103553e6f54c46901f8c14e62618efe7f18c5afd48cf26e96db9e393', '0b71c21b8000f05241283a848b99e38f27a94a188def7ef1b93f8b0828caba49', '97b711632054189cbeb1ef4707b7d48c84e6af9a0395a4484030fb3202e691e3', '6b5e6201d81381a49fa7febe15f46d440360d8e7b1a0ddbe42e59889f32af56e', 'fdc65dbd3675a01a39343dd0c4a05eea471c3bd7015bb96cea0bde7143e24c5d', 'eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '1eb8f2c848828831c0e50f13b6ea54cac99494031ebad0318c7b142acb5540b7', 'f349fc0f570a636a440ed3853cc533faa2c4616160e1d9eb6f5d656a90da30fb', '2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94'], twins=[], accumulated_weight=2.0, score=2.0, first_block='9b83e5dbc7145a5a161c34da4bec4e1a64dc02a3f2495a2db78457426c9ee6bf', height=0, validation='full'), aux_pow=None), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One VERTEX_REMOVED for the tx above + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=39, timestamp=0, type=EventType.VERTEX_REMOVED, data=VertexIdData(vertex_id='5453759e15a6413a06390868cbb56509704c6f3f7d25f443556d8d6b2dacc650'), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # REORG_FINISHED + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=40, timestamp=0, type=EventType.REORG_FINISHED, data=EmptyData(), group_id=0), latest_event_id=41, stream_id=stream_id), # noqa: E501 + # One NEW_VERTEX_ACCEPTED for the block that caused the reorg + EventResponse(type='EVENT', peer_id=self.peer_id, network='unittests', event=BaseEvent(id=41, timestamp=0, type=EventType.NEW_VERTEX_ACCEPTED, data=TxData(hash='2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94', nonce=0, timestamp=1578879030, signal_bits=0, version=0, weight=10.0, inputs=[], outputs=[TxOutput(value=6400, token_data=0, script='', decoded=None)], parents=['eb3c4684dfad95a5b9d1c88f3463b91fe44bbe7b00e4b810648ca9e9ff5685a6', '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952', '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869'], tokens=[], token_name=None, token_symbol=None, metadata=TxMetadata(hash='2e3122412eb129c7f0d03e37d8a5637da9354df980a2259332b2b14e7a340d94', spent_outputs=[], conflict_with=[], voided_by=[], received_by=[], children=[], twins=[], accumulated_weight=10.0, score=10.066089190457772, first_block=None, height=10, validation='full'), aux_pow=None), group_id=None), latest_event_id=41, stream_id=stream_id) # noqa: E501 + ] + + responses = _remove_timestamp(responses) + expected = _remove_timestamp(expected) + assert responses == expected, f'expected: {expected}\n\nactual: {responses}' + def _start_stream(self) -> None: start_stream = StartStreamRequest(type='START_STREAM', window_size=1_000_000, last_ack_event_id=None) self._send_request(start_stream) diff --git a/tests/others/test_hathor_settings.py b/tests/others/test_hathor_settings.py index 3994e2a42..ba8a258e6 100644 --- a/tests/others/test_hathor_settings.py +++ b/tests/others/test_hathor_settings.py @@ -13,6 +13,8 @@ # limitations under the License. from pathlib import Path +from typing import Any +from unittest.mock import Mock, patch import pytest from pydantic import ValidationError @@ -26,7 +28,7 @@ ) from hathor.conf.mainnet import SETTINGS as MAINNET_SETTINGS from hathor.conf.nano_testnet import SETTINGS as NANO_TESTNET_SETTINGS -from hathor.conf.settings import HathorSettings +from hathor.conf.settings import DECIMAL_PLACES, GENESIS_TOKEN_UNITS, GENESIS_TOKENS, HathorSettings from hathor.conf.testnet import SETTINGS as TESTNET_SETTINGS from hathor.conf.unittests import SETTINGS as UNITTESTS_SETTINGS @@ -107,6 +109,134 @@ def test_missing_hathor_settings_from_yaml(filepath): assert "missing 1 required positional argument: 'NETWORK_NAME'" in str(e.value) +def test_tokens() -> None: + yaml_mock = Mock() + required_settings = dict(P2PKH_VERSION_BYTE='x01', MULTISIG_VERSION_BYTE='x02', NETWORK_NAME='test') + + def mock_settings(settings_: dict[str, Any]) -> None: + yaml_mock.dict_from_extended_yaml = Mock(return_value=required_settings | settings_) + + with patch('hathor.conf.settings.yaml', yaml_mock): + # Test default values passes + mock_settings(dict( + GENESIS_TOKENS=GENESIS_TOKENS, + GENESIS_TOKEN_UNITS=GENESIS_TOKEN_UNITS, + DECIMAL_PLACES=DECIMAL_PLACES, + )) + HathorSettings.from_yaml(filepath='some_path') + + # Test failures + mock_settings(dict( + GENESIS_TOKENS=GENESIS_TOKENS + 1, + GENESIS_TOKEN_UNITS=GENESIS_TOKEN_UNITS, + DECIMAL_PLACES=DECIMAL_PLACES, + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert ( + 'invalid tokens: GENESIS_TOKENS=100000000001, GENESIS_TOKEN_UNITS=1000000000, DECIMAL_PLACES=2' + ) in str(e.value) + + mock_settings(dict( + GENESIS_TOKENS=GENESIS_TOKENS, + GENESIS_TOKEN_UNITS=GENESIS_TOKEN_UNITS + 1, + DECIMAL_PLACES=DECIMAL_PLACES, + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert ( + 'invalid tokens: GENESIS_TOKENS=100000000000, GENESIS_TOKEN_UNITS=1000000001, DECIMAL_PLACES=2' + ) in str(e.value) + + mock_settings(dict( + GENESIS_TOKENS=GENESIS_TOKENS, + GENESIS_TOKEN_UNITS=GENESIS_TOKEN_UNITS, + DECIMAL_PLACES=DECIMAL_PLACES + 1, + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert ( + 'invalid tokens: GENESIS_TOKENS=100000000000, GENESIS_TOKEN_UNITS=1000000000, DECIMAL_PLACES=3' + ) in str(e.value) + + +def test_consensus_algorithm() -> None: + yaml_mock = Mock() + required_settings = dict(P2PKH_VERSION_BYTE='x01', MULTISIG_VERSION_BYTE='x02', NETWORK_NAME='test') + + def mock_settings(settings_: dict[str, Any]) -> None: + yaml_mock.dict_from_extended_yaml = Mock(return_value=required_settings | settings_) + + with patch('hathor.conf.settings.yaml', yaml_mock): + # Test passes when PoA is disabled with default settings + mock_settings(dict()) + HathorSettings.from_yaml(filepath='some_path') + + # Test fails when PoA is enabled with default settings + mock_settings(dict( + CONSENSUS_ALGORITHM=dict( + type='PROOF_OF_AUTHORITY', + signers=(dict(public_key=b'some_signer'),) + ) + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert 'PoA networks do not support block rewards' in str(e.value) + + # Test passes when PoA is enabled without block rewards + mock_settings(dict( + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=dict(type='PROOF_OF_AUTHORITY', signers=(dict(public_key=b'some_signer'),)), + )) + HathorSettings.from_yaml(filepath='some_path') + + # Test fails when no signer is provided + mock_settings(dict( + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=dict(type='PROOF_OF_AUTHORITY', signers=()), + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert 'At least one signer must be provided in PoA networks' in str(e.value) + + # Test fails when PoA is enabled with BLOCKS_PER_HALVING + mock_settings(dict( + BLOCKS_PER_HALVING=123, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=dict(type='PROOF_OF_AUTHORITY', signers=(dict(public_key=b'some_signer'),)), + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert 'PoA networks do not support block rewards' in str(e.value) + + # Test fails when PoA is enabled with INITIAL_TOKEN_UNITS_PER_BLOCK + mock_settings(dict( + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=123, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=dict(type='PROOF_OF_AUTHORITY', signers=(dict(public_key=b'some_signer'),)), + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert 'PoA networks do not support block rewards' in str(e.value) + + # Test fails when PoA is enabled with MINIMUM_TOKEN_UNITS_PER_BLOCK + mock_settings(dict( + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=123, + CONSENSUS_ALGORITHM=dict(type='PROOF_OF_AUTHORITY', signers=(dict(public_key=b'some_signer'),)), + )) + with pytest.raises(ValidationError) as e: + HathorSettings.from_yaml(filepath='some_path') + assert 'PoA networks do not support block rewards' in str(e.value) + + # TODO: Tests below are temporary while settings via python coexist with settings via yaml, just to make sure # the conversion was made correctly. After python settings are removed, this file can be removed too. diff --git a/tests/others/test_init_manager.py b/tests/others/test_init_manager.py index eaa96e454..a4211f136 100644 --- a/tests/others/test_init_manager.py +++ b/tests/others/test_init_manager.py @@ -30,7 +30,7 @@ def _get_all_transactions(self) -> Iterator[BaseTransaction]: class SimpleManagerInitializationTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = ModifiedTransactionMemoryStorage() + self.tx_storage = ModifiedTransactionMemoryStorage(settings=self._settings) self.pubsub = PubSubManager(self.clock) def test_invalid_arguments(self): @@ -87,7 +87,7 @@ class BaseManagerInitializationTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = ModifiedTransactionMemoryStorage() + self.tx_storage = ModifiedTransactionMemoryStorage(settings=self._settings) self.network = 'testnet' self.manager = self.create_peer(self.network, tx_storage=self.tx_storage) diff --git a/tests/others/test_metrics.py b/tests/others/test_metrics.py index f799fc961..2d21d0c57 100644 --- a/tests/others/test_metrics.py +++ b/tests/others/test_metrics.py @@ -3,6 +3,7 @@ import pytest +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.manager import PeerConnectionsMetrics from hathor.p2p.peer_id import PeerId from hathor.p2p.protocol import HathorProtocol @@ -53,7 +54,7 @@ def test_connections_manager_integration(self): to update the Metrics class with info from ConnectionsManager class """ # Preparation - tx_storage = TransactionMemoryStorage() + tx_storage = TransactionMemoryStorage(settings=self._settings) tmpdir = tempfile.mkdtemp() self.tmpdirs.append(tmpdir) wallet = Wallet(directory=tmpdir) @@ -65,7 +66,7 @@ def test_connections_manager_integration(self): manager.connections.handshaking_peers.update({Mock()}) # Execution - endpoint = 'tcp://127.0.0.1:8005' + endpoint = Entrypoint.parse('tcp://127.0.0.1:8005') # This will trigger sending to the pubsub one of the network events manager.connections.connect_to(endpoint, use_ssl=True) @@ -191,7 +192,7 @@ def test_tx_storage_data_collection_with_memory_storage(self): The expected result is that nothing is done, because we currently only collect data for RocksDB storage """ - tx_storage = TransactionMemoryStorage() + tx_storage = TransactionMemoryStorage(settings=self._settings) # All manager = self.create_peer('testnet', tx_storage=tx_storage) @@ -217,7 +218,8 @@ def build_hathor_protocol(): my_peer=my_peer, p2p_manager=manager.connections, use_ssl=False, - inbound=False + inbound=False, + settings=self._settings ) protocol.peer = PeerId() @@ -260,8 +262,8 @@ def test_cache_data_collection(self): TransactionCacheStorage """ # Preparation - base_storage = TransactionMemoryStorage() - tx_storage = TransactionCacheStorage(base_storage, self.clock, indexes=None) + base_storage = TransactionMemoryStorage(settings=self._settings) + tx_storage = TransactionCacheStorage(base_storage, self.clock, indexes=None, settings=self._settings) manager = self.create_peer('testnet', tx_storage=tx_storage) diff --git a/tests/p2p/test_bootstrap.py b/tests/p2p/test_bootstrap.py new file mode 100644 index 000000000..b6a851a5c --- /dev/null +++ b/tests/p2p/test_bootstrap.py @@ -0,0 +1,92 @@ +from typing import Callable + +from twisted.internet.defer import Deferred +from twisted.names.dns import TXT, A, Record_A, Record_TXT, RRHeader +from typing_extensions import override + +from hathor.p2p.entrypoint import Entrypoint, Protocol +from hathor.p2p.manager import ConnectionsManager +from hathor.p2p.peer_discovery import DNSPeerDiscovery, PeerDiscovery +from hathor.p2p.peer_discovery.dns import LookupResult +from hathor.p2p.peer_id import PeerId +from hathor.pubsub import PubSubManager +from tests import unittest +from tests.test_memory_reactor_clock import TestMemoryReactorClock + + +class MockPeerDiscovery(PeerDiscovery): + def __init__(self, mocked_host_ports: list[tuple[str, int]]): + self.mocked_host_ports = mocked_host_ports + + @override + async def discover_and_connect(self, connect_to: Callable[[Entrypoint], None]) -> None: + for host, port in self.mocked_host_ports: + connect_to(Entrypoint(Protocol.TCP, host, port)) + + +class MockDNSPeerDiscovery(DNSPeerDiscovery): + def __init__(self, reactor: TestMemoryReactorClock, bootstrap_txt: list[tuple[str, int]], bootstrap_a: list[str]): + super().__init__(['test.example']) + self.reactor = reactor + self.mocked_lookup_a = [RRHeader(type=A, payload=Record_A(address)) for address in bootstrap_a] + txt_entries = [f'tcp://{h}:{p}'.encode() for h, p in bootstrap_txt] + self.mocked_lookup_txt = [RRHeader(type=TXT, payload=Record_TXT(*txt_entries))] + + def do_lookup_address(self, host: str) -> Deferred[LookupResult]: + deferred: Deferred[LookupResult] = Deferred() + lookup_result = [self.mocked_lookup_a, [], []] + self.reactor.callLater(0, deferred.callback, lookup_result) + return deferred + + def do_lookup_text(self, host: str) -> Deferred[LookupResult]: + deferred: Deferred[LookupResult] = Deferred() + lookup_result = [self.mocked_lookup_txt, [], []] + self.reactor.callLater(0, deferred.callback, lookup_result) + return deferred + + +class BootstrapTestCase(unittest.TestCase): + def test_mock_discovery(self) -> None: + pubsub = PubSubManager(self.clock) + connections = ConnectionsManager(self._settings, self.clock, 'testnet', PeerId(), pubsub, True, self.rng, True) + host_ports1 = [ + ('foobar', 1234), + ('127.0.0.99', 9999), + ] + host_ports2 = [ + ('baz', 456), + ('127.0.0.88', 8888), + ] + connections.add_peer_discovery(MockPeerDiscovery(host_ports1)) + connections.add_peer_discovery(MockPeerDiscovery(host_ports2)) + connections.do_discovery() + self.clock.advance(1) + connecting_entrypoints = {str(entrypoint) for entrypoint, _ in connections.connecting_peers.values()} + self.assertEqual(connecting_entrypoints, { + 'tcp://foobar:1234', + 'tcp://127.0.0.99:9999', + 'tcp://baz:456', + 'tcp://127.0.0.88:8888', + }) + + def test_dns_discovery(self) -> None: + pubsub = PubSubManager(self.clock) + connections = ConnectionsManager(self._settings, self.clock, 'testnet', PeerId(), pubsub, True, self.rng, True) + bootstrap_a = [ + '127.0.0.99', + '127.0.0.88', + ] + bootstrap_txt = [ + ('foobar', 1234), + ('baz', 456), + ] + connections.add_peer_discovery(MockDNSPeerDiscovery(self.clock, bootstrap_txt, bootstrap_a)) + connections.do_discovery() + self.clock.advance(1) + connecting_entrypoints = {str(entrypoint) for entrypoint, _ in connections.connecting_peers.values()} + self.assertEqual(connecting_entrypoints, { + 'tcp://127.0.0.99:40403', + 'tcp://127.0.0.88:40403', + 'tcp://foobar:1234', + 'tcp://baz:456', + }) diff --git a/tests/p2p/test_connections.py b/tests/p2p/test_connections.py index c75abea7e..a9e33b79f 100644 --- a/tests/p2p/test_connections.py +++ b/tests/p2p/test_connections.py @@ -2,6 +2,7 @@ import pytest +from hathor.p2p.entrypoint import Entrypoint from tests import unittest from tests.utils import run_server @@ -20,7 +21,7 @@ def test_connections(self) -> None: def test_manager_connections(self) -> None: manager = self.create_peer('testnet', enable_sync_v1=True, enable_sync_v2=False) - endpoint = 'tcp://127.0.0.1:8005' + endpoint = Entrypoint.parse('tcp://127.0.0.1:8005') manager.connections.connect_to(endpoint, use_ssl=True) self.assertIn(endpoint, manager.connections.iter_not_ready_endpoints()) diff --git a/tests/p2p/test_peer_id.py b/tests/p2p/test_peer_id.py index bccb9bcb2..aec32921a 100644 --- a/tests/p2p/test_peer_id.py +++ b/tests/p2p/test_peer_id.py @@ -6,6 +6,7 @@ from twisted.internet.interfaces import ITransport +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.peer_id import InvalidPeerIdException, PeerId from hathor.p2p.peer_storage import PeerStorage from hathor.util import not_none @@ -90,17 +91,21 @@ def test_merge_peer(self) -> None: self.assertEqual(peer.public_key, p1.public_key) self.assertEqual(peer.entrypoints, []) + ep1 = Entrypoint.parse('tcp://127.0.0.1:1001') + ep2 = Entrypoint.parse('tcp://127.0.0.1:1002') + ep3 = Entrypoint.parse('tcp://127.0.0.1:1003') + p3 = PeerId() - p3.entrypoints.append('1') - p3.entrypoints.append('3') + p3.entrypoints.append(ep1) + p3.entrypoints.append(ep2) p3.public_key = None p4 = PeerId() p4.public_key = None p4.private_key = None p4.id = p3.id - p4.entrypoints.append('2') - p4.entrypoints.append('3') + p4.entrypoints.append(ep2) + p4.entrypoints.append(ep3) peer_storage.add_or_merge(p4) self.assertEqual(len(peer_storage), 2) @@ -111,7 +116,7 @@ def test_merge_peer(self) -> None: peer = peer_storage[not_none(p3.id)] self.assertEqual(peer.id, p3.id) self.assertEqual(peer.private_key, p3.private_key) - self.assertEqual(peer.entrypoints, ['2', '3', '1']) + self.assertEqual(set(peer.entrypoints), {ep1, ep2, ep3}) with self.assertRaises(ValueError): peer_storage.add(p1) @@ -216,25 +221,25 @@ class BasePeerIdTest(unittest.TestCase): async def test_validate_entrypoint(self) -> None: manager = self.create_peer('testnet', unlock_wallet=False) peer_id = manager.my_peer - peer_id.entrypoints = ['tcp://127.0.0.1:40403'] + peer_id.entrypoints = [Entrypoint.parse('tcp://127.0.0.1:40403')] # we consider that we are starting the connection to the peer protocol = manager.connections.client_factory.buildProtocol('127.0.0.1') - protocol.connection_string = 'tcp://127.0.0.1:40403' + protocol.entrypoint = Entrypoint.parse('tcp://127.0.0.1:40403') result = await peer_id.validate_entrypoint(protocol) self.assertTrue(result) # if entrypoint is an URI - peer_id.entrypoints = ['uri_name'] + peer_id.entrypoints = [Entrypoint.parse('tcp://uri_name:40403')] result = await peer_id.validate_entrypoint(protocol) self.assertTrue(result) # test invalid. DNS in test mode will resolve to '127.0.0.1:40403' - protocol.connection_string = 'tcp://45.45.45.45:40403' + protocol.entrypoint = Entrypoint.parse('tcp://45.45.45.45:40403') result = await peer_id.validate_entrypoint(protocol) self.assertFalse(result) # now test when receiving the connection - i.e. the peer starts it - protocol.connection_string = None - peer_id.entrypoints = ['tcp://127.0.0.1:40403'] + protocol.entrypoint = None + peer_id.entrypoints = [Entrypoint.parse('tcp://127.0.0.1:40403')] from collections import namedtuple Peer = namedtuple('Peer', 'host') @@ -246,7 +251,7 @@ def getPeer(self) -> Peer: result = await peer_id.validate_entrypoint(protocol) self.assertTrue(result) # if entrypoint is an URI - peer_id.entrypoints = ['uri_name'] + peer_id.entrypoints = [Entrypoint.parse('tcp://uri_name:40403')] result = await peer_id.validate_entrypoint(protocol) self.assertTrue(result) diff --git a/tests/p2p/test_protocol.py b/tests/p2p/test_protocol.py index a834f9e20..317675c81 100644 --- a/tests/p2p/test_protocol.py +++ b/tests/p2p/test_protocol.py @@ -5,10 +5,11 @@ from twisted.internet.protocol import Protocol from twisted.python.failure import Failure +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.peer_id import PeerId from hathor.p2p.protocol import HathorLineReceiver, HathorProtocol from hathor.simulator import FakeConnection -from hathor.util import json_dumps +from hathor.util import json_dumps, json_loadb from tests import unittest @@ -69,6 +70,25 @@ def _check_cmd_and_value(self, result: bytes, expected: tuple[bytes, bytes]) -> def test_on_connect(self) -> None: self._check_result_only_cmd(self.conn.peek_tr1_value(), b'HELLO') + def test_peer_id_with_entrypoint(self) -> None: + entrypoint_str = 'tcp://192.168.1.1:54321' + entrypoint = Entrypoint.parse(entrypoint_str) + self.peer_id1.entrypoints.append(entrypoint) + self.peer_id2.entrypoints.append(entrypoint) + self.conn.run_one_step() # HELLO + + msg1 = self.conn.peek_tr1_value() + cmd1, val1 = msg1.split(b' ', 1) + data1 = json_loadb(val1) + self.assertEqual(cmd1, b'PEER-ID') + self.assertEqual(data1['entrypoints'], [entrypoint_str]) + + msg2 = self.conn.peek_tr2_value() + cmd2, val2 = msg2.split(b' ', 1) + data2 = json_loadb(val2) + self.assertEqual(cmd2, b'PEER-ID') + self.assertEqual(data2['entrypoints'], [entrypoint_str]) + def test_invalid_command(self) -> None: self._send_cmd(self.conn.proto1, 'INVALID-CMD') self.conn.proto1.state.handle_error('') diff --git a/tests/p2p/test_sync.py b/tests/p2p/test_sync.py index 0b23a23e3..889715f7b 100644 --- a/tests/p2p/test_sync.py +++ b/tests/p2p/test_sync.py @@ -268,9 +268,13 @@ def test_downloader(self) -> None: downloader = conn.proto2.connections.get_sync_factory(SyncVersion.V1_1).get_downloader() - node_sync1 = NodeSyncTimestamp(conn.proto1, downloader, reactor=conn.proto1.node.reactor) + node_sync1 = NodeSyncTimestamp( + conn.proto1, downloader, reactor=conn.proto1.node.reactor, vertex_parser=self.manager1.vertex_parser + ) node_sync1.start() - node_sync2 = NodeSyncTimestamp(conn.proto2, downloader, reactor=conn.proto2.node.reactor) + node_sync2 = NodeSyncTimestamp( + conn.proto2, downloader, reactor=conn.proto2.node.reactor, vertex_parser=manager2.vertex_parser + ) node_sync2.start() self.assertTrue(isinstance(conn.proto1.state, PeerIdState)) diff --git a/tests/p2p/test_whitelist.py b/tests/p2p/test_whitelist.py index 5cbc7e4ae..db854ff63 100644 --- a/tests/p2p/test_whitelist.py +++ b/tests/p2p/test_whitelist.py @@ -12,13 +12,11 @@ from hathor.simulator import FakeConnection from tests import unittest -settings = get_global_settings() - class WhitelistTestCase(unittest.SyncV1Params, unittest.TestCase): - @patch('hathor.p2p.states.peer_id.settings', new=settings._replace(ENABLE_PEER_WHITELIST=True)) def test_sync_v11_whitelist_no_no(self) -> None: network = 'testnet' + self._settings = get_global_settings()._replace(ENABLE_PEER_WHITELIST=True) manager1 = self.create_peer(network) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V1_1}) @@ -38,9 +36,9 @@ def test_sync_v11_whitelist_no_no(self) -> None: self.assertTrue(conn.tr1.disconnecting) self.assertTrue(conn.tr2.disconnecting) - @patch('hathor.p2p.states.peer_id.settings', new=settings._replace(ENABLE_PEER_WHITELIST=True)) def test_sync_v11_whitelist_yes_no(self) -> None: network = 'testnet' + self._settings = get_global_settings()._replace(ENABLE_PEER_WHITELIST=True) manager1 = self.create_peer(network) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V1_1}) @@ -62,9 +60,9 @@ def test_sync_v11_whitelist_yes_no(self) -> None: self.assertFalse(conn.tr1.disconnecting) self.assertTrue(conn.tr2.disconnecting) - @patch('hathor.p2p.states.peer_id.settings', new=settings._replace(ENABLE_PEER_WHITELIST=True)) def test_sync_v11_whitelist_yes_yes(self) -> None: network = 'testnet' + self._settings = get_global_settings()._replace(ENABLE_PEER_WHITELIST=True) manager1 = self.create_peer(network) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V1_1}) diff --git a/tests/poa/__init__.py b/tests/poa/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/poa/test_poa.py b/tests/poa/test_poa.py new file mode 100644 index 000000000..3ab188975 --- /dev/null +++ b/tests/poa/test_poa.py @@ -0,0 +1,350 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from pydantic import ValidationError + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings, PoaSignerSettings +from hathor.consensus.poa.poa_signer import PoaSigner, PoaSignerFile +from hathor.crypto.util import get_address_b58_from_public_key, get_private_key_bytes, get_public_key_bytes_compressed +from hathor.transaction import Block, TxOutput +from hathor.transaction.exceptions import PoaValidationError +from hathor.transaction.poa import PoaBlock +from hathor.verification.poa_block_verifier import PoaBlockVerifier + + +def test_get_hashed_poa_data() -> None: + block = PoaBlock( + timestamp=123, + signal_bits=0b1010, + weight=2, + parents=[b'\xFF' * 32, b'\xFF' * 32], + data=b'some data', + signer_id=b'\xAB\xCD', + signature=b'some signature' + ) + + def clone_block() -> PoaBlock: + return PoaBlock.create_from_struct(block.get_struct()) + + # Test that each field changes the PoA data + test_block = clone_block() + test_block.nonce += 1 + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.timestamp += 1 + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.signal_bits += 1 + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.weight += 1 + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.parents.pop() + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.data = b'some other data' + assert poa.get_hashed_poa_data(test_block) != poa.get_hashed_poa_data(block) + + # Test that changing PoA fields do not change PoA data + test_block = clone_block() + test_block.signer_id = b'\x00\xFF' + assert poa.get_hashed_poa_data(test_block) == poa.get_hashed_poa_data(block) + + test_block = clone_block() + test_block.signature = b'some other signature' + assert poa.get_hashed_poa_data(test_block) == poa.get_hashed_poa_data(block) + + +def test_verify_poa() -> None: + def get_signer() -> tuple[PoaSigner, bytes]: + private_key = ec.generate_private_key(ec.SECP256K1()) + private_key_bytes = get_private_key_bytes(private_key) # type: ignore[arg-type] + public_key = private_key.public_key() + public_key_bytes = get_public_key_bytes_compressed(public_key) + address = get_address_b58_from_public_key(public_key) + file = PoaSignerFile.parse_obj(dict( + private_key_hex=private_key_bytes.hex(), + public_key_hex=public_key_bytes.hex(), + address=address + )) + return file.get_signer(), public_key_bytes + + poa_signer, public_key_bytes = get_signer() + settings = Mock(spec_set=HathorSettings) + settings.CONSENSUS_ALGORITHM = PoaSettings.construct(signers=()) + settings.AVG_TIME_BETWEEN_BLOCKS = 30 + block_verifier = PoaBlockVerifier(settings=settings) + storage = Mock() + storage.get_transaction = Mock(return_value=Block(timestamp=123)) + + block = PoaBlock( + storage=storage, + timestamp=153, + signal_bits=0b1010, + weight=poa.BLOCK_WEIGHT_IN_TURN, + parents=[b'parent1', b'parent2'], + ) + block._metadata = Mock() + block._metadata.height = 2 + + # Test no rewards + block.outputs = [TxOutput(123, b'')] + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'blocks must not have rewards in a PoA network' + block.outputs = [] + + # Test timestamp + block.timestamp = 152 + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'blocks must have at least 30 seconds between them' + block.timestamp = 153 + + # Test no signers + settings.CONSENSUS_ALGORITHM = PoaSettings.construct(signers=()) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'invalid PoA signature' + + # Test no data + settings.CONSENSUS_ALGORITHM = PoaSettings(signers=(PoaSignerSettings(public_key=public_key_bytes),)) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'invalid PoA signature' + + # Test invalid data + block.data = b'some_data' + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'invalid PoA signature' + + # Test incorrect private key + PoaSigner(ec.generate_private_key(ec.SECP256K1())).sign_block(block) + block.signer_id = poa_signer._signer_id # we set the correct signer id to test only the signature + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'invalid PoA signature' + + # Test valid signature + poa_signer.sign_block(block) + block_verifier.verify_poa(block) + + # Test some random weight fails + block.weight = 123 + poa_signer.sign_block(block) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'block weight is 123, expected 2.0' + + # For this part we use two signers, so the ordering matters + signer_and_keys: list[tuple[PoaSigner, bytes]] = [get_signer(), get_signer()] + settings.CONSENSUS_ALGORITHM = PoaSettings(signers=tuple( + [PoaSignerSettings(public_key=key_pair[1]) for key_pair in signer_and_keys] + )) + first_poa_signer, second_poa_signer = [key_pair[0] for key_pair in signer_and_keys] + + # Test valid signature with two signers, in turn + block.weight = poa.BLOCK_WEIGHT_IN_TURN + first_poa_signer.sign_block(block) + block_verifier.verify_poa(block) + + # And the other signature fails for the weight + second_poa_signer.sign_block(block) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'block weight is 2.0, expected 1.0' + + # Test valid signature with two signers, out of turn + block.weight = poa.BLOCK_WEIGHT_OUT_OF_TURN + second_poa_signer.sign_block(block) + block_verifier.verify_poa(block) + + # And the other signature fails for the weight + first_poa_signer.sign_block(block) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'block weight is 1.0, expected 2.0' + + # When we increment the height, the turn inverts + block._metadata.height += 1 + + # Test valid signature with two signers, in turn + block.weight = poa.BLOCK_WEIGHT_IN_TURN + second_poa_signer.sign_block(block) + block_verifier.verify_poa(block) + + # And the other signature fails for the weight + first_poa_signer.sign_block(block) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'block weight is 2.0, expected 1.0' + + # Test valid signature with two signers, out of turn + block.weight = poa.BLOCK_WEIGHT_OUT_OF_TURN + first_poa_signer.sign_block(block) + block_verifier.verify_poa(block) + + # And the other signature fails for the weight + second_poa_signer.sign_block(block) + with pytest.raises(PoaValidationError) as e: + block_verifier.verify_poa(block) + assert str(e.value) == 'block weight is 1.0, expected 2.0' + + +@pytest.mark.parametrize( + ['n_signers', 'height', 'signer_index', 'expected'], + [ + (1, 1, 0, 0), + (1, 2, 0, 0), + (1, 3, 0, 0), + + (2, 1, 0, 1), + (2, 2, 0, 0), + (2, 3, 0, 1), + + (2, 1, 1, 0), + (2, 2, 1, 1), + (2, 3, 1, 0), + + (5, 1, 0, 4), + (5, 2, 0, 3), + (5, 3, 0, 2), + (5, 4, 0, 1), + (5, 5, 0, 0), + ] +) +def test_get_signer_index_distance(n_signers: int, height: int, signer_index: int, expected: int) -> None: + settings = PoaSettings.construct(signers=tuple(PoaSignerSettings(public_key=b'') for _ in range(n_signers))) + + result = poa.get_signer_index_distance(settings=settings, signer_index=signer_index, height=height) + assert result == expected + + +@pytest.mark.parametrize( + ['n_signers', 'height', 'signer_index', 'expected'], + [ + (1, 0, 0, poa.BLOCK_WEIGHT_IN_TURN), + (1, 1, 0, poa.BLOCK_WEIGHT_IN_TURN), + (1, 2, 0, poa.BLOCK_WEIGHT_IN_TURN), + (1, 3, 0, poa.BLOCK_WEIGHT_IN_TURN), + + (2, 0, 0, poa.BLOCK_WEIGHT_IN_TURN), + (2, 1, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN), + (2, 2, 0, poa.BLOCK_WEIGHT_IN_TURN), + (2, 3, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN), + + (2, 0, 1, poa.BLOCK_WEIGHT_OUT_OF_TURN), + (2, 1, 1, poa.BLOCK_WEIGHT_IN_TURN), + (2, 2, 1, poa.BLOCK_WEIGHT_OUT_OF_TURN), + (2, 3, 1, poa.BLOCK_WEIGHT_IN_TURN), + + (5, 0, 0, poa.BLOCK_WEIGHT_IN_TURN), + (5, 1, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN / 4), + (5, 2, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN / 3), + (5, 3, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN / 2), + (5, 4, 0, poa.BLOCK_WEIGHT_OUT_OF_TURN / 1), + ] +) +def test_calculate_weight(n_signers: int, height: int, signer_index: int, expected: float) -> None: + settings = PoaSettings.construct(signers=tuple(PoaSignerSettings(public_key=b'') for _ in range(n_signers))) + block = Mock() + block.get_height = Mock(return_value=height) + + result = poa.calculate_weight(settings, block, signer_index) + assert result == expected + + +@pytest.mark.parametrize( + ['signers', 'heights_and_expected'], + [ + ( + (PoaSignerSettings(public_key=b'a'),), + [ + (0, [b'a']), + (10, [b'a']), + (100, [b'a']), + ], + ), + ( + (PoaSignerSettings(public_key=b'a', start_height=0, end_height=10),), + [ + (0, [b'a']), + (10, [b'a']), + (100, []), + ], + ), + ( + (PoaSignerSettings(public_key=b'a', start_height=10, end_height=None),), + [ + (0, []), + (10, [b'a']), + (100, [b'a']), + ], + ), + ( + ( + PoaSignerSettings(public_key=b'a', start_height=0, end_height=10), + PoaSignerSettings(public_key=b'b', start_height=5, end_height=20), + PoaSignerSettings(public_key=b'c', start_height=10, end_height=30), + ), + [ + (0, [b'a']), + (5, [b'a', b'b']), + (10, [b'a', b'b', b'c']), + (15, [b'b', b'c']), + (20, [b'b', b'c']), + (30, [b'c']), + (100, []), + ] + ), + ] +) +def test_get_active_signers( + signers: tuple[PoaSignerSettings, ...], + heights_and_expected: list[tuple[int, list[bytes]]], +) -> None: + settings = PoaSettings(signers=signers) + + for height, expected in heights_and_expected: + result = poa.get_active_signers(settings, height) + assert result == expected, f'height={height}' + + +def test_poa_signer_settings() -> None: + # Test passes + _ = PoaSignerSettings(public_key=b'some_key') + _ = PoaSignerSettings(public_key=b'some_key', start_height=0, end_height=10) + _ = PoaSignerSettings(public_key=b'some_key', start_height=0, end_height=None) + + # Test fails + with pytest.raises(ValidationError) as e: + _ = PoaSignerSettings(public_key=b'some_key', start_height=10, end_height=10) + assert 'end_height (10) must be greater than start_height (10)' in str(e.value) + + with pytest.raises(ValidationError) as e: + _ = PoaSignerSettings(public_key=b'some_key', start_height=10, end_height=5) + assert 'end_height (5) must be greater than start_height (10)' in str(e.value) diff --git a/tests/poa/test_poa_block_producer.py b/tests/poa/test_poa_block_producer.py new file mode 100644 index 000000000..3be849470 --- /dev/null +++ b/tests/poa/test_poa_block_producer.py @@ -0,0 +1,192 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +import pytest + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings, PoaSignerSettings +from hathor.consensus.poa import PoaBlockProducer +from hathor.crypto.util import get_public_key_bytes_compressed +from hathor.manager import HathorManager +from hathor.transaction.poa import PoaBlock +from tests.poa.utils import get_settings, get_signer +from tests.test_memory_reactor_clock import TestMemoryReactorClock +from tests.unittest import TestBuilder + + +def _get_manager(settings: HathorSettings) -> HathorManager: + reactor = TestMemoryReactorClock() + reactor.advance(settings.GENESIS_BLOCK_TIMESTAMP) + + artifacts = TestBuilder() \ + .set_settings(settings) \ + .set_reactor(reactor) \ + .build() + + return artifacts.manager + + +def test_poa_block_producer_one_signer() -> None: + signer = get_signer() + settings = get_settings(signer, time_between_blocks=10) + manager = _get_manager(settings) + reactor = manager.reactor + assert isinstance(reactor, TestMemoryReactorClock) + manager = Mock(wraps=manager) + producer = PoaBlockProducer(settings=settings, reactor=reactor, poa_signer=signer) + producer.manager = manager + producer.start() + + # at the beginning no blocks are produced + reactor.advance(60) + manager.on_new_tx.assert_not_called() + + # when we can start mining, we start producing blocks + manager.can_start_mining = Mock(return_value=True) + + # we produce our first block + reactor.advance(10) + manager.on_new_tx.assert_called_once() + block1 = manager.on_new_tx.call_args.args[0] + assert isinstance(block1, PoaBlock) + assert block1.timestamp == reactor.seconds() + assert block1.weight == poa.BLOCK_WEIGHT_IN_TURN + assert block1.outputs == [] + assert block1.get_block_parent_hash() == settings.GENESIS_BLOCK_HASH + manager.on_new_tx.reset_mock() + + # haven't produced the second block yet + reactor.advance(9) + + # we produce our second block + reactor.advance(1) + manager.on_new_tx.assert_called_once() + block2 = manager.on_new_tx.call_args.args[0] + assert isinstance(block2, PoaBlock) + assert block2.timestamp == block1.timestamp + 10 + assert block2.weight == poa.BLOCK_WEIGHT_IN_TURN + assert block2.outputs == [] + assert block2.get_block_parent_hash() == block1.hash + manager.on_new_tx.reset_mock() + + # haven't produced the third block yet + reactor.advance(9) + + # we produce our third block + reactor.advance(1) + manager.on_new_tx.assert_called_once() + block3 = manager.on_new_tx.call_args.args[0] + assert isinstance(block3, PoaBlock) + assert block3.timestamp == block2.timestamp + 10 + assert block3.weight == poa.BLOCK_WEIGHT_IN_TURN + assert block3.outputs == [] + assert block3.get_block_parent_hash() == block2.hash + manager.on_new_tx.reset_mock() + + +def test_poa_block_producer_two_signers() -> None: + signer1, signer2 = get_signer(), get_signer() + settings = get_settings(signer1, signer2, time_between_blocks=10) + manager = _get_manager(settings) + reactor = manager.reactor + assert isinstance(reactor, TestMemoryReactorClock) + manager = Mock(wraps=manager) + producer = PoaBlockProducer(settings=settings, reactor=reactor, poa_signer=signer1) + producer.manager = manager + producer.start() + + # at the beginning no blocks are produced + reactor.advance(60) + manager.on_new_tx.assert_not_called() + + # when we can start mining, we start producing blocks + manager.can_start_mining = Mock(return_value=True) + + # we produce our first block + reactor.advance(10) + manager.on_new_tx.assert_called_once() + block1 = manager.on_new_tx.call_args.args[0] + assert isinstance(block1, PoaBlock) + assert block1.timestamp == reactor.seconds() + assert block1.weight == poa.BLOCK_WEIGHT_OUT_OF_TURN + assert block1.outputs == [] + assert block1.get_block_parent_hash() == settings.GENESIS_BLOCK_HASH + manager.on_new_tx.reset_mock() + + # haven't produced the second block yet + reactor.advance(9) + + # we produce our second block + reactor.advance(1) + manager.on_new_tx.assert_called_once() + block2 = manager.on_new_tx.call_args.args[0] + assert isinstance(block2, PoaBlock) + assert block2.timestamp == block1.timestamp + 10 + assert block2.weight == poa.BLOCK_WEIGHT_IN_TURN + assert block2.outputs == [] + assert block2.get_block_parent_hash() == block1.hash + manager.on_new_tx.reset_mock() + + # haven't produced the third block yet + reactor.advance(29) + + # we produce our third block + reactor.advance(1) + manager.on_new_tx.assert_called_once() + block3 = manager.on_new_tx.call_args.args[0] + assert isinstance(block3, PoaBlock) + assert block3.timestamp == block2.timestamp + 30 + assert block3.weight == poa.BLOCK_WEIGHT_OUT_OF_TURN + assert block3.outputs == [] + assert block3.get_block_parent_hash() == block2.hash + manager.on_new_tx.reset_mock() + + +@pytest.mark.parametrize( + ['previous_height', 'signer_index', 'expected_delay'], + [ + (0, 0, 90), + (0, 1, 30), + (0, 2, 70), + (0, 3, 80), + + (1, 0, 80), + (1, 1, 90), + (1, 2, 30), + (1, 3, 70), + ] +) +def test_expected_block_timestamp(previous_height: int, signer_index: int, expected_delay: int) -> None: + signers = [get_signer(), get_signer(), get_signer(), get_signer()] + keys_and_signers = [ + (get_public_key_bytes_compressed(signer.get_public_key()), signer) + for signer in signers + ] + signer = keys_and_signers[signer_index][1] + settings = Mock() + settings.CONSENSUS_ALGORITHM = PoaSettings(signers=tuple( + [PoaSignerSettings(public_key=key_and_signer[0]) for key_and_signer in keys_and_signers] + )) + settings.AVG_TIME_BETWEEN_BLOCKS = 30 + producer = PoaBlockProducer(settings=settings, reactor=Mock(), poa_signer=signer) + previous_block = Mock() + previous_block.timestamp = 100 + previous_block.get_height = Mock(return_value=previous_height) + + result = producer._expected_block_timestamp(previous_block, signer_index) + + assert result == previous_block.timestamp + expected_delay diff --git a/tests/poa/test_poa_simulation.py b/tests/poa/test_poa_simulation.py new file mode 100644 index 000000000..33c455426 --- /dev/null +++ b/tests/poa/test_poa_simulation.py @@ -0,0 +1,565 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +from collections import defaultdict +from typing import Iterator + +import base58 +import pytest +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from twisted.python.failure import Failure + +from hathor.conf.settings import HathorSettings +from hathor.consensus import poa +from hathor.consensus.consensus_settings import PoaSettings, PoaSignerSettings +from hathor.consensus.poa import PoaSigner +from hathor.crypto.util import get_address_b58_from_public_key_bytes, get_public_key_bytes_compressed +from hathor.manager import HathorManager +from hathor.simulator import FakeConnection +from hathor.transaction import BaseTransaction, Block, TxInput, TxOutput +from hathor.transaction.genesis import generate_new_genesis +from hathor.transaction.poa import PoaBlock +from hathor.transaction.scripts import P2PKH +from hathor.transaction.token_creation_tx import TokenCreationTransaction +from hathor.util import not_none +from tests import unittest +from tests.poa.utils import get_settings, get_signer +from tests.simulation.base import SimulatorTestCase +from tests.utils import HAS_ROCKSDB + + +def _get_blocks_by_height(manager: HathorManager) -> defaultdict[int, list[PoaBlock]]: + blocks_by_height: defaultdict[int, list[PoaBlock]] = defaultdict(list) + + for vertex in manager.tx_storage.get_all_transactions(): + if vertex.is_genesis or not isinstance(vertex, Block): + continue + assert isinstance(vertex, PoaBlock) + blocks_by_height[vertex.get_height()].append(vertex) + + return blocks_by_height + + +def _assert_block_in_turn(block: PoaBlock, signer: PoaSigner) -> None: + assert not block.get_metadata().voided_by + assert block.weight == poa.BLOCK_WEIGHT_IN_TURN + assert block.signer_id == signer._signer_id + + +def _assert_height_weight_signer_id( + vertices: Iterator[BaseTransaction], + expected: list[tuple[int, float, bytes]] +) -> None: + non_voided_blocks: list[tuple[int, float, bytes]] = [] + + for vertex in vertices: + meta = vertex.get_metadata() + if not isinstance(vertex, PoaBlock) or meta.voided_by: + continue + non_voided_blocks.append((vertex.get_height(), vertex.weight, vertex.signer_id)) + + assert sorted(non_voided_blocks) == expected + + +class BasePoaSimulationTest(SimulatorTestCase): + def _get_manager(self, signer: PoaSigner | None = None) -> HathorManager: + builder = self.simulator.get_default_builder().disable_full_verification() + if signer: + builder.set_poa_signer(signer) + artifacts = self.simulator.create_artifacts(builder) + return artifacts.manager + + def test_no_producers(self) -> None: + signer = get_signer() + self.simulator.settings = get_settings(signer, time_between_blocks=10) + manager1 = self._get_manager() + manager2 = self._get_manager() + + connection = FakeConnection(manager1, manager2) + self.simulator.add_connection(connection) + + # no producers, so no blocks + self.simulator.run(120) + assert manager1.tx_storage.get_block_count() == 1 + assert manager2.tx_storage.get_block_count() == 1 + + def test_different_signer_settings(self) -> None: + signer1, signer2 = get_signer(), get_signer() + self.simulator.settings = get_settings(signer1) + manager1 = self._get_manager() + self.simulator.settings = get_settings(signer2) + manager2 = self._get_manager() + + connection = FakeConnection(manager1, manager2) + self.simulator.add_connection(connection) + + connection.run_one_step() + assert b'ERROR Settings values are different' in connection.peek_tr1_value() + assert connection.tr1.disconnecting + + def test_one_producer_allowed(self) -> None: + signer = get_signer() + signer_id = signer._signer_id + self.simulator.settings = get_settings(signer, time_between_blocks=10) + manager = self._get_manager(signer) + + # manager is allowed to produce blocks, so it does + manager.allow_mining_without_peers() + self.simulator.run(90) + assert manager.tx_storage.get_block_count() == 10 + + _assert_height_weight_signer_id( + manager.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (7, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (9, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + ] + ) + + def test_one_producer_not_allowed(self) -> None: + signer = get_signer() + self.simulator.settings = get_settings(signer, time_between_blocks=10) + manager = self._get_manager(signer) + + # manager is not allowed to produce blocks, so it does not + self.simulator.run(120) + assert manager.tx_storage.get_block_count() == 1 + + def test_two_producers(self) -> None: + signer1, signer2 = get_signer(), get_signer() + signer_id1, signer_id2 = signer1._signer_id, signer2._signer_id + self.simulator.settings = get_settings(signer1, signer2, time_between_blocks=10) + manager1 = self._get_manager(signer1) + manager2 = self._get_manager(signer2) + + connection = FakeConnection(manager1, manager2) + self.simulator.add_connection(connection) + + # both managers are producing blocks + self.simulator.run(100) + assert manager1.tx_storage.get_block_count() == 12 + assert manager2.tx_storage.get_block_count() == 12 + assert manager1.tx_storage.get_best_block_tips() == manager2.tx_storage.get_best_block_tips() + + _assert_height_weight_signer_id( + manager1.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (7, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (9, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + manager1_blocks_by_height = _get_blocks_by_height(manager1) + manager2_blocks_by_height = _get_blocks_by_height(manager2) + + # both managers produce and propagate block #1 instantly + assert len(manager1_blocks_by_height[1]) == 2 + assert set(manager1_blocks_by_height[1]) == set(manager2_blocks_by_height[1]) + + # but only the block from signer2 becomes non-voided, as it is in turn + non_voided_block1 = manager1.tx_storage.get_transaction_by_height(1) + assert isinstance(non_voided_block1, PoaBlock) + _assert_block_in_turn(non_voided_block1, signer2) + + # from blocks #2 to #10, the last one, the behavior alternates + for height in range(2, 11): + blocks_manager1 = manager1_blocks_by_height[height] + blocks_manager2 = manager2_blocks_by_height[height] + + if height % 2 == 0: + # if the height is even, it's manager1's turn. + assert len(blocks_manager1) == 1 + _assert_block_in_turn(blocks_manager1[0], signer1) + else: + # if the height is odd, the opposite happens + assert len(blocks_manager2) == 1 + _assert_block_in_turn(blocks_manager2[0], signer2) + + def test_four_signers(self) -> None: + signer1, signer2, signer3, signer4 = get_signer(), get_signer(), get_signer(), get_signer() + signer_id1, signer_id2, signer_id3 = signer1._signer_id, signer2._signer_id, signer3._signer_id + self.simulator.settings = get_settings(signer1, signer2, signer3, signer4, time_between_blocks=10) + manager1 = self._get_manager(signer1) + manager2 = self._get_manager(signer2) + manager3 = self._get_manager(signer3) + + connection12 = FakeConnection(manager1, manager2) + connection13 = FakeConnection(manager1, manager3) + self.simulator.add_connection(connection12) + self.simulator.add_connection(connection13) + + # all managers are producing blocks + self.simulator.run(110) + + # manager2 and manager3 leave + manager2.stop() + manager3.stop() + self.simulator.run(160) + + # manager1 produces out of turn blocks with decreasing weights + _assert_height_weight_signer_id( + manager1.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id3), + (3, poa.BLOCK_WEIGHT_OUT_OF_TURN / 1, signer_id1), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id3), + (7, poa.BLOCK_WEIGHT_OUT_OF_TURN / 1, signer_id1), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (9, poa.BLOCK_WEIGHT_OUT_OF_TURN / 3, signer_id1), + (10, poa.BLOCK_WEIGHT_OUT_OF_TURN / 2, signer_id1), + (11, poa.BLOCK_WEIGHT_OUT_OF_TURN / 1, signer_id1), + (12, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + def test_producer_leave_and_comeback(self) -> None: + signer1, signer2 = get_signer(), get_signer() + signer_id1, signer_id2 = signer1._signer_id, signer2._signer_id + self.simulator.settings = get_settings(signer1, signer2, time_between_blocks=10) + + # here we create a situation with an intermittent producer, testing that the other producer produces blocks + # out of turn + manager1 = self._get_manager(signer1) + manager1.allow_mining_without_peers() + self.simulator.run(50) + + manager2 = self._get_manager(signer2) + connection = FakeConnection(manager1, manager2) + self.simulator.add_connection(connection) + self.simulator.run(80) + + manager2.stop() + connection.disconnect(Failure(Exception('testing'))) + self.simulator.remove_connection(connection) + self.simulator.run(70) + + assert not manager2.can_start_mining() + self.simulator.add_connection(connection) + connection.reconnect() + manager2.start() + self.simulator.run(30) + + assert manager1.tx_storage.get_block_count() == 19 + assert manager2.tx_storage.get_block_count() == 19 + assert manager1.tx_storage.get_best_block_tips() == manager2.tx_storage.get_best_block_tips() + + _assert_height_weight_signer_id( + manager1.tx_storage.get_all_transactions(), + [ + # Before manager2 joins, only manager1 produces blocks + (1, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (3, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + # When manager2 joins, both of them start taking turns + (7, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (9, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (11, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (12, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + # manager2 leaves so manager1 produces all the next blocks + (13, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (14, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (15, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + # manager2 comes back again, so both of them take turns again + (16, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (17, poa.BLOCK_WEIGHT_IN_TURN, signer_id2), + (18, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + @pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') + def test_existing_storage(self) -> None: + import tempfile + rocksdb_directory = tempfile.mkdtemp() + self.tmpdirs.append(rocksdb_directory) + signer = get_signer() + signer_id = signer._signer_id + + self.simulator.settings = get_settings(signer, time_between_blocks=10) + builder = self.simulator.get_default_builder() \ + .set_poa_signer(signer) \ + .use_rocksdb(path=rocksdb_directory) + + artifacts1 = self.simulator.create_artifacts(builder) + manager1 = artifacts1.manager + manager1.allow_mining_without_peers() + + self.simulator.run(50) + assert manager1.tx_storage.get_block_count() == 6 + + _assert_height_weight_signer_id( + manager1.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + ] + ) + manager1.stop() + not_none(artifacts1.rocksdb_storage).close() + + builder = self.simulator.get_default_builder() \ + .set_poa_signer(signer) \ + .use_rocksdb(path=rocksdb_directory) + + artifacts = self.simulator.create_artifacts(builder) + manager2 = artifacts.manager + manager2.allow_mining_without_peers() + + self.simulator.run(60) + assert manager2.tx_storage.get_block_count() == 12 + + _assert_height_weight_signer_id( + manager2.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (7, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (9, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (11, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + ] + ) + + def test_new_signer_added(self) -> None: + signer1, signer2 = get_signer(), get_signer() + key1 = get_public_key_bytes_compressed(signer1.get_public_key()) + key2 = get_public_key_bytes_compressed(signer2.get_public_key()) + signer_settings1 = PoaSignerSettings(public_key=key1) + signer_settings2 = PoaSignerSettings(public_key=key2, start_height=6, end_height=13) + signer_id1 = signer1._signer_id + self.simulator.settings = get_settings(signer_settings1, time_between_blocks=10) + + builder_1a = self.simulator.get_default_builder() \ + .set_poa_signer(signer1) + artifacts_1a = self.simulator.create_artifacts(builder_1a) + storage_1a = artifacts_1a.tx_storage + manager_1a = artifacts_1a.manager + manager_1a.allow_mining_without_peers() + + self.simulator.run(50) + assert manager_1a.tx_storage.get_block_count() == 6 + + _assert_height_weight_signer_id( + manager_1a.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + # we stop the network and add a new signer to the settings + manager_1a.stop() + self.simulator.settings = get_settings(signer_settings1, signer_settings2, time_between_blocks=10) + + builder_1b = self.simulator.get_default_builder() \ + .set_tx_storage(storage_1a) \ + .set_poa_signer(signer1) \ + .disable_full_verification() + artifacts_1b = self.simulator.create_artifacts(builder_1b) + manager_1b = artifacts_1b.manager + manager_1b.allow_mining_without_peers() + + self.simulator.run(90) + assert manager_1b.tx_storage.get_block_count() == 11 + + # after we restart it, new blocks are alternating + _assert_height_weight_signer_id( + manager_1b.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (7, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (9, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + # we add a non-producer + manager_2 = self._get_manager() + + connection = FakeConnection(manager_1b, manager_2) + self.simulator.add_connection(connection) + self.simulator.run(60) + + # it should sync to the same blockchain + _assert_height_weight_signer_id( + manager_2.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (7, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (9, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (11, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (12, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (13, poa.BLOCK_WEIGHT_OUT_OF_TURN, signer_id1), + (14, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + (15, poa.BLOCK_WEIGHT_IN_TURN, signer_id1), + ] + ) + + def test_use_case(self) -> None: + """Simulate a use case that uses a PoA network to mint all native tokens into custom tokens.""" + self.simulator.stop() + private_key = ec.generate_private_key(ec.SECP256K1()) + signer = PoaSigner(private_key) + public_key = signer.get_public_key() + public_key_bytes = get_public_key_bytes_compressed(public_key) + address = get_address_b58_from_public_key_bytes(public_key_bytes) + script = P2PKH.create_output_script(base58.b58decode(address)) + signer_id = signer._signer_id + + tokens = 100_000 + block_timestamp = 1718894758 + min_block_weight = 0 + min_tx_weight = 0 + genesis_block, genesis_tx1, genesis_tx2 = generate_new_genesis( + tokens=tokens, + address=address, + block_timestamp=block_timestamp, + min_block_weight=min_block_weight, + min_tx_weight=min_tx_weight, + ) + + self.simulator.settings = HathorSettings( + P2PKH_VERSION_BYTE=b'\x49', + MULTISIG_VERSION_BYTE=b'\x87', + NETWORK_NAME='use-case-testnet', + GENESIS_BLOCK_HASH=genesis_block.hash, + GENESIS_TX1_HASH=genesis_tx1.hash, + GENESIS_TX2_HASH=genesis_tx2.hash, + GENESIS_OUTPUT_SCRIPT=script, + GENESIS_BLOCK_TIMESTAMP=block_timestamp, + GENESIS_BLOCK_NONCE=0, + GENESIS_TX1_NONCE=0, + GENESIS_TX2_NONCE=0, + DECIMAL_PLACES=0, + GENESIS_TOKENS=tokens, + GENESIS_TOKEN_UNITS=tokens, + TOKEN_DEPOSIT_PERCENTAGE=0.0000001, + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + MIN_BLOCK_WEIGHT=min_block_weight, + MIN_TX_WEIGHT_K=0, + MIN_TX_WEIGHT_COEFFICIENT=0, + MIN_TX_WEIGHT=min_tx_weight, + REWARD_SPEND_MIN_BLOCKS=1, + AVG_TIME_BETWEEN_BLOCKS=10, + CONSENSUS_ALGORITHM=PoaSettings( + signers=(PoaSignerSettings(public_key=public_key_bytes),), + ), + ) + self.simulator.start() + + manager = self._get_manager(signer) + manager.allow_mining_without_peers() + self.simulator.run(100) + assert manager.tx_storage.get_block_count() == 11 + + _assert_height_weight_signer_id( + manager.tx_storage.get_all_transactions(), + [ + (1, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (2, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (3, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (4, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (5, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (6, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (7, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (8, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (9, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + (10, poa.BLOCK_WEIGHT_IN_TURN, signer_id), + ] + ) + + token_tx = TokenCreationTransaction( + timestamp=self.simulator.settings.GENESIS_BLOCK_TIMESTAMP + 3, + weight=0, + parents=[self.simulator.settings.GENESIS_TX1_HASH, self.simulator.settings.GENESIS_TX2_HASH], + inputs=[TxInput(self.simulator.settings.GENESIS_BLOCK_HASH, 0, b'')], + outputs=[ + TxOutput(1_000_000_000_000, script, 0b00000001), + TxOutput(TxOutput.TOKEN_MINT_MASK, script, 0b10000001), + TxOutput(TxOutput.TOKEN_MELT_MASK, script, 0b10000001), + ], + token_name='custom-token', + token_symbol='CTK', + ) + + data_to_sign = token_tx.get_sighash_all() + hashed_data = hashlib.sha256(data_to_sign).digest() + signature = private_key.sign(hashed_data, ec.ECDSA(hashes.SHA256())) + token_tx.inputs[0].data = P2PKH.create_input_data(public_key_bytes, signature) + token_tx.update_hash() + + assert manager.on_new_tx(token_tx, fails_silently=False) + + +class SyncV1PoaSimulationTest(unittest.SyncV1Params, BasePoaSimulationTest): + __test__ = True + + +class SyncV2PoaSimulationTest(unittest.SyncV2Params, BasePoaSimulationTest): + __test__ = True + + +# sync-bridge should behave like sync-v2 +class SyncBridgePoaSimulationTest(unittest.SyncBridgeParams, SyncV2PoaSimulationTest): + pass diff --git a/tests/poa/test_poa_verification.py b/tests/poa/test_poa_verification.py new file mode 100644 index 000000000..88508c7fc --- /dev/null +++ b/tests/poa/test_poa_verification.py @@ -0,0 +1,287 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock, patch + +from cryptography.hazmat.primitives.asymmetric import ec + +from hathor.consensus.consensus_settings import ConsensusType, PoaSettings, PoaSignerSettings +from hathor.consensus.poa import PoaSigner +from hathor.crypto.util import get_public_key_bytes_compressed +from hathor.transaction.poa import PoaBlock +from hathor.transaction.validation_state import ValidationState +from hathor.verification.block_verifier import BlockVerifier +from hathor.verification.poa_block_verifier import PoaBlockVerifier +from hathor.verification.vertex_verifier import VertexVerifier +from tests import unittest + + +class BasePoaVerificationTest(unittest.TestCase): + __test__ = False + + def setUp(self) -> None: + super().setUp() + + self.signer = PoaSigner(ec.generate_private_key(ec.SECP256K1())) + public_key = self.signer.get_public_key() + public_key_bytes = get_public_key_bytes_compressed(public_key) + + settings = self._settings._replace( + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=PoaSettings( + type=ConsensusType.PROOF_OF_AUTHORITY, + signers=(PoaSignerSettings(public_key=public_key_bytes),), + ), + ) + + builder = self.get_builder('network').set_settings(settings) + self.manager = self.create_peer_from_builder(builder) + self.verifiers = self.manager.verification_service.verifiers + + def _get_valid_poa_block(self) -> PoaBlock: + block = PoaBlock( + hash=b'some_hash', + storage=self.manager.tx_storage, + weight=2, + outputs=[], + parents=[ + self._settings.GENESIS_BLOCK_HASH, + self._settings.GENESIS_TX1_HASH, + self._settings.GENESIS_TX2_HASH + ], + ) + self.signer.sign_block(block) + block.update_reward_lock_metadata() + return block + + def test_poa_block_verify_basic(self) -> None: + block = self._get_valid_poa_block() + + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) + verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) + verify_poa_wrapped = Mock(wraps=self.verifiers.poa_block.verify_poa) + + with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), + patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), + patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), + patch.object(PoaBlockVerifier, 'verify_poa', verify_poa_wrapped), + ): + self.manager.verification_service.verify_basic(block) + + # Vertex methods + verify_version_wrapped.assert_called_once() + + # Block methods + verify_weight_wrapped.assert_not_called() + verify_reward_wrapped.assert_called_once() + verify_poa_wrapped.assert_called_once() + + def test_poa_block_verify_without_storage(self) -> None: + block = self._get_valid_poa_block() + + verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) + + verify_pow_wrapped = Mock(wraps=self.verifiers.vertex.verify_pow) + verify_no_inputs_wrapped = Mock(wraps=self.verifiers.block.verify_no_inputs) + verify_output_token_indexes_wrapped = Mock(wraps=self.verifiers.block.verify_output_token_indexes) + verify_number_of_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_number_of_outputs) + verify_data_wrapped = Mock(wraps=self.verifiers.block.verify_data) + verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) + + with ( + patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), + patch.object(VertexVerifier, 'verify_pow', verify_pow_wrapped), + patch.object(BlockVerifier, 'verify_no_inputs', verify_no_inputs_wrapped), + patch.object(BlockVerifier, 'verify_output_token_indexes', verify_output_token_indexes_wrapped), + patch.object(VertexVerifier, 'verify_number_of_outputs', verify_number_of_outputs_wrapped), + patch.object(BlockVerifier, 'verify_data', verify_data_wrapped), + patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped), + ): + self.manager.verification_service.verify_without_storage(block) + + # Vertex methods + verify_outputs_wrapped.assert_called_once() + + # Block methods + verify_pow_wrapped.assert_not_called() + verify_no_inputs_wrapped.assert_called_once() + verify_output_token_indexes_wrapped.assert_called_once() + verify_number_of_outputs_wrapped.assert_called_once() + verify_data_wrapped.assert_called_once() + verify_sigops_output_wrapped.assert_called_once() + + def test_poa_block_verify(self) -> None: + block = self._get_valid_poa_block() + + verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) + + verify_pow_wrapped = Mock(wraps=self.verifiers.vertex.verify_pow) + verify_no_inputs_wrapped = Mock(wraps=self.verifiers.block.verify_no_inputs) + verify_output_token_indexes_wrapped = Mock(wraps=self.verifiers.block.verify_output_token_indexes) + verify_number_of_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_number_of_outputs) + verify_data_wrapped = Mock(wraps=self.verifiers.block.verify_data) + verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) + verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents) + verify_height_wrapped = Mock(wraps=self.verifiers.block.verify_height) + verify_mandatory_signaling_wrapped = Mock(wraps=self.verifiers.block.verify_mandatory_signaling) + + with ( + patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), + patch.object(VertexVerifier, 'verify_pow', verify_pow_wrapped), + patch.object(BlockVerifier, 'verify_no_inputs', verify_no_inputs_wrapped), + patch.object(BlockVerifier, 'verify_output_token_indexes', verify_output_token_indexes_wrapped), + patch.object(VertexVerifier, 'verify_number_of_outputs', verify_number_of_outputs_wrapped), + patch.object(BlockVerifier, 'verify_data', verify_data_wrapped), + patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped), + patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped), + patch.object(BlockVerifier, 'verify_height', verify_height_wrapped), + patch.object(BlockVerifier, 'verify_mandatory_signaling', verify_mandatory_signaling_wrapped), + ): + self.manager.verification_service.verify(block) + + # Vertex methods + verify_outputs_wrapped.assert_called_once() + + # Block methods + verify_pow_wrapped.assert_not_called() + verify_no_inputs_wrapped.assert_called_once() + verify_output_token_indexes_wrapped.assert_called_once() + verify_number_of_outputs_wrapped.assert_called_once() + verify_data_wrapped.assert_called_once() + verify_sigops_output_wrapped.assert_called_once() + verify_parents_wrapped.assert_called_once() + verify_height_wrapped.assert_called_once() + verify_mandatory_signaling_wrapped.assert_called_once() + + def test_poa_block_validate_basic(self) -> None: + block = self._get_valid_poa_block() + + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) + verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) + verify_poa_wrapped = Mock(wraps=self.verifiers.poa_block.verify_poa) + + with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), + patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), + patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), + patch.object(PoaBlockVerifier, 'verify_poa', verify_poa_wrapped), + ): + self.manager.verification_service.validate_basic(block) + + # Vertex methods + verify_version_wrapped.assert_called_once() + + # Block methods + verify_weight_wrapped.assert_not_called() + verify_reward_wrapped.assert_called_once() + verify_poa_wrapped.assert_called_once() + + # validation should be BASIC + self.assertEqual(block.get_metadata().validation, ValidationState.BASIC) + + # full validation should still pass and the validation updated to FULL + self.manager.verification_service.validate_full(block) + self.assertEqual(block.get_metadata().validation, ValidationState.FULL) + + # and if running basic validation again it shouldn't validate or change the validation state + verify_weight_wrapped2 = Mock(wraps=self.verifiers.block.verify_weight) + verify_reward_wrapped2 = Mock(wraps=self.verifiers.block.verify_reward) + + with ( + patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped2), + patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped2), + ): + self.manager.verification_service.validate_basic(block) + + # Block methods + verify_weight_wrapped2.assert_not_called() + verify_reward_wrapped2.assert_not_called() + + # validation should still be FULL, it must not be BASIC + self.assertEqual(block.get_metadata().validation, ValidationState.FULL) + + def test_poa_block_validate_full(self) -> None: + block = self._get_valid_poa_block() + + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) + + verify_pow_wrapped = Mock(wraps=self.verifiers.vertex.verify_pow) + verify_no_inputs_wrapped = Mock(wraps=self.verifiers.block.verify_no_inputs) + verify_output_token_indexes_wrapped = Mock(wraps=self.verifiers.block.verify_output_token_indexes) + verify_number_of_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_number_of_outputs) + verify_data_wrapped = Mock(wraps=self.verifiers.block.verify_data) + verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) + verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents) + verify_height_wrapped = Mock(wraps=self.verifiers.block.verify_height) + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) + verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) + verify_mandatory_signaling_wrapped = Mock(wraps=self.verifiers.block.verify_mandatory_signaling) + verify_poa_wrapped = Mock(wraps=self.verifiers.poa_block.verify_poa) + + with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), + patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), + patch.object(VertexVerifier, 'verify_pow', verify_pow_wrapped), + patch.object(BlockVerifier, 'verify_no_inputs', verify_no_inputs_wrapped), + patch.object(BlockVerifier, 'verify_output_token_indexes', verify_output_token_indexes_wrapped), + patch.object(VertexVerifier, 'verify_number_of_outputs', verify_number_of_outputs_wrapped), + patch.object(BlockVerifier, 'verify_data', verify_data_wrapped), + patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped), + patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped), + patch.object(BlockVerifier, 'verify_height', verify_height_wrapped), + patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), + patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), + patch.object(BlockVerifier, 'verify_mandatory_signaling', verify_mandatory_signaling_wrapped), + patch.object(PoaBlockVerifier, 'verify_poa', verify_poa_wrapped), + ): + self.manager.verification_service.validate_full(block) + + # Vertex methods + verify_version_wrapped.assert_called_once() + verify_outputs_wrapped.assert_called_once() + + # Block methods + verify_pow_wrapped.assert_not_called() + verify_no_inputs_wrapped.assert_called_once() + verify_output_token_indexes_wrapped.assert_called_once() + verify_number_of_outputs_wrapped.assert_called_once() + verify_data_wrapped.assert_called_once() + verify_sigops_output_wrapped.assert_called_once() + verify_parents_wrapped.assert_called_once() + verify_height_wrapped.assert_called_once() + verify_weight_wrapped.assert_not_called() + verify_reward_wrapped.assert_called_once() + verify_mandatory_signaling_wrapped.assert_called_once() + verify_poa_wrapped.assert_called_once() + + +class SyncV1PoaVerificationTest(unittest.SyncV1Params, BasePoaVerificationTest): + __test__ = True + + +class SyncV2PoaVerificationTest(unittest.SyncV2Params, BasePoaVerificationTest): + __test__ = True + + +# sync-bridge should behave like sync-v2 +class SyncBridgePoaVerificationTest(unittest.SyncBridgeParams, SyncV2PoaVerificationTest): + pass diff --git a/tests/poa/utils.py b/tests/poa/utils.py new file mode 100644 index 000000000..096045c4f --- /dev/null +++ b/tests/poa/utils.py @@ -0,0 +1,53 @@ +# Copyright 2024 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cryptography.hazmat.primitives.asymmetric import ec + +from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings +from hathor.consensus.consensus_settings import ConsensusType, PoaSettings, PoaSignerSettings +from hathor.consensus.poa import PoaSigner +from hathor.crypto.util import get_public_key_bytes_compressed + + +def get_signer() -> PoaSigner: + return PoaSigner(ec.generate_private_key(ec.SECP256K1())) + + +def get_settings( + *poa_signers: PoaSigner | PoaSignerSettings, + time_between_blocks: int | None = None +) -> HathorSettings: + signers = [] + for signer in poa_signers: + if isinstance(signer, PoaSignerSettings): + poa_settings = signer + else: + public_key = signer.get_public_key() + public_key_bytes = get_public_key_bytes_compressed(public_key) + poa_settings = PoaSignerSettings(public_key=public_key_bytes) + signers.append(poa_settings) + + settings = get_global_settings() + settings = settings._replace( + AVG_TIME_BETWEEN_BLOCKS=time_between_blocks or settings.AVG_TIME_BETWEEN_BLOCKS, + BLOCKS_PER_HALVING=None, + INITIAL_TOKEN_UNITS_PER_BLOCK=0, + MINIMUM_TOKEN_UNITS_PER_BLOCK=0, + CONSENSUS_ALGORITHM=PoaSettings( + type=ConsensusType.PROOF_OF_AUTHORITY, + signers=tuple(signers), + ), + ) + return settings diff --git a/tests/resources/p2p/test_add_peer.py b/tests/resources/p2p/test_add_peer.py index 4d61fcb3b..c22598b8a 100644 --- a/tests/resources/p2p/test_add_peer.py +++ b/tests/resources/p2p/test_add_peer.py @@ -1,5 +1,6 @@ from twisted.internet.defer import inlineCallbacks +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.peer_id import PeerId from hathor.p2p.resources import AddPeersResource from tests import unittest @@ -21,7 +22,7 @@ def test_connecting_peers(self): # test when we send a peer we're already connected to peer = PeerId() - peer.entrypoints = ['tcp://localhost:8006'] + peer.entrypoints = [Entrypoint.parse('tcp://localhost:8006')] self.manager.connections.peer_storage.add(peer) response = yield self.web.post('p2p/peers', ['tcp://localhost:8006', 'tcp://localhost:8007']) data = response.json_value() diff --git a/tests/resources/p2p/test_status.py b/tests/resources/p2p/test_status.py index ea80ece6e..7ab42ae68 100644 --- a/tests/resources/p2p/test_status.py +++ b/tests/resources/p2p/test_status.py @@ -3,6 +3,7 @@ import hathor from hathor.conf.unittests import SETTINGS +from hathor.p2p.entrypoint import Entrypoint from hathor.p2p.resources import StatusResource from hathor.simulator import FakeConnection from tests import unittest @@ -15,8 +16,11 @@ class BaseStatusTest(_BaseResourceTest._ResourceTest): def setUp(self): super().setUp() self.web = StubSite(StatusResource(self.manager)) + self.entrypoint = Entrypoint.parse('tcp://192.168.1.1:54321') + self.manager.connections.my_peer.entrypoints.append(self.entrypoint) self.manager2 = self.create_peer('testnet') + self.manager2.connections.my_peer.entrypoints.append(self.entrypoint) self.conn1 = FakeConnection(self.manager, self.manager2) @inlineCallbacks diff --git a/tests/resources/transaction/test_block_at_height.py b/tests/resources/transaction/test_block_at_height.py index 9800b3816..c7c006e8e 100644 --- a/tests/resources/transaction/test_block_at_height.py +++ b/tests/resources/transaction/test_block_at_height.py @@ -1,9 +1,10 @@ from twisted.internet.defer import inlineCallbacks -from hathor.simulator.utils import add_new_blocks +from hathor.simulator.utils import add_new_block, add_new_blocks from hathor.transaction.resources import BlockAtHeightResource from tests import unittest from tests.resources.base_resource import StubSite, _BaseResourceTest +from tests.utils import add_blocks_unlock_reward, add_new_tx class BaseBlockAtHeightTest(_BaseResourceTest._ResourceTest): @@ -14,6 +15,62 @@ def setUp(self): self.web = StubSite(BlockAtHeightResource(self.manager)) self.manager.wallet.unlock(b'MYPASS') + @inlineCallbacks + def test_include_full(self): + add_new_block(self.manager, advance_clock=1) + add_blocks_unlock_reward(self.manager) + address = self.manager.wallet.get_unused_address() + + confirmed_tx_list = [] + for _ in range(15): + confirmed_tx_list.append(add_new_tx(self.manager, address, 1)) + + block = add_new_block(self.manager, advance_clock=1) + height = block.get_height() + + # non-confirmed transactions + for _ in range(15): + add_new_tx(self.manager, address, 1) + + response = yield self.web.get("block_at_height", { + b'height': str(height).encode('ascii'), + b'include_transactions': b'full', + }) + data = response.json_value() + + self.assertTrue(data['success']) + response_tx_ids = set(x['tx_id'] for x in data['transactions']) + expected_tx_ids = set(tx.hash.hex() for tx in confirmed_tx_list) + self.assertTrue(response_tx_ids.issubset(expected_tx_ids)) + + @inlineCallbacks + def test_include_txids(self): + add_new_block(self.manager, advance_clock=1) + add_blocks_unlock_reward(self.manager) + address = self.manager.wallet.get_unused_address() + + confirmed_tx_list = [] + for _ in range(15): + confirmed_tx_list.append(add_new_tx(self.manager, address, 1)) + + block = add_new_block(self.manager, advance_clock=1) + height = block.get_height() + + # non-confirmed transactions + for _ in range(15): + add_new_tx(self.manager, address, 1) + + response = yield self.web.get("block_at_height", { + b'height': str(height).encode('ascii'), + b'include_transactions': b'txid', + }) + data = response.json_value() + + self.assertTrue(data['success']) + response_tx_ids = set(data['tx_ids']) + expected_tx_ids = set(tx.hash.hex() for tx in confirmed_tx_list) + self.assertTrue(response_tx_ids.issubset(expected_tx_ids)) + @inlineCallbacks def test_get(self): blocks = add_new_blocks(self.manager, 4, advance_clock=1) diff --git a/tests/sysctl/test_websocket.py b/tests/sysctl/test_websocket.py index 3c7749f3e..920eb3b87 100644 --- a/tests/sysctl/test_websocket.py +++ b/tests/sysctl/test_websocket.py @@ -5,8 +5,15 @@ class WebsocketSysctlTestCase(unittest.TestCase): + _enable_sync_v1 = True + _enable_sync_v2 = True + + def setUp(self): + super().setUp() + self.manager = self.create_peer('testnet') + def test_max_subs_addrs_conn(self): - ws_factory = HathorAdminWebsocketFactory() + ws_factory = HathorAdminWebsocketFactory(self.manager) sysctl = WebsocketManagerSysctl(ws_factory) sysctl.unsafe_set('max_subs_addrs_conn', 10) @@ -25,7 +32,7 @@ def test_max_subs_addrs_conn(self): sysctl.unsafe_set('max_subs_addrs_conn', -2) def test_max_subs_addrs_empty(self): - ws_factory = HathorAdminWebsocketFactory() + ws_factory = HathorAdminWebsocketFactory(self.manager) sysctl = WebsocketManagerSysctl(ws_factory) sysctl.unsafe_set('max_subs_addrs_empty', 10) diff --git a/tests/tx/test_accumulated_weight.py b/tests/tx/test_accumulated_weight.py index 8f19e00ff..74507b754 100644 --- a/tests/tx/test_accumulated_weight.py +++ b/tests/tx/test_accumulated_weight.py @@ -10,7 +10,7 @@ class BaseAccumulatedWeightTestCase(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_block.py b/tests/tx/test_block.py index c5f698965..7bef0f834 100644 --- a/tests/tx/test_block.py +++ b/tests/tx/test_block.py @@ -28,7 +28,7 @@ def test_calculate_feature_activation_bit_counts_genesis(): settings = get_global_settings() - storage = TransactionMemoryStorage() + storage = TransactionMemoryStorage(settings=settings) genesis_block = storage.get_transaction(settings.GENESIS_BLOCK_HASH) assert isinstance(genesis_block, Block) result = genesis_block.get_feature_activation_bit_counts() diff --git a/tests/tx/test_blockchain.py b/tests/tx/test_blockchain.py index 3e30adb28..b14a16f56 100644 --- a/tests/tx/test_blockchain.py +++ b/tests/tx/test_blockchain.py @@ -24,7 +24,7 @@ class BaseBlockchainTestCase(unittest.TestCase): """ def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_genesis.py b/tests/tx/test_genesis.py index fe08117bf..f2839bf6b 100644 --- a/tests/tx/test_genesis.py +++ b/tests/tx/test_genesis.py @@ -29,12 +29,12 @@ def get_genesis_output(): class GenesisTest(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: super().setUp() self._daa = DifficultyAdjustmentAlgorithm(settings=self._settings) verifiers = VertexVerifiers.create_defaults(settings=self._settings, daa=self._daa, feature_service=Mock()) - self._verification_service = VerificationService(verifiers=verifiers) - self.storage = TransactionMemoryStorage() + self._verification_service = VerificationService(settings=self._settings, verifiers=verifiers) + self.storage = TransactionMemoryStorage(settings=settings) def test_pow(self): verifier = VertexVerifier(settings=self._settings) diff --git a/tests/tx/test_indexes.py b/tests/tx/test_indexes.py index cfdc607be..d119e9580 100644 --- a/tests/tx/test_indexes.py +++ b/tests/tx/test_indexes.py @@ -5,6 +5,7 @@ from hathor.simulator.utils import add_new_block, add_new_blocks from hathor.storage.rocksdb_storage import RocksDBStorage from hathor.transaction import Transaction +from hathor.transaction.vertex_parser import VertexParser from hathor.util import iwindows from hathor.wallet import Wallet from tests import unittest @@ -694,7 +695,7 @@ def setUp(self): super().setUp() self.wallet = Wallet() - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] @@ -724,7 +725,8 @@ def setUp(self): directory = tempfile.mkdtemp() self.tmpdirs.append(directory) rocksdb_storage = RocksDBStorage(path=directory) - self.tx_storage = TransactionRocksDBStorage(rocksdb_storage) + parser = VertexParser(settings=self._settings) + self.tx_storage = TransactionRocksDBStorage(rocksdb_storage, settings=self._settings, vertex_parser=parser) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_indexes4.py b/tests/tx/test_indexes4.py index 7777d69e1..4fee7ca40 100644 --- a/tests/tx/test_indexes4.py +++ b/tests/tx/test_indexes4.py @@ -11,7 +11,7 @@ class BaseSimulatorIndexesTestCase(unittest.TestCase): __test__ = False def _build_randomized_blockchain(self, *, utxo_index=False): - tx_storage = TransactionMemoryStorage() + tx_storage = TransactionMemoryStorage(settings=self._settings) manager = self.create_peer('testnet', tx_storage=tx_storage, unlock_wallet=True, wallet_index=True, use_memory_index=True, utxo_index=utxo_index) diff --git a/tests/tx/test_mining.py b/tests/tx/test_mining.py index a6731dbad..2a9a0a260 100644 --- a/tests/tx/test_mining.py +++ b/tests/tx/test_mining.py @@ -24,7 +24,7 @@ class BaseMiningTest(unittest.TestCase): def setUp(self): super().setUp() - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] diff --git a/tests/tx/test_reward_lock.py b/tests/tx/test_reward_lock.py index 80f6f6e18..5f8c943fa 100644 --- a/tests/tx/test_reward_lock.py +++ b/tests/tx/test_reward_lock.py @@ -24,7 +24,7 @@ def setUp(self): self.genesis_public_key = self.genesis_private_key.public_key() # this makes sure we can spend the genesis outputs - self.tx_storage = TransactionMemoryStorage() + self.tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis = self.tx_storage.get_all_genesis() self.genesis_blocks = [tx for tx in self.genesis if tx.is_block] self.genesis_txs = [tx for tx in self.genesis if not tx.is_block] @@ -179,6 +179,17 @@ def test_mempool_tx_invalid_after_reorg(self): self.assertTrue(tx.get_metadata().validation.is_invalid()) self.assertFalse(self.manager.tx_storage.transaction_exists(tx.hash)) + # assert that the tx has been removed from its dependencies' metadata + for parent_id in tx.parents: + parent = self.manager.tx_storage.get_transaction(parent_id) + assert tx.hash not in parent.get_metadata().children + + for tx_input in tx.inputs: + spent_tx = tx.get_spent_tx(tx_input) + spent_outputs = spent_tx.get_metadata().spent_outputs + assert len(spent_outputs) == 1 + assert tx.hash not in spent_outputs[0] + @pytest.mark.xfail(reason='this is no longer the case, timestamp will not matter', strict=True) def test_classic_reward_lock_timestamp_expected_to_fail(self): # add block with a reward we can spend diff --git a/tests/tx/test_scripts.py b/tests/tx/test_scripts.py index b6cf99566..34ce6ac25 100644 --- a/tests/tx/test_scripts.py +++ b/tests/tx/test_scripts.py @@ -65,7 +65,7 @@ class TestScripts(unittest.TestCase): def setUp(self): super().setUp() - tx_storage = TransactionMemoryStorage() + tx_storage = TransactionMemoryStorage(settings=self._settings) self.genesis_blocks = [tx for tx in tx_storage.get_all_genesis() if tx.is_block] self.genesis_txs = [tx for tx in tx_storage.get_all_genesis() if not tx.is_block] diff --git a/tests/tx/test_stratum.py b/tests/tx/test_stratum.py index 47331a341..1445684b4 100644 --- a/tests/tx/test_stratum.py +++ b/tests/tx/test_stratum.py @@ -253,7 +253,7 @@ class BaseStratumClientTest(unittest.TestCase): def setUp(self): super().setUp() - storage = TransactionMemoryStorage() + storage = TransactionMemoryStorage(settings=self._settings) self.block = storage.get_transaction(self._settings.GENESIS_BLOCK_HASH) self.transport = StringTransportWithDisconnection() self.protocol = StratumClient(reactor=self.clock) diff --git a/tests/tx/test_tokens.py b/tests/tx/test_tokens.py index 0906477e1..a54e4f765 100644 --- a/tests/tx/test_tokens.py +++ b/tests/tx/test_tokens.py @@ -135,7 +135,7 @@ def test_token_mint(self): # mint tokens and transfer mint authority mint_amount = 10000000 - deposit_amount = get_deposit_amount(mint_amount) + deposit_amount = get_deposit_amount(self._settings, mint_amount) _input1 = TxInput(tx.hash, 1, b'') _input2 = TxInput(tx.hash, 3, b'') token_output1 = TxOutput(mint_amount, script, 1) @@ -195,7 +195,7 @@ def test_token_mint(self): # try to mint and deposit less tokens than necessary mint_amount = 10000000 - deposit_amount = get_deposit_amount(mint_amount) - 1 + deposit_amount = get_deposit_amount(self._settings, mint_amount) - 1 _input1 = TxInput(tx.hash, 1, b'') _input2 = TxInput(tx.hash, 3, b'') token_output1 = TxOutput(mint_amount, script, 1) @@ -241,7 +241,7 @@ def test_token_melt(self): # melt tokens and transfer melt authority melt_amount = 100 new_amount = tx.outputs[0].value - melt_amount - withdraw_amount = get_withdraw_amount(melt_amount) + withdraw_amount = get_withdraw_amount(self._settings, melt_amount) _input1 = TxInput(tx.hash, 0, b'') _input2 = TxInput(tx.hash, 2, b'') token_output1 = TxOutput(new_amount, script, 1) @@ -279,7 +279,7 @@ def test_token_melt(self): # melt tokens and withdraw more than what's allowed melt_amount = 100 - withdraw_amount = get_withdraw_amount(melt_amount) + withdraw_amount = get_withdraw_amount(self._settings, melt_amount) _input1 = TxInput(tx.hash, 0, b'') _input2 = TxInput(tx.hash, 2, b'') token_output1 = TxOutput(tx.outputs[0].value - melt_amount, script, 1) @@ -371,7 +371,7 @@ def test_token_index_with_conflict(self, mint_amount=0): # new tx minting tokens mint_amount = 300 - deposit_amount = get_deposit_amount(mint_amount) + deposit_amount = get_deposit_amount(self._settings, mint_amount) script = P2PKH.create_output_script(self.address) # inputs mint_input = TxInput(tx.hash, 1, b'') diff --git a/tests/tx/test_tx_deserialization.py b/tests/tx/test_tx_deserialization.py index ba19abc28..f467c26f0 100644 --- a/tests/tx/test_tx_deserialization.py +++ b/tests/tx/test_tx_deserialization.py @@ -14,7 +14,7 @@ def setUp(self) -> None: super().setUp() daa = DifficultyAdjustmentAlgorithm(settings=self._settings) verifiers = VertexVerifiers.create_defaults(settings=self._settings, daa=daa, feature_service=Mock()) - self._verification_service = VerificationService(verifiers=verifiers) + self._verification_service = VerificationService(settings=self._settings, verifiers=verifiers) def test_deserialize(self): cls = self.get_tx_class() diff --git a/tests/tx/test_verification.py b/tests/tx/test_verification.py index b3414d0b6..90622eae6 100644 --- a/tests/tx/test_verification.py +++ b/tests/tx/test_verification.py @@ -114,15 +114,21 @@ def _get_valid_token_creation_tx(self) -> TokenCreationTransaction: def test_block_verify_basic(self) -> None: block = self._get_valid_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), ): self.manager.verification_service.verify_basic(block) + # Vertex methods + verify_version_wrapped.assert_called_once() + # Block methods verify_weight_wrapped.assert_called_once() verify_reward_wrapped.assert_called_once() @@ -207,15 +213,21 @@ def test_block_verify(self) -> None: def test_block_validate_basic(self) -> None: block = self._get_valid_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), ): self.manager.verification_service.validate_basic(block) + # Vertex methods + verify_version_wrapped.assert_called_once() + # Block methods verify_weight_wrapped.assert_called_once() verify_reward_wrapped.assert_called_once() @@ -247,6 +259,7 @@ def test_block_validate_basic(self) -> None: def test_block_validate_full(self) -> None: block = self._get_valid_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_pow_wrapped = Mock(wraps=self.verifiers.vertex.verify_pow) @@ -262,6 +275,7 @@ def test_block_validate_full(self) -> None: verify_mandatory_signaling_wrapped = Mock(wraps=self.verifiers.block.verify_mandatory_signaling) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(VertexVerifier, 'verify_pow', verify_pow_wrapped), patch.object(BlockVerifier, 'verify_no_inputs', verify_no_inputs_wrapped), @@ -278,6 +292,7 @@ def test_block_validate_full(self) -> None: self.manager.verification_service.validate_full(block) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Block methods @@ -296,15 +311,21 @@ def test_block_validate_full(self) -> None: def test_merge_mined_block_verify_basic(self) -> None: block = self._get_valid_merge_mined_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), ): self.manager.verification_service.verify_basic(block) + # Vertex methods + verify_version_wrapped.assert_called_once() + # Block methods verify_weight_wrapped.assert_called_once() verify_reward_wrapped.assert_called_once() @@ -401,15 +422,21 @@ def test_merge_mined_block_verify(self) -> None: def test_merge_mined_block_validate_basic(self) -> None: block = self._get_valid_merge_mined_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) + verify_weight_wrapped = Mock(wraps=self.verifiers.block.verify_weight) verify_reward_wrapped = Mock(wraps=self.verifiers.block.verify_reward) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(BlockVerifier, 'verify_weight', verify_weight_wrapped), patch.object(BlockVerifier, 'verify_reward', verify_reward_wrapped), ): self.manager.verification_service.validate_basic(block) + # Vertex methods + verify_version_wrapped.assert_called_once() + # Block methods verify_weight_wrapped.assert_called_once() verify_reward_wrapped.assert_called_once() @@ -441,6 +468,7 @@ def test_merge_mined_block_validate_basic(self) -> None: def test_merge_mined_block_validate_full(self) -> None: block = self._get_valid_merge_mined_block() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_pow_wrapped = Mock(wraps=self.verifiers.vertex.verify_pow) @@ -458,6 +486,7 @@ def test_merge_mined_block_validate_full(self) -> None: verify_aux_pow_wrapped = Mock(wraps=self.verifiers.merge_mined_block.verify_aux_pow) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(VertexVerifier, 'verify_pow', verify_pow_wrapped), patch.object(BlockVerifier, 'verify_no_inputs', verify_no_inputs_wrapped), @@ -475,6 +504,7 @@ def test_merge_mined_block_validate_full(self) -> None: self.manager.verification_service.validate_full(block) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Block methods @@ -496,6 +526,7 @@ def test_merge_mined_block_validate_full(self) -> None: def test_transaction_verify_basic(self) -> None: tx = self._get_valid_tx() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -507,6 +538,7 @@ def test_transaction_verify_basic(self) -> None: verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -519,6 +551,7 @@ def test_transaction_verify_basic(self) -> None: self.manager.verification_service.verify_basic(tx) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Transaction methods @@ -616,6 +649,7 @@ def test_transaction_validate_basic(self) -> None: add_blocks_unlock_reward(self.manager) tx = self._get_valid_tx() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -627,6 +661,7 @@ def test_transaction_validate_basic(self) -> None: verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -639,6 +674,7 @@ def test_transaction_validate_basic(self) -> None: self.manager.verification_service.validate_basic(tx) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Transaction methods @@ -693,6 +729,7 @@ def test_transaction_validate_full(self) -> None: add_blocks_unlock_reward(self.manager) tx = self._get_valid_tx() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -710,6 +747,7 @@ def test_transaction_validate_full(self) -> None: verify_reward_locked_wrapped = Mock(wraps=self.verifiers.tx.verify_reward_locked) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -728,6 +766,7 @@ def test_transaction_validate_full(self) -> None: self.manager.verification_service.validate_full(tx) # Vertex methods + verify_version_wrapped.assert_called_once() assert verify_outputs_wrapped.call_count == 2 # Transaction methods @@ -783,6 +822,7 @@ def test_transaction_validate_full(self) -> None: def test_token_creation_transaction_verify_basic(self) -> None: tx = self._get_valid_token_creation_tx() + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -794,6 +834,7 @@ def test_token_creation_transaction_verify_basic(self) -> None: verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -806,6 +847,7 @@ def test_token_creation_transaction_verify_basic(self) -> None: self.manager.verification_service.verify_basic(tx) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Transaction methods @@ -910,6 +952,7 @@ def test_token_creation_transaction_validate_basic(self) -> None: tx = self._get_valid_token_creation_tx() tx.get_metadata().validation = ValidationState.INITIAL + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -921,6 +964,7 @@ def test_token_creation_transaction_validate_basic(self) -> None: verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -933,6 +977,7 @@ def test_token_creation_transaction_validate_basic(self) -> None: self.manager.verification_service.validate_basic(tx) # Vertex methods + verify_version_wrapped.assert_called_once() verify_outputs_wrapped.assert_called_once() # Transaction methods @@ -987,6 +1032,7 @@ def test_token_creation_transaction_validate_full(self) -> None: tx = self._get_valid_token_creation_tx() tx.get_metadata().validation = ValidationState.INITIAL + verify_version_wrapped = Mock(wraps=self.verifiers.vertex.verify_version) verify_outputs_wrapped = Mock(wraps=self.verifiers.vertex.verify_outputs) verify_parents_basic_wrapped = Mock(wraps=self.verifiers.tx.verify_parents_basic) @@ -1007,6 +1053,7 @@ def test_token_creation_transaction_validate_full(self) -> None: verify_minted_tokens_wrapped = Mock(wraps=self.verifiers.token_creation_tx.verify_minted_tokens) with ( + patch.object(VertexVerifier, 'verify_version', verify_version_wrapped), patch.object(VertexVerifier, 'verify_outputs', verify_outputs_wrapped), patch.object(TransactionVerifier, 'verify_parents_basic', verify_parents_basic_wrapped), patch.object(TransactionVerifier, 'verify_weight', verify_weight_wrapped), @@ -1027,6 +1074,7 @@ def test_token_creation_transaction_validate_full(self) -> None: self.manager.verification_service.validate_full(tx) # Vertex methods + verify_version_wrapped.assert_called_once() assert verify_outputs_wrapped.call_count == 2 # Transaction methods diff --git a/tests/unittest.py b/tests/unittest.py index f92bc0f50..bac8fca7c 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -11,8 +11,8 @@ from hathor.builder import BuildArtifacts, Builder from hathor.checkpoint import Checkpoint -from hathor.conf import HathorSettings from hathor.conf.get_settings import get_global_settings +from hathor.conf.settings import HathorSettings from hathor.daa import DifficultyAdjustmentAlgorithm, TestMode from hathor.event import EventManager from hathor.event.storage import EventStorage @@ -33,7 +33,6 @@ logger = get_logger() main = ut_main -settings = HathorSettings() USE_MEMORY_STORAGE = os.environ.get('HATHOR_TEST_MEMORY_STORAGE', 'false').lower() == 'true' @@ -84,11 +83,12 @@ class SyncBridgeParams: class TestBuilder(Builder): __test__ = False - def __init__(self) -> None: + def __init__(self, settings: HathorSettings | None = None) -> None: super().__init__() self.set_network('testnet') # default builder has sync-v2 enabled for tests self.enable_sync_v2() + self.set_settings(settings or get_global_settings()) def build(self) -> BuildArtifacts: artifacts = super().build() @@ -118,6 +118,7 @@ def setUp(self) -> None: self.tmpdirs: list[str] = [] self.clock = TestMemoryReactorClock() self.clock.advance(time.time()) + self.reactor = self.clock self.log = logger.new() self.reset_peer_id_pool() self.seed = secrets.randbits(64) if self.seed_config is None else self.seed_config @@ -215,7 +216,8 @@ def create_peer( # type: ignore[no-untyped-def] enable_sync_v1, enable_sync_v2 = self._syncVersionFlags(enable_sync_v1, enable_sync_v2) builder = self.get_builder(network) \ - .set_full_verification(full_verification) + .set_full_verification(full_verification) \ + .set_settings(self._settings) if checkpoints is not None: builder.set_checkpoints(checkpoints) diff --git a/tests/utils.py b/tests/utils.py index c72682c4d..ad7a81a57 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -399,7 +399,7 @@ def create_tokens(manager: 'HathorManager', address_b58: Optional[str] = None, m address = decode_address(address_b58) script = P2PKH.create_output_script(address) - deposit_amount = get_deposit_amount(mint_amount) + deposit_amount = get_deposit_amount(manager._settings, mint_amount) if nft_data: # NFT creation needs 0.01 HTR of fee deposit_amount += 1 diff --git a/tests/websocket/test_async_iterators.py b/tests/websocket/test_async_iterators.py new file mode 100644 index 000000000..a3435e723 --- /dev/null +++ b/tests/websocket/test_async_iterators.py @@ -0,0 +1,139 @@ +from typing import AsyncIterator, TypeVar + +from twisted.internet.defer import Deferred + +from hathor.wallet import HDWallet +from hathor.websocket.exception import InvalidAddress, InvalidXPub +from hathor.websocket.iterators import ( + AddressItem, + ManualAddressSequencer, + VertexItem, + aiter_xpub_addresses, + gap_limit_search, +) +from tests.unittest import TestCase +from tests.utils import GENESIS_ADDRESS_B58 + +T = TypeVar('T') + + +async def async_islice(iterable: AsyncIterator[T], stop: int) -> AsyncIterator[T]: + count = 0 + async for item in iterable: + if count >= stop: + break + yield item + count += 1 + + +class AsyncIteratorsTestCase(TestCase): + _enable_sync_v1 = True + _enable_sync_v2 = True + + def setUp(self) -> None: + super().setUp() + + self.manager = self.create_peer('mainnet', wallet_index=True) + self.settings = self.manager._settings + + # Create wallet. + wallet = HDWallet() + wallet.unlock(self.manager.tx_storage) + + # Create xpub and list of addresses. + self.xpub = wallet.get_xpub() + self.xpub_addresses = [ + AddressItem(idx, wallet.get_address(wallet.get_key_at_index(idx))) + for idx in range(20) + ] + + async def test_xpub_sequencer_default_first_index(self) -> None: + xpub = self.xpub + expected_result = self.xpub_addresses + + sequencer = aiter_xpub_addresses(xpub) + result = [item async for item in async_islice(aiter(sequencer), len(expected_result))] + self.assertEqual(result, expected_result) + + async def test_xpub_sequencer_other_first_index(self) -> None: + xpub = self.xpub + first_index = 8 + expected_result = self.xpub_addresses[first_index:] + + sequencer = aiter_xpub_addresses(xpub, first_index=first_index) + result = [item async for item in async_islice(aiter(sequencer), len(expected_result))] + self.assertEqual(result, expected_result) + + async def test_xpub_sequencer_invalid(self) -> None: + with self.assertRaises(InvalidXPub): + async for _ in aiter_xpub_addresses('invalid xpub'): + pass + + async def test_manual_invalid(self) -> None: + address_iter = ManualAddressSequencer() + with self.assertRaises(InvalidAddress): + address_iter.add_addresses([AddressItem(0, 'a')], last=True) + + async def test_manual_last_true(self) -> None: + expected_result = self.xpub_addresses + + iterable = ManualAddressSequencer() + iterable.add_addresses(expected_result, last=True) + + result = [item async for item in iterable] + self.assertEqual(result, expected_result) + + async def test_manual_two_tranches(self) -> None: + expected_result = self.xpub_addresses + + iterable = ManualAddressSequencer() + n = 8 + iterable.add_addresses(expected_result[:n], last=False) + + result = [] + is_running = False + + async def collect_results(): + nonlocal is_running + nonlocal result + is_running = True + result = [item async for item in iterable] + is_running = False + + self.reactor.callLater(0, lambda: Deferred.fromCoroutine(collect_results())) + self.reactor.advance(5) + self.assertTrue(is_running) + + self.reactor.callLater(0, lambda: iterable.add_addresses(expected_result[n:], last=True)) + self.reactor.advance(5) + self.assertFalse(is_running) + + self.assertEqual(result, expected_result) + + async def test_gap_limit_xpub(self) -> None: + xpub = self.xpub + gap_limit = 8 + expected_result = self.xpub_addresses[:gap_limit] + + address_iter = aiter_xpub_addresses(xpub) + search = gap_limit_search(self.manager, address_iter, gap_limit=gap_limit) + + result = [item async for item in search] + self.assertEqual(result, expected_result) + + async def test_gap_limit_manual(self) -> None: + genesis = self.manager.tx_storage.get_genesis(self.settings.GENESIS_BLOCK_HASH) + genesis_address = GENESIS_ADDRESS_B58 + + gap_limit = 8 + addresses: list[AddressItem] = [AddressItem(0, genesis_address)] + self.xpub_addresses + expected_result: list[AddressItem | VertexItem] = list(addresses[:gap_limit + 1]) + expected_result.insert(1, VertexItem(genesis)) + + address_iter = ManualAddressSequencer() + # Adding more addresses than the gap limit. + address_iter.add_addresses(addresses, last=True) + search = gap_limit_search(self.manager, address_iter, gap_limit=gap_limit) + + result = [item async for item in search] + self.assertEqual(result, expected_result) diff --git a/tests/websocket/test_streamer.py b/tests/websocket/test_streamer.py new file mode 100644 index 000000000..a76148952 --- /dev/null +++ b/tests/websocket/test_streamer.py @@ -0,0 +1,99 @@ +import json +from typing import Any, Iterator + +from twisted.internet.testing import StringTransport + +from hathor.wallet import HDWallet +from hathor.websocket.factory import HathorAdminWebsocketFactory +from hathor.websocket.iterators import AddressItem, ManualAddressSequencer, gap_limit_search +from hathor.websocket.streamer import HistoryStreamer +from tests.unittest import TestCase +from tests.utils import GENESIS_ADDRESS_B58 + + +class AsyncIteratorsTestCase(TestCase): + _enable_sync_v1 = True + _enable_sync_v2 = True + + WS_PROTOCOL_MESSAGE_SEPARATOR = b'\x81' + + def test_streamer(self) -> None: + manager = self.create_peer('mainnet', wallet_index=True) + settings = manager._settings + + # Settings. + stream_id = 'A001' + gap_limit = 8 + + # Get genesis information. + genesis = manager.tx_storage.get_genesis(settings.GENESIS_BLOCK_HASH) + genesis_address = GENESIS_ADDRESS_B58 + + # Create wallet. + wallet = HDWallet() + wallet.unlock(manager.tx_storage) + + # Create list of addresses. + addresses: list[AddressItem] = [AddressItem(0, genesis_address)] + for idx in range(1, 30): + addresses.append(AddressItem(idx, wallet.get_address(wallet.get_key_at_index(idx)))) + + # Create the expected result. + expected_result: list[dict[str, Any]] = [{'type': 'stream:history:begin', 'id': stream_id}] + expected_result += [ + { + 'type': 'stream:history:address', + 'id': stream_id, + 'index': item.index, + 'address': item.address, + 'subscribed': True + } + for item in addresses[:gap_limit + 1] + ] + expected_result.insert(2, { + 'type': 'stream:history:vertex', + 'id': stream_id, + 'data': genesis.to_json_extended(), + }) + expected_result.append({'type': 'stream:history:end', 'id': stream_id}) + + # Create both the address iterator and the GAP limit searcher. + address_iter = ManualAddressSequencer() + address_iter.add_addresses(addresses, last=True) + search = gap_limit_search(manager, address_iter, gap_limit=gap_limit) + + # Create the websocket factory and protocol. + factory = HathorAdminWebsocketFactory(manager) + factory.openHandshakeTimeout = 0 + protocol = factory.buildProtocol(None) + + # Create the transport and create a fake connection. + transport = StringTransport() + protocol.makeConnection(transport) + factory.connections.add(protocol) + protocol.state = protocol.STATE_OPEN + + # Create the history streamer. + streamer = HistoryStreamer(protocol=protocol, stream_id=stream_id, search=search) + streamer.start() + + # Run the streamer. + manager.reactor.advance(10) + + # Check the results. + items_iter = self._parse_ws_raw(transport.value()) + result = list(items_iter) + self.assertEqual(result, expected_result) + + def _parse_ws_raw(self, content: bytes) -> Iterator[dict]: + raw_messages = content.split(self.WS_PROTOCOL_MESSAGE_SEPARATOR) + for x in raw_messages: + if not x: + continue + if x[-1:] != b'}': + continue + idx = x.find(b'{') + if idx == -1: + continue + json_raw = x[idx:] + yield json.loads(json_raw) diff --git a/tests/websocket/test_websocket.py b/tests/websocket/test_websocket.py index 1583bba09..3b1df54cd 100644 --- a/tests/websocket/test_websocket.py +++ b/tests/websocket/test_websocket.py @@ -22,7 +22,7 @@ def setUp(self): self.network = 'testnet' self.manager = self.create_peer(self.network, wallet_index=True) - self.factory = HathorAdminWebsocketFactory(self.manager.metrics) + self.factory = HathorAdminWebsocketFactory(self.manager, self.manager.metrics) self.factory.subscribe(self.manager.pubsub) self.factory._setup_rate_limit() self.factory.openHandshakeTimeout = 0