Release-candidate v0.57.0-rc.1

.codecov.yml

@@ -1,5 +1,5 @@
 codecov:
-  branch: dev
+  branch: master
 
 coverage:
   # https://docs.codecov.com/docs/coverage-configuration

.github/workflows/docker.yml

@@ -24,7 +24,6 @@ jobs:
           - python
           - pypy
         python-version:
-          - '3.9'
           - '3.10'
           - '3.11'
         exclude:

.github/workflows/main.yml

@@ -20,17 +20,17 @@ jobs:
           import os
           import json
           full_matrix = {
-            'python': ['3.9', '3.10', '3.11'],
+            'python': ['3.10', '3.11'],
             # available OS's: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on
             'os': ['ubuntu-22.04', 'macos-12', 'windows-2022'],
             'include': [
               # XXX: tests fail on these, not sure why, when running them individually each on passes, but not on `make tests`
-              # {'os': 'ubuntu-22.04', 'python': 'pypy-3.9'},
+              # {'os': 'ubuntu-22.04', 'python': 'pypy-3.10'},
             ],
           }
           # this is the fastest one:
           reduced_matrix = {
-            'python': ['3.9'],
+            'python': ['3.11'],
             'os': ['ubuntu-22.04'],
           }
           github_repository = os.environ['GITHUB_REPOSITORY']
@@ -99,4 +99,4 @@ jobs:
         run: poetry run make tests
       - name: Upload coverage
         uses: codecov/codecov-action@v3
-        if: matrix.python == 3.9 && startsWith(matrix.os, 'ubuntu')
+        if: matrix.python == 3.11 && startsWith(matrix.os, 'ubuntu')

hathor/builder/builder.py

@@ -18,7 +18,7 @@
 from structlog import get_logger
 
 from hathor.checkpoint import Checkpoint
-from hathor.conf import HathorSettings
+from hathor.conf.get_settings import get_settings
 from hathor.conf.settings import HathorSettings as HathorSettingsType
 from hathor.consensus import ConsensusAlgorithm
 from hathor.event import EventManager
@@ -41,6 +41,7 @@
     TransactionStorage,
 )
 from hathor.util import Random, Reactor, get_environment_info
+from hathor.verification.verification_service import VerificationService, VertexVerifiers
 from hathor.wallet import BaseWallet, Wallet
 
 logger = get_logger()
@@ -101,6 +102,9 @@ def __init__(self) -> None:
         self._feature_service: Optional[FeatureService] = None
         self._bit_signaling_service: Optional[BitSignalingService] = None
 
+        self._vertex_verifiers: Optional[VertexVerifiers] = None
+        self._verification_service: Optional[VerificationService] = None
+
         self._rocksdb_path: Optional[str] = None
         self._rocksdb_storage: Optional[RocksDBStorage] = None
         self._rocksdb_cache_capacity: Optional[int] = None
@@ -157,6 +161,7 @@ def build(self) -> BuildArtifacts:
         tx_storage = self._get_or_create_tx_storage(indexes)
         feature_service = self._get_or_create_feature_service(tx_storage)
         bit_signaling_service = self._get_or_create_bit_signaling_service(tx_storage)
+        verification_service = self._get_or_create_verification_service()
 
         if self._enable_address_index:
             indexes.enable_address_index(pubsub)
@@ -177,6 +182,7 @@ def build(self) -> BuildArtifacts:
 
         manager = HathorManager(
             reactor,
+            settings=settings,
             network=self._network,
             pubsub=pubsub,
             consensus_algorithm=consensus_algorithm,
@@ -191,6 +197,7 @@ def build(self) -> BuildArtifacts:
             environment_info=get_environment_info(self._cmdline, peer_id.id),
             feature_service=feature_service,
             bit_signaling_service=bit_signaling_service,
+            verification_service=verification_service,
             **kwargs
         )
 
@@ -259,7 +266,7 @@ def set_peer_id(self, peer_id: PeerId) -> 'Builder':
 
     def _get_or_create_settings(self) -> HathorSettingsType:
         if self._settings is None:
-            self._settings = HathorSettings()
+            self._settings = get_settings()
         return self._settings
 
     def _get_reactor(self) -> Reactor:
@@ -424,6 +431,20 @@ def _get_or_create_bit_signaling_service(self, tx_storage: TransactionStorage) -
 
         return self._bit_signaling_service
 
+    def _get_or_create_verification_service(self) -> VerificationService:
+        if self._verification_service is None:
+            verifiers = self._get_or_create_vertex_verifiers()
+            self._verification_service = VerificationService(verifiers=verifiers)
+
+        return self._verification_service
+
+    def _get_or_create_vertex_verifiers(self) -> VertexVerifiers:
+        if self._vertex_verifiers is None:
+            settings = self._get_or_create_settings()
+            self._vertex_verifiers = VertexVerifiers.create_defaults(settings=settings)
+
+        return self._vertex_verifiers
+
     def use_memory(self) -> 'Builder':
         self.check_if_can_modify()
         self._storage_type = StorageType.MEMORY
@@ -516,6 +537,16 @@ def set_event_storage(self, event_storage: EventStorage) -> 'Builder':
         self._event_storage = event_storage
         return self
 
+    def set_verification_service(self, verification_service: VerificationService) -> 'Builder':
+        self.check_if_can_modify()
+        self._verification_service = verification_service
+        return self
+
+    def set_vertex_verifiers(self, vertex_verifiers: VertexVerifiers) -> 'Builder':
+        self.check_if_can_modify()
+        self._vertex_verifiers = vertex_verifiers
+        return self
+
     def set_reactor(self, reactor: Reactor) -> 'Builder':
         self.check_if_can_modify()
         self._reactor = reactor

hathor/builder/cli_builder.py

@@ -31,10 +31,11 @@
 from hathor.manager import HathorManager
 from hathor.p2p.manager import ConnectionsManager
 from hathor.p2p.peer_id import PeerId
-from hathor.p2p.utils import discover_hostname
+from hathor.p2p.utils import discover_hostname, get_genesis_short_hash
 from hathor.pubsub import PubSubManager
 from hathor.stratum import StratumFactory
 from hathor.util import Random, Reactor
+from hathor.verification.verification_service import VerificationService, VertexVerifiers
 from hathor.wallet import BaseWallet, HDWallet, Wallet
 
 logger = get_logger()
@@ -56,15 +57,13 @@ def check_or_raise(self, condition: bool, message: str) -> None:
 
     def create_manager(self, reactor: Reactor) -> HathorManager:
         import hathor
-        from hathor.conf import HathorSettings
-        from hathor.conf.get_settings import get_settings_source
+        from hathor.conf.get_settings import get_settings, get_settings_source
         from hathor.daa import TestMode, _set_test_mode
         from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage
         from hathor.event.websocket.factory import EventWebsocketFactory
         from hathor.p2p.netfilter.utils import add_peer_id_blacklist
         from hathor.p2p.peer_discovery import BootstrapPeerDiscovery, DNSPeerDiscovery
         from hathor.storage import RocksDBStorage
-        from hathor.transaction import genesis
         from hathor.transaction.storage import (
             TransactionCacheStorage,
             TransactionMemoryStorage,
@@ -73,7 +72,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
         )
         from hathor.util import get_environment_info
 
-        settings = HathorSettings()
+        settings = get_settings()
 
         # only used for logging its location
         settings_source = get_settings_source()
@@ -89,7 +88,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             'hathor-core v{hathor}',
             hathor=hathor.__version__,
             pid=os.getpid(),
-            genesis=genesis.GENESIS_HASH.hex()[:7],
+            genesis=get_genesis_short_hash(),
             my_peer_id=str(peer_id.id),
             python=python,
             platform=platform.platform(),
@@ -203,6 +202,9 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             not_support_features=self._args.signal_not_support
         )
 
+        vertex_verifiers = VertexVerifiers.create_defaults(settings=settings)
+        verification_service = VerificationService(verifiers=vertex_verifiers)
+
         p2p_manager = ConnectionsManager(
             reactor,
             network=network,
@@ -218,6 +220,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
 
         self.manager = HathorManager(
             reactor,
+            settings=settings,
             network=network,
             hostname=hostname,
             pubsub=pubsub,
@@ -232,7 +235,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             full_verification=full_verification,
             enable_event_queue=self._args.x_enable_event_queue,
             feature_service=self.feature_service,
-            bit_signaling_service=bit_signaling_service
+            bit_signaling_service=bit_signaling_service,
+            verification_service=verification_service,
         )
 
         p2p_manager.set_manager(self.manager)
@@ -259,10 +263,10 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             dns_hosts.extend(self._args.dns)
 
         if dns_hosts:
-            self.manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
+            p2p_manager.add_peer_discovery(DNSPeerDiscovery(dns_hosts))
 
         if self._args.bootstrap:
-            self.manager.add_peer_discovery(BootstrapPeerDiscovery(self._args.bootstrap))
+            p2p_manager.add_peer_discovery(BootstrapPeerDiscovery(self._args.bootstrap))
 
         if self._args.test_mode_tx_weight:
             _set_test_mode(TestMode.TEST_TX_WEIGHT)
@@ -278,7 +282,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             self.log.warn('--memory-indexes is implied for memory storage or JSON storage')
 
         for description in self._args.listen:
-            self.manager.add_listen_address(description)
+            p2p_manager.add_listen_address(description)
 
         if self._args.peer_id_blacklist:
             self.log.info('with peer id blacklist', blacklist=self._args.peer_id_blacklist)

hathor/builder/resources_builder.py

@@ -77,7 +77,7 @@ def create_prometheus(self) -> PrometheusMetricsExporter:
         return prometheus
 
     def create_resources(self) -> server.Site:
-        from hathor.conf import HathorSettings
+        from hathor.conf.get_settings import get_settings
         from hathor.debug_resources import (
             DebugCrashResource,
             DebugLogResource,
@@ -141,7 +141,7 @@ def create_resources(self) -> server.Site:
         )
         from hathor.websocket import HathorAdminWebsocketFactory, WebsocketStatsResource
 
-        settings = HathorSettings()
+        settings = get_settings()
         cpu = get_cpu_profiler()
 
         # TODO get this from a file. How should we do with the factory?

hathor/builder/sysctl_builder.py

@@ -18,6 +18,7 @@
 
 class SysctlBuilder:
     """Builder for the sysctl tree."""
+
     def __init__(self, artifacts: BuildArtifacts) -> None:
         self.artifacts = artifacts
 

hathor/cli/db_export.py

@@ -34,8 +34,8 @@ def register_signal_handlers(self) -> None:
 
     @classmethod
     def create_parser(cls) -> ArgumentParser:
-        from hathor.conf import HathorSettings
-        settings = HathorSettings()
+        from hathor.conf.get_settings import get_settings
+        settings = get_settings()
 
         def max_height(arg: str) -> Optional[int]:
             if arg.lower() == 'checkpoint':
@@ -80,8 +80,8 @@ def prepare(self, *, register_resources: bool = True) -> None:
         self.skip_voided = self._args.export_skip_voided
 
     def iter_tx(self) -> Iterator['BaseTransaction']:
-        from hathor.conf import HathorSettings
-        settings = HathorSettings()
+        from hathor.conf.get_settings import get_settings
+        settings = get_settings()
         soft_voided_ids = set(settings.SOFT_VOIDED_TX_IDS)
 
         for tx in self._iter_tx:

hathor/cli/events_simulator/event_forwarding_websocket_factory.py

@@ -0,0 +1,32 @@
+#  Copyright 2023 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Any
+
+from twisted.internet.interfaces import IAddress
+
+from hathor.cli.events_simulator.event_forwarding_websocket_protocol import EventForwardingWebsocketProtocol
+from hathor.event.websocket import EventWebsocketFactory
+from hathor.simulator import Simulator
+
+
+class EventForwardingWebsocketFactory(EventWebsocketFactory):
+    def __init__(self, simulator: Simulator, *args: Any, **kwargs: Any) -> None:
+        self._simulator = simulator
+        super().__init__(*args, **kwargs)
+
+    def buildProtocol(self, _: IAddress) -> EventForwardingWebsocketProtocol:
+        protocol = EventForwardingWebsocketProtocol(self._simulator)
+        protocol.factory = self
+        return protocol

hathor/cli/events_simulator/event_forwarding_websocket_protocol.py

@@ -0,0 +1,47 @@
+#  Copyright 2023 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from autobahn.websocket import ConnectionRequest
+
+from hathor.event.websocket import EventWebsocketProtocol
+from hathor.simulator import Simulator
+
+if TYPE_CHECKING:
+    from hathor.cli.events_simulator.event_forwarding_websocket_factory import EventForwardingWebsocketFactory
+
+
+class EventForwardingWebsocketProtocol(EventWebsocketProtocol):
+    factory: 'EventForwardingWebsocketFactory'
+
+    def __init__(self, simulator: Simulator) -> None:
+        self._simulator = simulator
+        super().__init__()
+
+    def onConnect(self, request: ConnectionRequest) -> None:
+        super().onConnect(request)
+        self._simulator.run(60)
+
+    def onOpen(self) -> None:
+        super().onOpen()
+        self._simulator.run(60)
+
+    def onClose(self, wasClean: bool, code: int, reason: str) -> None:
+        super().onClose(wasClean, code, reason)
+        self._simulator.run(60)
+
+    def onMessage(self, payload: bytes, isBinary: bool) -> None:
+        super().onMessage(payload, isBinary)
+        self._simulator.run(60)