Release v0.40.0

codecov.yml → .codecov.yml

@@ -1,11 +1,19 @@
 codecov:
+  branch: dev
+
+coverage:
+  # https://docs.codecov.com/docs/coverage-configuration
+  range: "80...90"
   # https://docs.codecov.io/docs/commit-status
   status:
+    # TODO: re-enable patch in the future
+    patch: off
     project:
       default:
         # minimum coverage ratio that the commit must meet to be considered a success
-        target: 82%
+        target: 83%
         if_ci_failed: error
         only_pulls: true
+
 github_checks:
-  annotations: false
+  annotations: true

Makefile

@@ -34,7 +34,7 @@ tests-doctests:
 
 .PHONY: tests-lib
 tests-lib:
-	pytest --durations=10 $(pytest_flags) --doctest-modules hathor --cov-fail-under=83 $(tests_lib)
+	pytest --durations=10 $(pytest_flags) --doctest-modules hathor $(tests_lib)
 
 .PHONY: tests-genesis
 tests-genesis:
@@ -46,7 +46,7 @@ tests: tests-cli tests-lib tests-genesis
 
 .PHONY: tests-full
 tests-full:
-	pytest $(pytest_flags) --durations=10 --cov-fail-under=90 --cov-config=.coveragerc_full ./tests
+	pytest $(pytest_flags) --durations=10 --cov-config=.coveragerc_full ./tests
 
 # checking:
 
@@ -82,7 +82,9 @@ isort:
 
 proto_dir = ./hathor/protos
 proto_srcs = $(wildcard $(proto_dir)/*.proto)
-proto_outputs = $(patsubst %.proto,%_pb2.py,$(proto_srcs)) $(patsubst %.proto,%_pb2_grpc.py,$(proto_srcs)) $(patsubst %.proto,%_pb2.pyi,$(proto_srcs))
+proto_srcs_old = $(proto_dir)/transaction_storage.proto
+proto_outputs = $(patsubst %.proto,%_pb2.py,$(proto_srcs)) $(patsubst %.proto,%_pb2.pyi,$(proto_srcs))
+proto_outputs_old = $(patsubst %.proto,%_pb2_grpc.py,$(proto_srcs) $(proto_srcs_old))
 GRPC_TOOLS_VERSION = "$(shell python -m grpc_tools.protoc --version 2>/dev/null || true)"
 #ifdef GRPC_TOOLS_VERSION
 ifneq ($(GRPC_TOOLS_VERSION),"")
@@ -94,8 +96,6 @@ endif
 # all proto_srcs are added as deps so when a change on any of them triggers all to be rebuilt
 %_pb2.pyi %_pb2.py: %.proto $(proto_srcs)
 	$(protoc) -I. --python_out=. --mypy_out=. $<
-%_pb2_grpc.py: %.proto $(proto_srcs)
-	$(protoc) -I. --grpc_python_out=. $< || true
 
 .PHONY: protos
 protos: $(proto_outputs)
@@ -104,7 +104,7 @@ protos: $(proto_outputs)
 
 .PHONY: clean-protos
 clean-protos:
-	rm -f $(proto_outputs)
+	rm -f $(proto_outputs) $(proto_outputs_old)
 
 .PHONY: clean-pyc
 clean-pyc:

hathor/checkpoint.py

@@ -0,0 +1,20 @@
+# Copyright 2021 Hathor Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import NamedTuple
+
+
+class Checkpoint(NamedTuple):
+    height: int
+    hash: bytes

hathor/cli/main.py

@@ -135,7 +135,11 @@ def execute_from_command_line(self):
             pudb.set_trace(paused=False)
             capture_stdout = False
 
-        setup_logging(debug, capture_stdout)
+        json_logs = '--json-logs' in sys.argv
+        if json_logs:
+            sys.argv.remove('--json-logs')
+
+        setup_logging(debug, capture_stdout, json_logs)
         module.main()
 
 

hathor/cli/run_node.py

@@ -47,7 +47,13 @@ def create_parser(self) -> ArgumentParser:
         parser.add_argument('--status', type=int, help='Port to run status server')
         parser.add_argument('--stratum', type=int, help='Port to run stratum server')
         parser.add_argument('--data', help='Data directory')
-        parser.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend')
+        storage = parser.add_mutually_exclusive_group()
+        storage.add_argument('--rocksdb-storage', action='store_true', help='Use RocksDB storage backend (default)')
+        storage.add_argument('--old-rocksdb-storage', action='store_true',
+                             help='Use old RocksDB storage backend (deprecated)')
+        storage.add_argument('--memory-storage', action='store_true', help='Do not use any storage')
+        storage.add_argument('--json-storage', action='store_true', help='Use legacy JSON storage (not recommended)')
+        parser.add_argument('--rocksdb-cache', type=int, help='RocksDB block-table cache size (bytes)', default=None)
         parser.add_argument('--wallet', help='Set wallet type. Options are hd (Hierarchical Deterministic) or keypair',
                             default=None)
         parser.add_argument('--wallet-enable-api', action='store_true',
@@ -69,10 +75,16 @@ def create_parser(self) -> ArgumentParser:
                             'This is still a beta feature as it may cause issues when restarting the full node '
                             'after a crash.')
         parser.add_argument('--procname-prefix', help='Add a prefix to the process name', default='')
+        parser.add_argument('--allow-non-standard-script', action='store_true', help='Accept non-standard scripts on '
+                            '/push-tx API')
+        parser.add_argument('--max-output-script-size', type=int, default=None, help='Custom max accepted script size '
+                            'on /push-tx API')
+        parser.add_argument('--sentry-dsn', help='Sentry DSN')
         return parser
 
     def prepare(self, args: Namespace) -> None:
         import hathor
+        from hathor.cli.util import check_or_exit
         from hathor.conf import HathorSettings
         from hathor.daa import TestMode, _set_test_mode
         from hathor.manager import HathorManager
@@ -84,6 +96,7 @@ def prepare(self, args: Namespace) -> None:
             TransactionCacheStorage,
             TransactionCompactStorage,
             TransactionMemoryStorage,
+            TransactionOldRocksDBStorage,
             TransactionRocksDBStorage,
             TransactionStorage,
         )
@@ -160,24 +173,34 @@ def create_wallet():
                 raise ValueError('Invalid type for wallet')
 
         tx_storage: TransactionStorage
-        if args.data:
-            if args.rocksdb_storage:
-                tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache))
-            else:
-                tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache))
-            self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data)
-            if args.cache:
-                tx_storage = TransactionCacheStorage(tx_storage, reactor)
-                if args.cache_size:
-                    tx_storage.capacity = args.cache_size
-                if args.cache_interval:
-                    tx_storage.interval = args.cache_interval
-                self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval)
-                tx_storage.start()
-        else:
+        if args.memory_storage:
+            check_or_exit(not args.data, '--data should not be used with --memory-storage')
             # if using MemoryStorage, no need to have cache
             tx_storage = TransactionMemoryStorage()
             self.log.info('with storage', storage_class=type(tx_storage).__name__)
+        elif args.json_storage:
+            check_or_exit(args.data, '--data is expected')
+            tx_storage = TransactionCompactStorage(path=args.data, with_index=(not args.cache))
+        elif args.old_rocksdb_storage:
+            check_or_exit(args.data, '--data is expected')
+            self.log.warn('the old rocksdb storage is deprecated and support will be removed')
+            tx_storage = TransactionOldRocksDBStorage(path=args.data)
+        else:
+            check_or_exit(args.data, '--data is expected')
+            if args.rocksdb_storage:
+                self.log.warn('--rocksdb-storage is now implied, no need to specify it')
+            cache_capacity = args.rocksdb_cache
+            tx_storage = TransactionRocksDBStorage(path=args.data, with_index=(not args.cache),
+                                                   cache_capacity=cache_capacity)
+        self.log.info('with storage', storage_class=type(tx_storage).__name__, path=args.data)
+        if args.cache:
+            check_or_exit(not args.memory_storage, '--cache should not be used with --memory-storage')
+            tx_storage = TransactionCacheStorage(tx_storage, reactor)
+            if args.cache_size:
+                tx_storage.capacity = args.cache_size
+            if args.cache_interval:
+                tx_storage.interval = args.cache_interval
+            self.log.info('with cache', capacity=tx_storage.capacity, interval=tx_storage.interval)
         self.tx_storage = tx_storage
 
         if args.wallet:
@@ -204,7 +227,7 @@ def create_wallet():
         network = settings.NETWORK_NAME
         self.manager = HathorManager(reactor, peer_id=peer_id, network=network, hostname=hostname,
                                      tx_storage=self.tx_storage, wallet=self.wallet, wallet_index=args.wallet_index,
-                                     stratum_port=args.stratum, ssl=True)
+                                     stratum_port=args.stratum, ssl=True, checkpoints=settings.CHECKPOINTS)
         if args.allow_mining_without_peers:
             self.manager.allow_mining_without_peers()
 
@@ -232,10 +255,32 @@ def create_wallet():
         for description in args.listen:
             self.manager.add_listen_address(description)
 
-        self.start_manager()
+        self.start_manager(args)
         self.register_resources(args)
 
-    def start_manager(self) -> None:
+    def start_sentry_if_possible(self, args: Namespace) -> None:
+        """Start Sentry integration if possible."""
+        if not args.sentry_dsn:
+            return
+        self.log.info('Starting Sentry', dsn=args.sentry_dsn)
+        try:
+            import sentry_sdk
+            from structlog_sentry import SentryProcessor  # noqa: F401
+        except ModuleNotFoundError:
+            self.log.error('Please use `poetry install -E sentry` for enabling Sentry.')
+            sys.exit(-3)
+
+        import hathor
+        from hathor.conf import HathorSettings
+        settings = HathorSettings()
+        sentry_sdk.init(
+            dsn=args.sentry_dsn,
+            release=hathor.__version__,
+            environment=settings.NETWORK_NAME,
+        )
+
+    def start_manager(self, args: Namespace) -> None:
+        self.start_sentry_if_possible(args)
         self.manager.start()
 
     def register_resources(self, args: Namespace) -> None:
@@ -246,6 +291,7 @@ def register_resources(self, args: Namespace) -> None:
         from hathor.profiler.resources import CPUProfilerResource, ProfilerResource
         from hathor.prometheus import PrometheusMetricsExporter
         from hathor.transaction.resources import (
+            BlockAtHeightResource,
             CreateTxResource,
             DashboardTransactionResource,
             DecodeTxResource,
@@ -254,8 +300,6 @@ def register_resources(self, args: Namespace) -> None:
             GraphvizNeighboursResource,
             PushTxResource,
             SubmitBlockResource,
-            TipsHistogramResource,
-            TipsResource,
             TransactionAccWeightResource,
             TransactionResource,
             TxParentsResource,
@@ -326,11 +370,12 @@ def register_resources(self, args: Namespace) -> None:
                 (b'create_tx', CreateTxResource(self.manager), root),
                 (b'decode_tx', DecodeTxResource(self.manager), root),
                 (b'validate_address', ValidateAddressResource(self.manager), root),
-                (b'push_tx', PushTxResource(self.manager), root),
+                (b'push_tx',
+                    PushTxResource(self.manager, args.max_output_script_size, args.allow_non_standard_script),
+                    root),
                 (b'graphviz', graphviz, root),
-                (b'tips-histogram', TipsHistogramResource(self.manager), root),
-                (b'tips', TipsResource(self.manager), root),
                 (b'transaction', TransactionResource(self.manager), root),
+                (b'block_at_height', BlockAtHeightResource(self.manager), root),
                 (b'transaction_acc_weight', TransactionAccWeightResource(self.manager), root),
                 (b'dashboard_tx', DashboardTransactionResource(self.manager), root),
                 (b'profiler', ProfilerResource(self.manager), root),

hathor/cli/shell.py

@@ -28,7 +28,7 @@ def run_ipython():
 
 
 class Shell(RunNode):
-    def start_manager(self) -> None:
+    def start_manager(self, args: Namespace) -> None:
         pass
 
     def register_signal_handlers(self, args: Namespace) -> None: