Release v0.40.1

.github/workflows/docker.yml

@@ -23,8 +23,6 @@ jobs:
         - 3.8
         - 3.9
         include:
-        - python-impl: pypy
-          python-version: 3.6
         - python-impl: pypy
           python-version: 3.7
     steps:
@@ -146,9 +144,7 @@ jobs:
         # this key is setup such that every branch has its cache and new branches can reuse dev's cache, but not the other way around
         key: ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-${{ github.head_ref || github.ref }}-${{ github.sha }}
         restore-keys: |
-          ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-${{ github.head_ref || github.ref }}-
-          ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-refs/heads/dev-
-          ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-
+          ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-refs/heads/master-
     - name: Build and push
       uses: docker/build-push-action@v2
       id: docker_build

.github/workflows/main.yml

@@ -20,23 +20,21 @@ jobs:
         python:
         - 3.6
         - 3.7
+        - 3.8
+        - 3.9
         os:
         - ubuntu-latest
         - macos-latest
-        tier: [1]
         include:
-        - os: ubuntu-latest
-          python: 3.8
-          tier: 1
-        # - os: macos-latest
-        #   python: 3.8
-        #   tier: 1
         - os: windows-latest
           python: 3.6
-          tier: 2
         - os: windows-latest
           python: 3.7
-          tier: 2
+        # these still have some errors:
+        # - os: windows-latest
+        #   python: 3.8
+        # - os: windows-latest
+        #   python: 3.9
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -73,7 +71,6 @@ jobs:
       run: poetry run make check
     - name: Run tests
       run: poetry run make tests
-      continue-on-error: ${{ matrix.tier > 1 }}
     - name: Upload coverage
       uses: codecov/codecov-action@v1
-      if: matrix.python == 3.6 && matrix.os == 'ubuntu-latest'
+      if: matrix.python == 3.7 && matrix.os == 'ubuntu-latest'

Makefile

@@ -67,7 +67,7 @@ check-version:
 	bash ./extras/check_version.sh
 
 .PHONY: check
-check: flake8 isort-check mypy check-version
+check: check-version flake8 isort-check mypy
 
 # formatting:
 

extras/check_version.sh

@@ -25,7 +25,4 @@ if [[ x${PACKAGE_VERSION}x != x${OPENAPI_VERSION}x ]]; then
 	EXITCODE=-1
 fi
 
-if [[ ${EXITCODE} == 0 ]]; then
-	echo OK
-fi
 exit $EXITCODE

extras/nginx_docker/Makefile

@@ -7,9 +7,10 @@ tag = 769498303037.dkr.ecr.us-east-1.amazonaws.com/webtank:latest
 docker: nginx.conf set_real_ip_from_cloudfront
 	docker buildx build --pull --push --platform linux/arm64/v8,linux/amd64 --tag $(tag) .
 
-
+nginx.conf: export PYTHONPATH := ../..
 nginx.conf:
-	python -m hathor generate_nginx_config nginx.conf
+	@python -c "import os; import hathor; print('Using hathor-core from:', os.path.dirname(hathor.__file__))"
+	python -m hathor generate_nginx_config - > $@
 
 set_real_ip_from_cloudfront:
 	curl https://ip-ranges.amazonaws.com/ip-ranges.json -s \

hathor/cli/main.py

@@ -139,7 +139,9 @@ def execute_from_command_line(self):
         if json_logs:
             sys.argv.remove('--json-logs')
 
-        setup_logging(debug, capture_stdout, json_logs)
+        sentry = '--sentry-dsn' in sys.argv
+
+        setup_logging(debug, capture_stdout, json_logs, sentry=sentry)
         module.main()
 
 

hathor/cli/openapi_files/openapi_base.json

@@ -7,7 +7,7 @@
         ],
 	"info": {
             "title": "Hathor API",
-            "version": "0.40.0"
+            "version": "0.40.1"
 	},
 	"consumes": [
             "application/json"

hathor/cli/openapi_files/register.py

@@ -30,13 +30,13 @@ def register_resource(resource_class: Resource) -> Resource:
 def get_registered_resources() -> List[Resource]:
     """ Returns a list with all the resources registered for the docs
     """
-    from hathor.p2p.resources import __all__  # noqa: 401
-    from hathor.resources import ProfilerResource  # noqa: 401
-    from hathor.stratum.resources import MiningStatsResource  # noqa: 401
-    from hathor.transaction.resources import __all__  # noqa: 401
-    from hathor.version_resource import VersionResource  # noqa: 401
-    from hathor.wallet.resources.nano_contracts import __all__  # noqa: 401
-    from hathor.wallet.resources.thin_wallet import __all__  # noqa: 401
-    from hathor.websocket import WebsocketStatsResource  # noqa: 401
+    import hathor.p2p.resources  # noqa: 401
+    import hathor.profiler.resources  # noqa: 401
+    import hathor.stratum.resources  # noqa: 401
+    import hathor.transaction.resources  # noqa: 401
+    import hathor.version_resource  # noqa: 401
+    import hathor.wallet.resources.nano_contracts  # noqa: 401
+    import hathor.wallet.resources.thin_wallet  # noqa: 401
+    import hathor.websocket  # noqa: 401
     global _registered_resources
     return _registered_resources

hathor/cli/run_node.py

@@ -298,6 +298,7 @@ def register_resources(self, args: Namespace) -> None:
             GetBlockTemplateResource,
             GraphvizFullResource,
             GraphvizNeighboursResource,
+            MempoolResource,
             PushTxResource,
             SubmitBlockResource,
             TransactionAccWeightResource,
@@ -380,6 +381,7 @@ def register_resources(self, args: Namespace) -> None:
                 (b'dashboard_tx', DashboardTransactionResource(self.manager), root),
                 (b'profiler', ProfilerResource(self.manager), root),
                 (b'top', CPUProfilerResource(self.manager, cpu), root),
+                (b'mempool', MempoolResource(self.manager), root),
                 # mining
                 (b'mining', MiningResource(self.manager), root),
                 (b'getmininginfo', MiningInfoResource(self.manager), root),

hathor/cli/top.py

@@ -13,8 +13,6 @@
 # limitations under the License.
 
 import asyncio
-import curses
-import curses.ascii
 import datetime
 import signal
 import sys
@@ -26,6 +24,12 @@
 from math import floor
 from typing import Any, Callable, DefaultDict, Dict, List, Optional, Tuple
 
+try:
+    import curses
+    import curses.ascii
+except ModuleNotFoundError:
+    pass
+
 from aiohttp import ClientSession
 
 Key = Tuple[str, ...]

hathor/cli/util.py

@@ -30,6 +30,7 @@ def setup_logging(
             capture_stdout: bool = False,
             json_logging: bool = False,
             *,
+            sentry: bool = False,
             _test_logging: bool = False,
         ) -> None:
     import logging
@@ -219,11 +220,8 @@ def kwargs_formatter(_, __, event_dict):
         structlog.stdlib.add_log_level,
     ]
 
-    try:
+    if sentry:
         from structlog_sentry import SentryProcessor
-    except ModuleNotFoundError:
-        pass
-    else:
         processors.append(SentryProcessor(level=logging.ERROR))
 
     processors.extend([

hathor/conf/settings.py

@@ -292,6 +292,9 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int:
     # As a normal tx has ~2-4 inputs and 2 outputs, I would say the maximum should be 150*6 = 900 elements
     MAX_INPUTS_OUTPUTS_ADDRESS_HISTORY: int = 6*MAX_TX_ADDRESSES_HISTORY
 
+    # Maximum number of TXs that will be sent by the Mempool API.
+    MEMPOOL_API_TX_LIMIT: int = 100
+
     # Multiplier coefficient to adjust the minimum weight of a normal tx to 18
     MIN_TX_WEIGHT_COEFFICIENT: float = 1.6
 

hathor/consensus.py

@@ -604,7 +604,7 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
 
         spent_tx = tx.storage.get_transaction(txin.tx_id)
         spent_meta = spent_tx.get_metadata()
-        spent_by = spent_meta.spent_outputs[txin.index]  # Set[bytes(hash)]
+        spent_by = spent_meta.spent_outputs[txin.index]
         assert tx.hash not in spent_by
 
         # Update our meta.conflict_with.
@@ -616,7 +616,7 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
             else:
                 meta.voided_by.add(tx.hash)
             if meta.conflict_with:
-                meta.conflict_with.extend(spent_by)
+                meta.conflict_with.extend(set(spent_by) - set(meta.conflict_with))
             else:
                 meta.conflict_with = spent_by.copy()
         tx.storage.save_transaction(tx, only_metadata=True)
@@ -626,7 +626,9 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
             conflict_tx = tx.storage.get_transaction(h)
             tx_meta = conflict_tx.get_metadata()
             if tx_meta.conflict_with:
-                tx_meta.conflict_with.append(tx.hash)
+                if tx.hash not in tx_meta.conflict_with:
+                    # We could use a set instead of a list but it consumes ~2.15 times more of memory.
+                    tx_meta.conflict_with.append(tx.hash)
             else:
                 tx_meta.conflict_with = [tx.hash]
             tx.storage.save_transaction(conflict_tx, only_metadata=True)

hathor/daa.py

@@ -41,6 +41,8 @@
 
 
 class TestMode(IntFlag):
+    __test__ = False
+
     DISABLED = 0
     TEST_TX_WEIGHT = 1
     TEST_BLOCK_WEIGHT = 2

hathor/manager.py

@@ -295,11 +295,11 @@ def do_discovery(self) -> None:
         for peer_discovery in self.peer_discoveries:
             peer_discovery.discover_and_connect(self.connections.connect_to)
 
-    def start_profiler(self) -> None:
+    def start_profiler(self, *, reset: bool = False) -> None:
         """
         Start profiler. It can be activated from a web resource, as well.
         """
-        if not self.profiler:
+        if reset or not self.profiler:
             import cProfile
             self.profiler = cProfile.Profile()
         self.profiler.enable()
@@ -408,6 +408,8 @@ def _initialize_components(self) -> None:
                     elif tx.is_block:
                         self.tx_storage.add_to_parent_blocks_index(tx.hash)
                     self.tx_storage.add_to_indexes(tx)
+                    if tx.is_transaction and tx_meta.voided_by:
+                        self.tx_storage.del_from_indexes(tx)
             except (InvalidNewTransaction, TxValidationError):
                 self.log.error('unexpected error when initializing', tx=tx, exc_info=True)
                 raise

hathor/profiler/resources/profiler.py

@@ -60,7 +60,10 @@ def render_POST(self, request):
         ret = {'success': True}
 
         if 'start' in post_data:
-            self.manager.start_profiler()
+            reset = False
+            if 'reset' in post_data:
+                reset = True
+            self.manager.start_profiler(reset=reset)
 
         elif 'stop' in post_data:
             if 'filepath' in post_data:
@@ -101,6 +104,13 @@ def render_OPTIONS(self, request: Request) -> int:
                                     'start': True
                                 }
                             },
+                            'start-reset': {
+                                'summary': 'Start profiler',
+                                'value': {
+                                    'start': True,
+                                    'reset': True
+                                }
+                            },
                             'stop': {
                                 'summary': 'Stop profiler',
                                 'value': {

hathor/transaction/resources/__init__.py

@@ -17,6 +17,7 @@
 from hathor.transaction.resources.dashboard import DashboardTransactionResource
 from hathor.transaction.resources.decode_tx import DecodeTxResource
 from hathor.transaction.resources.graphviz import GraphvizFullResource, GraphvizNeighboursResource
+from hathor.transaction.resources.mempool import MempoolResource
 from hathor.transaction.resources.mining import GetBlockTemplateResource, SubmitBlockResource
 from hathor.transaction.resources.push_tx import PushTxResource
 from hathor.transaction.resources.transaction import TransactionResource
@@ -38,4 +39,5 @@
     'DashboardTransactionResource',
     'TxParentsResource',
     'ValidateAddressResource',
+    'MempoolResource',
 ]