Release v0.40.1

extras/nginx_docker/Makefile

@@ -7,9 +7,10 @@ tag = 769498303037.dkr.ecr.us-east-1.amazonaws.com/webtank:latest
 docker: nginx.conf set_real_ip_from_cloudfront
 	docker buildx build --pull --push --platform linux/arm64/v8,linux/amd64 --tag $(tag) .
 
-
+nginx.conf: export PYTHONPATH := ../..
 nginx.conf:
-	python -m hathor generate_nginx_config nginx.conf
+	@python -c "import os; import hathor; print('Using hathor-core from:', os.path.dirname(hathor.__file__))"
+	python -m hathor generate_nginx_config - > $@
 
 set_real_ip_from_cloudfront:
 	curl https://ip-ranges.amazonaws.com/ip-ranges.json -s \

hathor/cli/main.py

@@ -139,7 +139,9 @@ def execute_from_command_line(self):
         if json_logs:
             sys.argv.remove('--json-logs')
 
-        setup_logging(debug, capture_stdout, json_logs)
+        sentry = '--sentry-dsn' in sys.argv
+
+        setup_logging(debug, capture_stdout, json_logs, sentry=sentry)
         module.main()
 
 

hathor/cli/openapi_files/register.py

@@ -30,13 +30,13 @@ def register_resource(resource_class: Resource) -> Resource:
 def get_registered_resources() -> List[Resource]:
     """ Returns a list with all the resources registered for the docs
     """
-    from hathor.p2p.resources import __all__  # noqa: 401
-    from hathor.resources import ProfilerResource  # noqa: 401
-    from hathor.stratum.resources import MiningStatsResource  # noqa: 401
-    from hathor.transaction.resources import __all__  # noqa: 401
-    from hathor.version_resource import VersionResource  # noqa: 401
-    from hathor.wallet.resources.nano_contracts import __all__  # noqa: 401
-    from hathor.wallet.resources.thin_wallet import __all__  # noqa: 401
-    from hathor.websocket import WebsocketStatsResource  # noqa: 401
+    import hathor.p2p.resources  # noqa: 401
+    import hathor.profiler.resources  # noqa: 401
+    import hathor.stratum.resources  # noqa: 401
+    import hathor.transaction.resources  # noqa: 401
+    import hathor.version_resource  # noqa: 401
+    import hathor.wallet.resources.nano_contracts  # noqa: 401
+    import hathor.wallet.resources.thin_wallet  # noqa: 401
+    import hathor.websocket  # noqa: 401
     global _registered_resources
     return _registered_resources

hathor/cli/run_node.py

@@ -298,6 +298,7 @@ def register_resources(self, args: Namespace) -> None:
             GetBlockTemplateResource,
             GraphvizFullResource,
             GraphvizNeighboursResource,
+            MempoolResource,
             PushTxResource,
             SubmitBlockResource,
             TransactionAccWeightResource,
@@ -380,6 +381,7 @@ def register_resources(self, args: Namespace) -> None:
                 (b'dashboard_tx', DashboardTransactionResource(self.manager), root),
                 (b'profiler', ProfilerResource(self.manager), root),
                 (b'top', CPUProfilerResource(self.manager, cpu), root),
+                (b'mempool', MempoolResource(self.manager), root),
                 # mining
                 (b'mining', MiningResource(self.manager), root),
                 (b'getmininginfo', MiningInfoResource(self.manager), root),

hathor/cli/util.py

@@ -30,6 +30,7 @@ def setup_logging(
             capture_stdout: bool = False,
             json_logging: bool = False,
             *,
+            sentry: bool = False,
             _test_logging: bool = False,
         ) -> None:
     import logging
@@ -219,11 +220,8 @@ def kwargs_formatter(_, __, event_dict):
         structlog.stdlib.add_log_level,
     ]
 
-    try:
+    if sentry:
         from structlog_sentry import SentryProcessor
-    except ModuleNotFoundError:
-        pass
-    else:
         processors.append(SentryProcessor(level=logging.ERROR))
 
     processors.extend([

hathor/conf/settings.py

@@ -292,6 +292,9 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int:
     # As a normal tx has ~2-4 inputs and 2 outputs, I would say the maximum should be 150*6 = 900 elements
     MAX_INPUTS_OUTPUTS_ADDRESS_HISTORY: int = 6*MAX_TX_ADDRESSES_HISTORY
 
+    # Maximum number of TXs that will be sent by the Mempool API.
+    MEMPOOL_API_TX_LIMIT: int = 100
+
     # Multiplier coefficient to adjust the minimum weight of a normal tx to 18
     MIN_TX_WEIGHT_COEFFICIENT: float = 1.6
 

hathor/consensus.py

@@ -604,7 +604,7 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
 
         spent_tx = tx.storage.get_transaction(txin.tx_id)
         spent_meta = spent_tx.get_metadata()
-        spent_by = spent_meta.spent_outputs[txin.index]  # Set[bytes(hash)]
+        spent_by = spent_meta.spent_outputs[txin.index]
         assert tx.hash not in spent_by
 
         # Update our meta.conflict_with.
@@ -616,7 +616,7 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
             else:
                 meta.voided_by.add(tx.hash)
             if meta.conflict_with:
-                meta.conflict_with.extend(spent_by)
+                meta.conflict_with.extend(set(spent_by) - set(meta.conflict_with))
             else:
                 meta.conflict_with = spent_by.copy()
         tx.storage.save_transaction(tx, only_metadata=True)
@@ -626,7 +626,9 @@ def mark_input_as_used(self, tx: Transaction, txin: TxInput) -> None:
             conflict_tx = tx.storage.get_transaction(h)
             tx_meta = conflict_tx.get_metadata()
             if tx_meta.conflict_with:
-                tx_meta.conflict_with.append(tx.hash)
+                if tx.hash not in tx_meta.conflict_with:
+                    # We could use a set instead of a list but it consumes ~2.15 times more of memory.
+                    tx_meta.conflict_with.append(tx.hash)
             else:
                 tx_meta.conflict_with = [tx.hash]
             tx.storage.save_transaction(conflict_tx, only_metadata=True)

hathor/daa.py

@@ -41,6 +41,8 @@
 
 
 class TestMode(IntFlag):
+    __test__ = False
+
     DISABLED = 0
     TEST_TX_WEIGHT = 1
     TEST_BLOCK_WEIGHT = 2

hathor/manager.py

@@ -295,11 +295,11 @@ def do_discovery(self) -> None:
         for peer_discovery in self.peer_discoveries:
             peer_discovery.discover_and_connect(self.connections.connect_to)
 
-    def start_profiler(self) -> None:
+    def start_profiler(self, *, reset: bool = False) -> None:
         """
         Start profiler. It can be activated from a web resource, as well.
         """
-        if not self.profiler:
+        if reset or not self.profiler:
             import cProfile
             self.profiler = cProfile.Profile()
         self.profiler.enable()
@@ -408,6 +408,8 @@ def _initialize_components(self) -> None:
                     elif tx.is_block:
                         self.tx_storage.add_to_parent_blocks_index(tx.hash)
                     self.tx_storage.add_to_indexes(tx)
+                    if tx.is_transaction and tx_meta.voided_by:
+                        self.tx_storage.del_from_indexes(tx)
             except (InvalidNewTransaction, TxValidationError):
                 self.log.error('unexpected error when initializing', tx=tx, exc_info=True)
                 raise

hathor/profiler/resources/profiler.py

@@ -60,7 +60,10 @@ def render_POST(self, request):
         ret = {'success': True}
 
         if 'start' in post_data:
-            self.manager.start_profiler()
+            reset = False
+            if 'reset' in post_data:
+                reset = True
+            self.manager.start_profiler(reset=reset)
 
         elif 'stop' in post_data:
             if 'filepath' in post_data:
@@ -101,6 +104,13 @@ def render_OPTIONS(self, request: Request) -> int:
                                     'start': True
                                 }
                             },
+                            'start-reset': {
+                                'summary': 'Start profiler',
+                                'value': {
+                                    'start': True,
+                                    'reset': True
+                                }
+                            },
                             'stop': {
                                 'summary': 'Stop profiler',
                                 'value': {

hathor/transaction/resources/__init__.py

@@ -17,6 +17,7 @@
 from hathor.transaction.resources.dashboard import DashboardTransactionResource
 from hathor.transaction.resources.decode_tx import DecodeTxResource
 from hathor.transaction.resources.graphviz import GraphvizFullResource, GraphvizNeighboursResource
+from hathor.transaction.resources.mempool import MempoolResource
 from hathor.transaction.resources.mining import GetBlockTemplateResource, SubmitBlockResource
 from hathor.transaction.resources.push_tx import PushTxResource
 from hathor.transaction.resources.transaction import TransactionResource
@@ -38,4 +39,5 @@
     'DashboardTransactionResource',
     'TxParentsResource',
     'ValidateAddressResource',
+    'MempoolResource',
 ]

hathor/transaction/resources/mempool.py

@@ -0,0 +1,113 @@
+# Copyright 2021 Hathor Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from itertools import islice
+from typing import TYPE_CHECKING
+
+from twisted.web import resource
+
+from hathor.api_util import set_cors
+from hathor.cli.openapi_files.register import register_resource
+from hathor.conf import HathorSettings
+
+settings = HathorSettings()
+
+if TYPE_CHECKING:
+    from twisted.web.http import Request
+
+    from hathor.manager import HathorManager
+
+
+@register_resource
+class MempoolResource(resource.Resource):
+    """ Implements a web server API to return transactions on the mempool.
+
+    You must run with option `--status <PORT>`.
+    """
+    isLeaf = True
+
+    def __init__(self, manager: 'HathorManager'):
+        # Important to have the manager so we can know the tx_storage
+        self.manager = manager
+
+    def render_GET(self, request: 'Request') -> bytes:
+        """ Get request /mempool/ that returns transactions on the mempool
+
+            :rtype: string (json list)
+        """
+        request.setHeader(b'content-type', b'application/json; charset=utf-8')
+        set_cors(request, 'GET')
+
+        # Get a list of all txs on the mempool
+        tx_ids = map(
+            # get only tx_ids
+            lambda tx: tx.hash_hex,
+            # order by timestamp
+            sorted(list(self.manager.tx_storage.iter_mempool()), key=lambda tx: tx.timestamp),
+        )
+        # Only return up to settings.MEMPOOL_API_TX_LIMIT txs per call (default: 100)
+        data = {'success': True, 'transactions': list(islice(tx_ids, settings.MEMPOOL_API_TX_LIMIT))}
+        return json.dumps(data, indent=4).encode('utf-8')
+
+
+MempoolResource.openapi = {
+    '/mempool': {
+        'x-visibility': 'public',
+        'x-rate-limit': {
+            'global': [
+                {
+                    'rate': '50r/s',
+                    'burst': 100,
+                    'delay': 50
+                }
+            ],
+            'per-ip': [
+                {
+                    'rate': '1r/s',
+                    'burst': 5,
+                    'delay': 3
+                }
+            ]
+        },
+        'get': {
+            'tags': ['mempool'],
+            'operationId': 'mempool',
+            'summary': 'List of mempool transactions',
+            'description': 'Returns a list of all transactions currently on the mempool',
+            'responses': {
+                '200': {
+                    'description': 'Success',
+                    'content': {
+                        'application/json': {
+                            'examples': {
+                                'success': {
+                                    'summary': 'Success',
+                                    'value': {
+                                        'success': True,
+                                        'transactions': [
+                                            '339f47da87435842b0b1b528ecd9eac2495ce983b3e9c923a37e1befbe12c792',
+                                            '16ba3dbe424c443e571b00840ca54b9ff4cff467e10b6a15536e718e2008f952',
+                                            '33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e869',
+                                        ],
+                                    },
+                                },
+                            },
+                        },
+                    }
+                }
+            }
+        }
+    }
+}

hathor/transaction/storage/transaction_storage.py

@@ -313,9 +313,10 @@ def iter_mempool(self) -> Iterator[Transaction]:
         bfs = BFSWalk(self, is_dag_verifications=True, is_left_to_right=False)
         for tx in bfs.run(map(self.get_transaction, self._tx_tips_index), skip_root=False):
             assert isinstance(tx, Transaction)
-            yield tx
             if tx.get_metadata().first_block is not None:
                 bfs.skip_neighbors(tx)
+            else:
+                yield tx
 
     def update_tx_tips(self, tx: BaseTransaction) -> None:
         """

hathor/transaction/transaction_metadata.py

@@ -260,7 +260,7 @@ def to_json(self) -> Dict[str, Any]:
             data['spent_outputs'].append([idx, [h_bytes.hex() for h_bytes in hashes]])
         data['received_by'] = list(self.received_by)
         data['children'] = [x.hex() for x in self.children]
-        data['conflict_with'] = [x.hex() for x in self.conflict_with] if self.conflict_with else []
+        data['conflict_with'] = [x.hex() for x in set(self.conflict_with)] if self.conflict_with else []
         data['voided_by'] = [x.hex() for x in self.voided_by] if self.voided_by else []
         data['twins'] = [x.hex() for x in self.twins]
         data['accumulated_weight'] = self.accumulated_weight
@@ -285,8 +285,8 @@ def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata':
         meta.received_by = list(data['received_by'])
         meta.children = [bytes.fromhex(h) for h in data['children']]
 
-        if 'conflict_with' in data:
-            meta.conflict_with = [bytes.fromhex(h) for h in data['conflict_with']] if data['conflict_with'] else None
+        if 'conflict_with' in data and data['conflict_with']:
+            meta.conflict_with = [bytes.fromhex(h) for h in set(data['conflict_with'])]
         else:
             meta.conflict_with = None
 

hathor/websocket/factory.py

@@ -196,6 +196,8 @@ def serialize_message_data(self, event, args):
             tx = data['tx']
             data = tx.to_json_extended()
             data['is_block'] = tx.is_block
+            # needed to check if the transaction is from the mempool
+            data['first_block'] = tx.get_metadata().first_block
             return data
         elif event == HathorEvents.WALLET_BALANCE_UPDATED:
             data['balance'] = data['balance'][settings.HATHOR_TOKEN_UID]._asdict()