refactor(indexes): improve mempool tips index

hathor/consensus/block_consensus.py

@@ -112,8 +112,19 @@ def execute_nano_contracts(self, block: Block) -> None:
                 to_be_executed.append(cur)
                 cur = cur.get_block_parent()
         else:
-            # no reorg occurred, so simply execute this new block.
-            to_be_executed = [block]
+            # No reorg occurred, so we execute all unexecuted blocks.
+            # Normally it's just the current block, but it's possible to have
+            # voided and therefore unexecuted blocks connected to the best chain,
+            # for example when a block is voided by a transaction.
+            cur = block
+            while True:
+                cur_meta = cur.get_metadata()
+                if cur_meta.nc_block_root_id is not None:
+                    break
+                to_be_executed.append(cur)
+                if cur.is_genesis:
+                    break
+                cur = cur.get_block_parent()
 
         for current in to_be_executed[::-1]:
             self._nc_execute_calls(current, is_reorg=is_reorg)
@@ -126,15 +137,17 @@ def _nc_execute_calls(self, block: Block, *, is_reorg: bool) -> None:
 
         assert self._settings.ENABLE_NANO_CONTRACTS
 
+        if block.is_genesis:
+            # XXX We can remove this call after the full node initialization is refactored and
+            #     the genesis block goes through the consensus protocol.
+            self._nc_initialize_genesis(block)
+            return
+
         meta = block.get_metadata()
         assert not meta.voided_by
         assert meta.nc_block_root_id is None
 
         parent = block.get_block_parent()
-        if parent.is_genesis:
-            # XXX We can remove this call after the full node initialization is refactored and
-            #     the genesis block goes through the consensus protocol.
-            self._nc_initialize_genesis(parent)
         parent_meta = parent.get_metadata()
         block_root_id = parent_meta.nc_block_root_id
         assert block_root_id is not None

hathor/graphviz.py

@@ -65,6 +65,8 @@ def get_node_label(self, tx: BaseTransaction) -> str:
         """
         if tx.hash in self.labels:
             parts = [self.labels[tx.hash]]
+        elif tx.name is not None:
+            parts = [tx.name]
         else:
             parts = [tx.hash.hex()[-4:]]
 

hathor/indexes/manager.py

@@ -206,8 +206,8 @@ def _manually_initialize(self, tx_storage: 'TransactionStorage') -> None:
     def update(self, tx: BaseTransaction) -> None:
         """ This is the new update method that indexes should use instead of add_tx/del_tx
         """
-        # XXX: this _should_ be here, but it breaks some tests, for now this is done explicitly in hathor.manager
-        # self.mempool_tips.update(tx)
+        if self.mempool_tips:
+            self.mempool_tips.update(tx)
         if self.utxo:
             self.utxo.update(tx)
 
@@ -434,7 +434,7 @@ def del_tx(self, tx: BaseTransaction, *, remove_all: bool = False, relax_assert:
         # mempool will pick-up if the transaction is voided/invalid and remove it
         if self.mempool_tips is not None and tx.storage.transaction_exists(tx.hash):
             logger.debug('remove from mempool tips', tx=tx.hash_hex)
-            self.mempool_tips.update(tx, remove=True)
+            self.mempool_tips.update(tx, force_remove=True)
 
         if tx.is_block:
             self.block_tips.del_tx(tx, relax_assert=relax_assert)

hathor/indexes/mempool_tips_index.py

@@ -56,7 +56,7 @@ def get_scope(self) -> Scope:
         return SCOPE
 
     @abstractmethod
-    def update(self, tx: BaseTransaction, *, remove: Optional[bool] = None) -> None:
+    def update(self, tx: BaseTransaction, *, force_remove: bool = False) -> None:
         """
         This should be called when a new `tx/block` is added to the best chain.
 
@@ -129,74 +129,62 @@ def _add_many(self, txs: Iterable[bytes]) -> None:
 
     # PROVIDES:
 
-    def update(self, tx: BaseTransaction, *, remove: Optional[bool] = None) -> None:
+    def update(self, tx: BaseTransaction, *, force_remove: bool = False) -> None:
         assert tx.storage is not None
         tx_meta = tx.get_metadata()
         to_remove: set[bytes] = set()
-        to_remove_parents: set[bytes] = set()
+        deps_to_check: set[bytes] = set()
         tx_storage = tx.storage
         for tip_tx in self.iter(tx_storage):
             meta = tip_tx.get_metadata()
-            # a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1 a
+            # a new tx/block added might cause a tx in the tips to become voided. For instance, there might be a tx1
             # double spending tx2, where tx1 is valid and tx2 voided. A new block confirming tx2 will make it valid
             # while tx1 becomes voided, so it has to be removed from the tips. The txs confirmed by tx1 need to be
-            # double checked, as they might themselves become tips (hence we use to_remove_parents)
+            # double checked, as they might themselves become tips (hence we use deps_to_check)
             if meta.voided_by or meta.validation.is_invalid():
                 to_remove.add(tip_tx.hash)
-                to_remove_parents.update(tip_tx.parents)
+                deps_to_check.update(tip_tx.get_all_dependencies())
                 continue
 
-            # might also happen that a tip has a child that became valid, so it's not a tip anymore
-            confirmed = False
-            for child_meta in filter(None, map(tx_storage.get_metadata, meta.children)):
-                if not child_meta.voided_by:
-                    confirmed = True
-                    break
-            if confirmed:
+            # might also happen that a tip has a child or a spender that became valid, so it's not a tip anymore
+            has_non_voided_child = lambda: any_non_voided(tx_storage, meta.children)
+            has_non_voided_spender = lambda: any_non_voided(tx_storage, chain(*meta.spent_outputs.values()))
+            if has_non_voided_child() or has_non_voided_spender():
                 to_remove.add(tip_tx.hash)
 
         if to_remove:
             self._discard_many(to_remove)
-            self.log.debug('removed voided txs from tips', txs=[tx.hex() for tx in to_remove])
+            self.log.debug('removed txs from tips', txs=[tx.hex() for tx in to_remove])
 
-        # Check if any of the txs being confirmed by the voided txs is a tip again. This happens
-        # if it doesn't have any other valid child.
+        # Check if any of the txs pointed by the removed tips is a tip again. This happens
+        # if it doesn't have any other valid child or spender.
         to_add = set()
-        for tx_hash in to_remove_parents:
-            confirmed = False
-            # check if it has any valid children
+        for tx_hash in deps_to_check:
             meta = not_none(tx_storage.get_metadata(tx_hash))
             if meta.voided_by:
                 continue
-            children = meta.children
-            for child_meta in filter(None, map(tx_storage.get_metadata, children)):
-                if not child_meta.voided_by:
-                    confirmed = True
-                    break
-            if not confirmed:
+            # check if it has any valid children or spenders
+            has_non_voided_child = lambda: any_non_voided(tx_storage, meta.children)
+            has_non_voided_spender = lambda: any_non_voided(tx_storage, chain(*meta.spent_outputs.values()))
+            if not has_non_voided_child() and not has_non_voided_spender():
                 to_add.add(tx_hash)
 
         if to_add:
             self._add_many(to_add)
             self.log.debug('added txs to tips', txs=[tx.hex() for tx in to_add])
 
-        actually_remove: bool
         voided_or_invalid = bool(tx_meta.voided_by) or tx_meta.validation.is_invalid()
-        if remove is None:
-            actually_remove = voided_or_invalid
-        else:
-            actually_remove = remove
-            if remove and not voided_or_invalid:
-                self.log.warn('removing tx even though it isn\'t voided or invalid, some tests can do this')
-            if not remove and voided_or_invalid:
-                raise ValueError('cannot add voided or invalid tx to mempool')
-
-        if actually_remove:
+        remove = force_remove or voided_or_invalid
+
+        if force_remove and not voided_or_invalid:
+            self.log.warn('removing tx even though it isn\'t voided or invalid, some tests can do this')
+
+        if remove:
             self.log.debug('remove from mempool', tx=tx.hash_hex, validation=tx_meta.validation,
                            is_voided=bool(tx_meta.voided_by))
             return
 
-        self._discard_many(set(tx.parents))
+        self._discard_many(tx.get_all_dependencies())
 
         if tx.is_transaction and tx_meta.first_block is None:
             self._add(tx.hash)
@@ -209,9 +197,10 @@ def iter(self, tx_storage: 'TransactionStorage', max_timestamp: Optional[float]
 
     def iter_all(self, tx_storage: 'TransactionStorage') -> Iterator[Transaction]:
         from hathor.transaction.storage.traversal import BFSTimestampWalk
-        bfs = BFSTimestampWalk(tx_storage, is_dag_verifications=True, is_left_to_right=False)
+        bfs = BFSTimestampWalk(tx_storage, is_dag_verifications=True, is_dag_funds=True, is_left_to_right=False)
         for tx in bfs.run(self.iter(tx_storage), skip_root=False):
-            assert isinstance(tx, Transaction)
+            if not isinstance(tx, Transaction):
+                continue
             if tx.get_metadata().first_block is not None:
                 bfs.skip_neighbors(tx)
             else:

hathor/transaction/base_transaction.py

@@ -225,6 +225,7 @@ def _get_formatted_fields_dict(self, short: bool = True) -> dict[str, str]:
         """
         from collections import OrderedDict
         d = OrderedDict(
+            name=self.name or '',
             nonce='%d' % (self.nonce or 0),
             timestamp='%s' % self.timestamp,
             version='%s' % int(self.version),

hathor/vertex_handler/vertex_handler.py

@@ -218,15 +218,10 @@ def _post_consensus(
             init_static_metadata=False,
         )
         self._tx_storage.indexes.update(vertex)
-        if self._tx_storage.indexes.mempool_tips:
-            self._tx_storage.indexes.mempool_tips.update(vertex)  # XXX: move to indexes.update
 
         # Publish to pubsub manager the new tx accepted, now that it's full validated
         self._pubsub.publish(HathorEvents.NETWORK_NEW_TX_ACCEPTED, tx=vertex)
 
-        if self._tx_storage.indexes.mempool_tips:
-            self._tx_storage.indexes.mempool_tips.update(vertex)
-
         if self._wallet:
             # TODO Remove it and use pubsub instead.
             self._wallet.on_new_tx(vertex)

setup.cfg

@@ -33,6 +33,8 @@ directory = coverage_html_report
 
 [flake8]
 max-line-length = 119
+# E731 do not assign a lambda expression, use a def
+extend-ignore = E731
 
 [mypy]
 ignore_missing_imports = True

tests/consensus/test_consensus6.py

@@ -0,0 +1,153 @@
+#  Copyright 2025 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from hathor.daa import DifficultyAdjustmentAlgorithm, TestMode
+from hathor.graphviz import GraphvizVisualizer
+from hathor.transaction import Block, Transaction
+from tests import unittest
+from tests.dag_builder.builder import TestDAGBuilder
+
+DEBUG: bool = False
+
+
+class TestConsensus6(unittest.TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        settings = self._settings._replace(REWARD_SPEND_MIN_BLOCKS=1)  # for simplicity
+        daa = DifficultyAdjustmentAlgorithm(settings=settings, test_mode=TestMode.TEST_ALL_WEIGHT)
+        builder = self.get_builder(settings).set_daa(daa)
+
+        self.manager = self.create_peer_from_builder(builder)
+        self.tx_storage = self.manager.tx_storage
+
+    def test_conflict_on_reorg(self) -> None:
+        dag_builder = TestDAGBuilder.from_manager(self.manager)
+        artifacts = dag_builder.build_from_str('''
+            blockchain genesis b[1..2]
+            blockchain b1 a[2..4]
+            b1 < dummy
+
+            b1 < tx1 < tx2 < tx3 < b2
+            tx3 <-- b2
+
+            # tx2 has a conflict with tx3
+            tx1.out[0] <<< tx2
+            tx1.out[0] <<< tx3
+
+            # a2 will generate a reorg
+            a2.weight = 10
+            b2 < a2
+            tx2 <-- a3
+        ''')
+
+        b1, b2, a2, a3 = artifacts.get_typed_vertices(['b1', 'b2', 'a2', 'a3'], Block)
+        tx1, tx2, tx3, dummy = artifacts.get_typed_vertices(['tx1', 'tx2', 'tx3', 'dummy'], Transaction)
+
+        artifacts.propagate_with(self.manager, up_to='b1')
+
+        assert b1.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().validation.is_initial()
+        assert tx2.get_metadata().validation.is_initial()
+        assert tx3.get_metadata().validation.is_initial()
+
+        assert tx1.get_metadata().voided_by is None
+        assert tx2.get_metadata().voided_by is None
+        assert tx3.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().first_block is None
+        assert tx2.get_metadata().first_block is None
+        assert tx3.get_metadata().first_block is None
+
+        assert tx1.get_metadata().accumulated_weight == 2
+        assert tx2.get_metadata().accumulated_weight == 2
+        assert tx3.get_metadata().accumulated_weight == 2
+
+        artifacts.propagate_with(self.manager, up_to='tx3')
+
+        if DEBUG:
+            dot = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True).dot()
+            dot.render('before-b2')
+
+        artifacts.propagate_with(self.manager, up_to='b2')
+
+        if DEBUG:
+            dot = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True).dot()
+            dot.render('after-b2')
+
+        assert b1.get_metadata().voided_by is None
+        assert b2.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().voided_by is None
+        assert tx2.get_metadata().voided_by == {tx2.hash}
+        assert tx3.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().first_block == b2.hash
+        assert tx2.get_metadata().first_block is None
+        assert tx3.get_metadata().first_block == b2.hash
+
+        assert tx1.get_metadata().accumulated_weight == 2
+        assert tx2.get_metadata().accumulated_weight == 2
+        assert tx3.get_metadata().accumulated_weight == 4
+
+        artifacts.propagate_with(self.manager, up_to='a2')
+
+        if DEBUG:
+            dot = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True).dot()
+            dot.render('after-a2')
+
+        assert b1.get_metadata().voided_by is None
+        assert b2.get_metadata().voided_by == {b2.hash}
+        assert a2.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().voided_by is None
+        assert tx2.get_metadata().voided_by == {tx2.hash}
+        assert tx3.get_metadata().voided_by is None
+
+        assert tx1.get_metadata().first_block is None
+        assert tx2.get_metadata().first_block is None
+        assert tx3.get_metadata().first_block is None
+
+        assert tx1.get_metadata().accumulated_weight == 2
+        assert tx2.get_metadata().accumulated_weight == 2
+        assert tx3.get_metadata().accumulated_weight == 4
+
+        artifacts.propagate_with(self.manager, up_to='a3')
+
+        if DEBUG:
+            dot = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True).dot()
+            dot.render('after-a3')
+
+        assert b1.get_metadata().voided_by is None
+        assert b2.get_metadata().voided_by == {b2.hash, tx3.hash}
+        assert a2.get_metadata().voided_by is None
+        assert a3.get_metadata().voided_by == {a3.hash, tx2.hash}
+
+        assert tx1.get_metadata().voided_by is None
+        assert tx2.get_metadata().voided_by == {tx2.hash}
+        assert tx3.get_metadata().voided_by == {tx3.hash}
+
+        assert tx1.get_metadata().first_block is None
+        assert tx2.get_metadata().first_block is None
+        assert tx3.get_metadata().first_block is None
+
+        assert tx1.get_metadata().accumulated_weight == 2
+        assert tx2.get_metadata().accumulated_weight == 4
+        assert tx3.get_metadata().accumulated_weight == 4
+
+        artifacts.propagate_with(self.manager, up_to='a4')
+
+        if DEBUG:
+            dot = GraphvizVisualizer(self.tx_storage, include_verifications=True, include_funds=True).dot()
+            dot.render('after-a4')