feat(sync-v2): add needed storage indexes

codecov.yml → .codecov.yml

@@ -1,11 +1,19 @@
 codecov:
+  branch: dev
+
+coverage:
+  # https://docs.codecov.com/docs/coverage-configuration
+  range: "80...90"
   # https://docs.codecov.io/docs/commit-status
   status:
+    # TODO: re-enable patch in the future
+    patch: off
     project:
       default:
         # minimum coverage ratio that the commit must meet to be considered a success
-        target: 82%
+        target: 83%
         if_ci_failed: error
         only_pulls: true
+
 github_checks:
-  annotations: false
+  annotations: true

Makefile

@@ -34,7 +34,7 @@ tests-doctests:
 
 .PHONY: tests-lib
 tests-lib:
-	pytest --durations=10 $(pytest_flags) --doctest-modules hathor --cov-fail-under=83 $(tests_lib)
+	pytest --durations=10 $(pytest_flags) --doctest-modules hathor $(tests_lib)
 
 .PHONY: tests-genesis
 tests-genesis:
@@ -46,7 +46,7 @@ tests: tests-cli tests-lib tests-genesis
 
 .PHONY: tests-full
 tests-full:
-	pytest $(pytest_flags) --durations=10 --cov-fail-under=90 --cov-config=.coveragerc_full ./tests
+	pytest $(pytest_flags) --durations=10 --cov-config=.coveragerc_full ./tests
 
 # checking:
 

hathor/consensus.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from itertools import chain
-from typing import Iterable, List, Optional, Set, cast
+from typing import Iterable, List, Optional, Set, Tuple, cast
 
 from structlog import get_logger
 
 from hathor.conf import HathorSettings
 from hathor.profiler import get_cpu_profiler
 from hathor.transaction import BaseTransaction, Block, Transaction, TxInput, sum_weights
-from hathor.util import classproperty
+from hathor.util import classproperty, not_none
 
 logger = get_logger()
 settings = HathorSettings()
@@ -184,6 +184,7 @@ def update_voided_info(self, block: Block) -> None:
             meta = block.get_metadata()
             if not meta.voided_by:
                 storage._best_block_tips = [block.hash]
+                storage.add_to_block_height_index(meta.height, block.hash)
             # The following assert must be true, but it is commented out for performance reasons.
             #     assert len(storage.get_best_block_tips(skip_cache=True)) == 1
         else:
@@ -242,14 +243,46 @@ def update_voided_info(self, block: Block) -> None:
                     meta = block.get_metadata()
                     if not meta.voided_by:
                         storage._best_block_tips = [block.hash]
+                        self.log.debug('index new winner block', height=meta.height, block=block.hash_hex)
+                        # We update the height cache index with the new winner chain
+                        self.update_block_height_cache_new_chain(meta.height, block)
                 else:
                     storage._best_block_tips = [blk.hash for blk in heads]
+                    # XXX Is it safe to select one of the heads?
+                    best_block = heads[0]
+                    assert best_block.hash is not None
+                    best_meta = best_block.get_metadata()
+                    self.log.debug('index previous best block', height=best_meta.height, block=best_block.hash_hex)
+                    storage.add_to_block_height_index(best_meta.height, best_block.hash)
 
         # Uncomment the following lines to check that the cache update is working properly.
         # You shouldn't run this test in production because it dampens performance.
         #     v = storage.get_best_block_tips(skip_cache=True)
         #     assert v == storage._best_block_tips
 
+    def update_block_height_cache_new_chain(self, height: int, block: Block) -> None:
+        """ When we have a new winner chain we must update all the height index
+            until the first height with a common block
+        """
+        storage = not_none(block.storage)
+        assert storage.get_from_block_height_index(height) != block.hash
+
+        block_height = height
+        side_chain_block = block
+        add_to_cache: List[Tuple[int, bytes]] = []
+        while storage.get_from_block_height_index(block_height) != side_chain_block.hash:
+            add_to_cache.append((block_height, not_none(side_chain_block.hash)))
+
+            side_chain_block = side_chain_block.get_block_parent()
+            block_height = side_chain_block.get_metadata().height
+
+        # Reverse the data because I was adding in the array from the highest block
+        reversed_add_to_cache = add_to_cache[::-1]
+
+        for height, tx_hash in reversed_add_to_cache:
+            # Add it to the index
+            storage.add_to_block_height_index(height, tx_hash)
+
     def union_voided_by_from_parents(self, block: Block) -> Set[bytes]:
         """Return the union of the voided_by of block's parents.
 
@@ -286,7 +319,7 @@ def update_voided_by_from_parents(self, block: Block) -> bool:
             else:
                 meta.voided_by = voided_by.copy()
             block.storage.save_transaction(block, only_metadata=True)
-            block.storage._del_from_cache(block, relax_assert=True)  # XXX: accessing private method
+            block.storage.del_from_indexes(block, relax_assert=True)
             return True
         return False
 
@@ -735,7 +768,7 @@ def update_voided_info(self, tx: Transaction) -> None:
         if voided_by:
             meta.voided_by = voided_by.copy()
             tx.storage.save_transaction(tx, only_metadata=True)
-            tx.storage._del_from_cache(tx)  # XXX: accessing private method
+            tx.storage.del_from_indexes(tx)
 
         # Check conflicts of the transactions voiding us.
         for h in voided_by:
@@ -853,7 +886,7 @@ def remove_voided_by(self, tx: Transaction, voided_hash: bytes) -> bool:
             tx2.storage.save_transaction(tx2, only_metadata=True)
             if not meta.voided_by:
                 meta.voided_by = None
-                tx.storage._add_to_cache(tx2)  # XXX: accessing private method
+                tx.storage.add_to_indexes(tx2)
 
         from hathor.transaction import Transaction
         for tx2 in check_list:
@@ -912,7 +945,7 @@ def add_voided_by(self, tx: Transaction, voided_hash: bytes) -> bool:
                 # All voided transactions with conflicts must have their accumulated weight calculated.
                 tx2.update_accumulated_weight(save_file=False)
             tx2.storage.save_transaction(tx2, only_metadata=True)
-            tx2.storage._del_from_cache(tx2, relax_assert=True)  # XXX: accessing private method
+            tx2.storage.del_from_indexes(tx2, relax_assert=True)
 
         for tx2 in check_list:
             self.check_conflicts(tx2)

hathor/daa.py

@@ -23,13 +23,16 @@
 from math import log
 from typing import TYPE_CHECKING, List
 
+from structlog import get_logger
+
 from hathor.conf import HathorSettings
 from hathor.profiler import get_cpu_profiler
 from hathor.util import iwindows
 
 if TYPE_CHECKING:
     from hathor.transaction import Block, Transaction
 
+logger = get_logger()
 settings = HathorSettings()
 cpu = get_cpu_profiler()
 
@@ -49,6 +52,7 @@ class TestMode(IntFlag):
 
 def _set_test_mode(mode: TestMode) -> None:
     global TEST_MODE
+    logger.debug('change DAA test mode', from_mode=TEST_MODE.name, to_mode=mode.name)
     TEST_MODE = mode