feat(feature-activation): bit count optimization

hathor/conf/unittests.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from hathor.conf.settings import HathorSettings
+from hathor.feature_activation.settings import Settings as FeatureActivationSettings
 
 SETTINGS = HathorSettings(
     P2PKH_VERSION_BYTE=b'\x28',
@@ -34,4 +35,9 @@
     REWARD_SPEND_MIN_BLOCKS=10,
     SLOW_ASSERTS=True,
     MAX_TX_WEIGHT_DIFF_ACTIVATION=0.0,
+    FEATURE_ACTIVATION=FeatureActivationSettings(
+        evaluation_interval=4,
+        max_signal_bits=4,
+        default_threshold=3
+    )
 )

hathor/conf/unittests.yml

@@ -17,3 +17,8 @@ GENESIS_TX2_HASH: 33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e8
 REWARD_SPEND_MIN_BLOCKS: 10
 SLOW_ASSERTS: true
 MAX_TX_WEIGHT_DIFF_ACTIVATION: 0.0
+
+FEATURE_ACTIVATION:
+  evaluation_interval: 4
+  max_signal_bits: 4
+  default_threshold: 3

hathor/feature_activation/feature_service.py

@@ -66,11 +66,11 @@ def _calculate_new_state(
     ) -> FeatureState:
         """Returns the new feature state based on the new block, the criteria, and the previous state."""
         height = boundary_block.get_height()
-        assert height % self._settings.evaluation_interval == 0, (
-            'cannot calculate new state for a non-boundary block'
-        )
         criteria = self._get_criteria(feature=feature)
 
+        assert not boundary_block.is_genesis, 'cannot calculate new state for genesis'
+        assert height % self._settings.evaluation_interval == 0, 'cannot calculate new state for a non-boundary block'
+
         if previous_state is FeatureState.DEFINED:
             if height >= criteria.start_height:
                 return FeatureState.STARTED
@@ -88,7 +88,10 @@ def _calculate_new_state(
             ):
                 return FeatureState.ACTIVE
 
-            count = self.get_bit_count(boundary_block=boundary_block, bit=criteria.bit)
+            # Get the count for this block's parent. Since this is a boundary block, its parent count represents the
+            # previous evaluation interval count.
+            counts = boundary_block.get_parent_feature_activation_bit_counts()
+            count = counts[criteria.bit]
             threshold = criteria.threshold if criteria.threshold is not None else self._settings.default_threshold
 
             if (
@@ -127,27 +130,6 @@ def get_bits_description(self, *, block: Block) -> dict[Feature, FeatureDescript
             for feature, criteria in self._settings.features.items()
         }
 
-    def get_bit_count(self, *, boundary_block: Block, bit: int) -> int:
-        """Returns the count of blocks with this bit enabled in the previous evaluation interval."""
-        assert not boundary_block.is_genesis, 'cannot calculate bit count for genesis'
-        assert boundary_block.get_height() % self._settings.evaluation_interval == 0, (
-            'cannot calculate bit count for a non-boundary block'
-        )
-        count = 0
-        block = boundary_block
-
-        # TODO: We can implement this as O(1) instead of O(evaluation_interval)
-        #  by persisting the count in block metadata incrementally
-        for _ in range(self._settings.evaluation_interval):
-            block = block.get_block_parent()
-            feature_bits = block.get_feature_activation_bits()
-            bit_is_active = (feature_bits >> bit) & 1
-
-            if bit_is_active:
-                count += 1
-
-        return count
-
 
 def _get_ancestor_at_height(*, block: Block, height: int) -> Block:
     """Given a block, returns its ancestor at a specific height."""

hathor/transaction/base_transaction.py

@@ -875,8 +875,21 @@ def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True)
             #        happens include generating new mining blocks and some tests
             height = self.calculate_height() if self.storage else 0
             score = self.weight if self.is_genesis else 0
-            metadata = TransactionMetadata(hash=self.hash, accumulated_weight=self.weight, height=height, score=score,
-                                           min_height=0)
+            kwargs: dict[str, Any] = {}
+
+            if self.is_block:
+                from hathor.transaction import Block
+                assert isinstance(self, Block)
+                kwargs['feature_activation_bit_counts'] = self.calculate_feature_activation_bit_counts()
+
+            metadata = TransactionMetadata(
+                hash=self.hash,
+                accumulated_weight=self.weight,
+                height=height,
+                score=score,
+                min_height=0,
+                **kwargs
+            )
             self._metadata = metadata
         if not metadata.hash:
             metadata.hash = self.hash
@@ -956,6 +969,7 @@ def update_initial_metadata(self, *, save: bool = True) -> None:
         self._update_height_metadata()
         self._update_parents_children_metadata()
         self._update_reward_lock_metadata()
+        self._update_feature_activation_bit_counts_metadata()
         if save:
             assert self.storage is not None
             self.storage.save_transaction(self, only_metadata=True)
@@ -981,6 +995,16 @@ def _update_parents_children_metadata(self) -> None:
                 metadata.children.append(self.hash)
                 self.storage.save_transaction(parent, only_metadata=True)
 
+    def _update_feature_activation_bit_counts_metadata(self) -> None:
+        """Update the block feature_activation_bit_counts metadata."""
+        if not self.is_block:
+            return
+
+        from hathor.transaction import Block
+        assert isinstance(self, Block)
+        metadata = self.get_metadata()
+        metadata.feature_activation_bit_counts = self.calculate_feature_activation_bit_counts()
+
     def update_timestamp(self, now: int) -> None:
         """Update this tx's timestamp
 

hathor/transaction/block.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import base64
+from itertools import starmap, zip_longest
+from operator import add
 from struct import pack
 from typing import TYPE_CHECKING, Any, Optional
 
@@ -31,6 +33,7 @@
     WeightError,
 )
 from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len
+from hathor.utils.int import get_bit_list
 
 if TYPE_CHECKING:
     from hathor.transaction.storage import TransactionStorage  # noqa: F401
@@ -110,6 +113,35 @@ def calculate_min_height(self) -> int:
         return max((self.storage.get_transaction(tx).get_metadata().min_height for tx in self.get_tx_parents()),
                    default=0)
 
+    def calculate_feature_activation_bit_counts(self) -> list[int]:
+        """
+        Calculates the feature_activation_bit_counts metadata attribute, which is a list of feature activation bit
+        counts.
+
+        Each list index corresponds to a bit position, and its respective value is the rolling count of active bits
+        from the previous boundary block up to this block, including it. LSB is on the left.
+        """
+        previous_counts = self._get_previous_feature_activation_bit_counts()
+        bit_list = self._get_feature_activation_bit_list()
+
+        count_and_bit_pairs = zip_longest(previous_counts, bit_list, fillvalue=0)
+        updated_counts = starmap(add, count_and_bit_pairs)
+
+        return list(updated_counts)
+
+    def _get_previous_feature_activation_bit_counts(self) -> list[int]:
+        """
+        Returns the feature_activation_bit_counts metadata attribute from the parent block,
+        or no previous counts if this is a boundary block.
+        """
+        evaluation_interval = settings.FEATURE_ACTIVATION.evaluation_interval
+        is_boundary_block = self.calculate_height() % evaluation_interval == 0
+
+        if is_boundary_block:
+            return []
+
+        return self.get_parent_feature_activation_bit_counts()
+
     def get_next_block_best_chain_hash(self) -> Optional[bytes]:
         """Return the hash of the next (child/left-to-right) block in the best blockchain.
         """
@@ -354,13 +386,26 @@ def get_height(self) -> int:
         """Returns the block's height."""
         return self.get_metadata().height
 
-    def get_feature_activation_bits(self) -> int:
-        """Returns the feature activation bits from the signal bits."""
+    def get_parent_feature_activation_bit_counts(self) -> list[int]:
+        """Returns the parent block's feature_activation_bit_counts metadata attribute."""
+        parent_metadata = self.get_block_parent().get_metadata()
+        assert parent_metadata.feature_activation_bit_counts is not None, 'Blocks must always have this attribute set.'
+
+        return parent_metadata.feature_activation_bit_counts
+
+    def _get_feature_activation_bit_list(self) -> list[int]:
+        """
+        Extracts feature activation bits from the signal bits, as a list where each index corresponds to the bit
+        position. LSB is on the left.
+        """
         assert self.signal_bits <= 0xFF, 'signal_bits must be one byte at most'
 
         bitmask = self._get_feature_activation_bitmask()
+        bits = self.signal_bits & bitmask
+
+        bit_list = get_bit_list(bits, min_size=settings.FEATURE_ACTIVATION.max_signal_bits)
 
-        return self.signal_bits & bitmask
+        return bit_list
 
     @classmethod
     def _get_feature_activation_bitmask(cls) -> int:

hathor/transaction/transaction_metadata.py

@@ -44,15 +44,28 @@ class TransactionMetadata:
     # metadata (that does not have this calculated, from a tx with a new format that does have this calculated)
     min_height: int
 
+    # A list of feature activation bit counts. Must only be used by Blocks, is None otherwise.
+    # Each list index corresponds to a bit position, and its respective value is the rolling count of active bits from
+    # the previous boundary block up to this block, including it. LSB is on the left.
+    feature_activation_bit_counts: Optional[list[int]]
+
     # It must be a weakref.
     _tx_ref: Optional['ReferenceType[BaseTransaction]']
 
     # Used to detect changes in voided_by.
     _last_voided_by_hash: Optional[int]
     _last_spent_by_hash: Optional[int]
 
-    def __init__(self, spent_outputs: Optional[dict[int, list[bytes]]] = None, hash: Optional[bytes] = None,
-                 accumulated_weight: float = 0, score: float = 0, height: int = 0, min_height: int = 0) -> None:
+    def __init__(
+        self,
+        spent_outputs: Optional[dict[int, list[bytes]]] = None,
+        hash: Optional[bytes] = None,
+        accumulated_weight: float = 0,
+        score: float = 0,
+        height: int = 0,
+        min_height: int = 0,
+        feature_activation_bit_counts: Optional[list[int]] = None
+    ) -> None:
         from hathor.transaction.genesis import is_genesis
 
         # Hash of the transaction.
@@ -107,6 +120,8 @@ def __init__(self, spent_outputs: Optional[dict[int, list[bytes]]] = None, hash:
         # Validation
         self.validation = ValidationState.INITIAL
 
+        self.feature_activation_bit_counts = feature_activation_bit_counts
+
         # Genesis specific:
         if hash is not None and is_genesis(hash):
             self.validation = ValidationState.FULL
@@ -168,7 +183,7 @@ def __eq__(self, other: Any) -> bool:
             return False
         for field in ['hash', 'conflict_with', 'voided_by', 'received_by',
                       'children', 'accumulated_weight', 'twins', 'score',
-                      'first_block', 'validation', 'min_height']:
+                      'first_block', 'validation', 'min_height', 'feature_activation_bit_counts']:
             if (getattr(self, field) or None) != (getattr(other, field) or None):
                 return False
 
@@ -203,6 +218,7 @@ def to_json(self) -> dict[str, Any]:
         data['score'] = self.score
         data['height'] = self.height
         data['min_height'] = self.min_height
+        data['feature_activation_bit_counts'] = self.feature_activation_bit_counts
         if self.first_block is not None:
             data['first_block'] = self.first_block.hex()
         else:
@@ -252,6 +268,7 @@ def create_from_json(cls, data: dict[str, Any]) -> 'TransactionMetadata':
         meta.score = data.get('score', 0)
         meta.height = data.get('height', 0)  # XXX: should we calculate the height if it's not defined?
         meta.min_height = data.get('min_height', 0)
+        meta.feature_activation_bit_counts = data.get('feature_activation_bit_counts', [])
 
         first_block_raw = data.get('first_block', None)
         if first_block_raw:

hathor/utils/int.py

@@ -0,0 +1,54 @@
+#  Copyright 2023 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Optional
+
+
+def get_bit_list(n: int, min_size: Optional[int] = None) -> list[int]:
+    """
+    Returns a list of bits corresponding to a non-negative number, with LSB on the left.
+
+    Args:
+        n: the number
+        min_size: if set, pads the returned list with zeroes until it reaches min_size
+
+    >>> get_bit_list(0b0)
+    []
+    >>> get_bit_list(0b1)
+    [1]
+    >>> get_bit_list(0b10)
+    [0, 1]
+    >>> get_bit_list(0b111001010)
+    [0, 1, 0, 1, 0, 0, 1, 1, 1]
+    >>> get_bit_list(0b0, min_size=4)
+    [0, 0, 0, 0]
+    >>> get_bit_list(0b1, min_size=3)
+    [1, 0, 0]
+    >>> get_bit_list(0b10, min_size=1)
+    [0, 1]
+    >>> get_bit_list(0b111001010, min_size=10)
+    [0, 1, 0, 1, 0, 0, 1, 1, 1, 0]
+    """
+    assert n >= 0
+    bits = []
+
+    while n > 0:
+        bits.append(n & 1)
+        n >>= 1
+
+    if min_size is not None:
+        while len(bits) < min_size:
+            bits.append(0)
+
+    return bits