diff --git a/hathor/conf/unittests.py b/hathor/conf/unittests.py index df85ae851..0b9d46144 100644 --- a/hathor/conf/unittests.py +++ b/hathor/conf/unittests.py @@ -13,6 +13,7 @@ # limitations under the License. from hathor.conf.settings import HathorSettings +from hathor.feature_activation.settings import Settings as FeatureActivationSettings SETTINGS = HathorSettings( P2PKH_VERSION_BYTE=b'\x28', @@ -34,4 +35,9 @@ REWARD_SPEND_MIN_BLOCKS=10, SLOW_ASSERTS=True, MAX_TX_WEIGHT_DIFF_ACTIVATION=0.0, + FEATURE_ACTIVATION=FeatureActivationSettings( + evaluation_interval=4, + max_signal_bits=4, + default_threshold=3 + ) ) diff --git a/hathor/conf/unittests.yml b/hathor/conf/unittests.yml index a2cf9dcd5..ebd25657d 100644 --- a/hathor/conf/unittests.yml +++ b/hathor/conf/unittests.yml @@ -17,3 +17,8 @@ GENESIS_TX2_HASH: 33e14cb555a96967841dcbe0f95e9eab5810481d01de8f4f73afb8cce365e8 REWARD_SPEND_MIN_BLOCKS: 10 SLOW_ASSERTS: true MAX_TX_WEIGHT_DIFF_ACTIVATION: 0.0 + +FEATURE_ACTIVATION: + evaluation_interval: 4 + max_signal_bits: 4 + default_threshold: 3 diff --git a/hathor/feature_activation/feature_service.py b/hathor/feature_activation/feature_service.py index bc1bba5af..c70c206ed 100644 --- a/hathor/feature_activation/feature_service.py +++ b/hathor/feature_activation/feature_service.py @@ -66,11 +66,11 @@ def _calculate_new_state( ) -> FeatureState: """Returns the new feature state based on the new block, the criteria, and the previous state.""" height = boundary_block.get_height() - assert height % self._settings.evaluation_interval == 0, ( - 'cannot calculate new state for a non-boundary block' - ) criteria = self._get_criteria(feature=feature) + assert not boundary_block.is_genesis, 'cannot calculate new state for genesis' + assert height % self._settings.evaluation_interval == 0, 'cannot calculate new state for a non-boundary block' + if previous_state is FeatureState.DEFINED: if height >= criteria.start_height: return FeatureState.STARTED @@ -88,7 +88,10 @@ def _calculate_new_state( ): return FeatureState.ACTIVE - count = self.get_bit_count(boundary_block=boundary_block, bit=criteria.bit) + # Get the count for this block's parent. Since this is a boundary block, its parent count represents the + # previous evaluation interval count. + counts = boundary_block.get_parent_feature_activation_bit_counts() + count = counts[criteria.bit] threshold = criteria.threshold if criteria.threshold is not None else self._settings.default_threshold if ( @@ -127,27 +130,6 @@ def get_bits_description(self, *, block: Block) -> dict[Feature, FeatureDescript for feature, criteria in self._settings.features.items() } - def get_bit_count(self, *, boundary_block: Block, bit: int) -> int: - """Returns the count of blocks with this bit enabled in the previous evaluation interval.""" - assert not boundary_block.is_genesis, 'cannot calculate bit count for genesis' - assert boundary_block.get_height() % self._settings.evaluation_interval == 0, ( - 'cannot calculate bit count for a non-boundary block' - ) - count = 0 - block = boundary_block - - # TODO: We can implement this as O(1) instead of O(evaluation_interval) - # by persisting the count in block metadata incrementally - for _ in range(self._settings.evaluation_interval): - block = block.get_block_parent() - feature_bits = block.get_feature_activation_bits() - bit_is_active = (feature_bits >> bit) & 1 - - if bit_is_active: - count += 1 - - return count - def _get_ancestor_at_height(*, block: Block, height: int) -> Block: """Given a block, returns its ancestor at a specific height.""" diff --git a/hathor/transaction/base_transaction.py b/hathor/transaction/base_transaction.py index 7ff003d69..c91292813 100644 --- a/hathor/transaction/base_transaction.py +++ b/hathor/transaction/base_transaction.py @@ -875,8 +875,21 @@ def get_metadata(self, *, force_reload: bool = False, use_storage: bool = True) # happens include generating new mining blocks and some tests height = self.calculate_height() if self.storage else 0 score = self.weight if self.is_genesis else 0 - metadata = TransactionMetadata(hash=self.hash, accumulated_weight=self.weight, height=height, score=score, - min_height=0) + kwargs: dict[str, Any] = {} + + if self.is_block: + from hathor.transaction import Block + assert isinstance(self, Block) + kwargs['feature_activation_bit_counts'] = self.calculate_feature_activation_bit_counts() + + metadata = TransactionMetadata( + hash=self.hash, + accumulated_weight=self.weight, + height=height, + score=score, + min_height=0, + **kwargs + ) self._metadata = metadata if not metadata.hash: metadata.hash = self.hash @@ -956,6 +969,7 @@ def update_initial_metadata(self, *, save: bool = True) -> None: self._update_height_metadata() self._update_parents_children_metadata() self._update_reward_lock_metadata() + self._update_feature_activation_bit_counts_metadata() if save: assert self.storage is not None self.storage.save_transaction(self, only_metadata=True) @@ -981,6 +995,16 @@ def _update_parents_children_metadata(self) -> None: metadata.children.append(self.hash) self.storage.save_transaction(parent, only_metadata=True) + def _update_feature_activation_bit_counts_metadata(self) -> None: + """Update the block feature_activation_bit_counts metadata.""" + if not self.is_block: + return + + from hathor.transaction import Block + assert isinstance(self, Block) + metadata = self.get_metadata() + metadata.feature_activation_bit_counts = self.calculate_feature_activation_bit_counts() + def update_timestamp(self, now: int) -> None: """Update this tx's timestamp diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 4a8ac0c5e..2ede7fca8 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -13,6 +13,8 @@ # limitations under the License. import base64 +from itertools import starmap, zip_longest +from operator import add from struct import pack from typing import TYPE_CHECKING, Any, Optional @@ -31,6 +33,7 @@ WeightError, ) from hathor.transaction.util import VerboseCallback, int_to_bytes, unpack, unpack_len +from hathor.utils.int import get_bit_list if TYPE_CHECKING: from hathor.transaction.storage import TransactionStorage # noqa: F401 @@ -110,6 +113,35 @@ def calculate_min_height(self) -> int: return max((self.storage.get_transaction(tx).get_metadata().min_height for tx in self.get_tx_parents()), default=0) + def calculate_feature_activation_bit_counts(self) -> list[int]: + """ + Calculates the feature_activation_bit_counts metadata attribute, which is a list of feature activation bit + counts. + + Each list index corresponds to a bit position, and its respective value is the rolling count of active bits + from the previous boundary block up to this block, including it. LSB is on the left. + """ + previous_counts = self._get_previous_feature_activation_bit_counts() + bit_list = self._get_feature_activation_bit_list() + + count_and_bit_pairs = zip_longest(previous_counts, bit_list, fillvalue=0) + updated_counts = starmap(add, count_and_bit_pairs) + + return list(updated_counts) + + def _get_previous_feature_activation_bit_counts(self) -> list[int]: + """ + Returns the feature_activation_bit_counts metadata attribute from the parent block, + or no previous counts if this is a boundary block. + """ + evaluation_interval = settings.FEATURE_ACTIVATION.evaluation_interval + is_boundary_block = self.calculate_height() % evaluation_interval == 0 + + if is_boundary_block: + return [] + + return self.get_parent_feature_activation_bit_counts() + def get_next_block_best_chain_hash(self) -> Optional[bytes]: """Return the hash of the next (child/left-to-right) block in the best blockchain. """ @@ -354,13 +386,26 @@ def get_height(self) -> int: """Returns the block's height.""" return self.get_metadata().height - def get_feature_activation_bits(self) -> int: - """Returns the feature activation bits from the signal bits.""" + def get_parent_feature_activation_bit_counts(self) -> list[int]: + """Returns the parent block's feature_activation_bit_counts metadata attribute.""" + parent_metadata = self.get_block_parent().get_metadata() + assert parent_metadata.feature_activation_bit_counts is not None, 'Blocks must always have this attribute set.' + + return parent_metadata.feature_activation_bit_counts + + def _get_feature_activation_bit_list(self) -> list[int]: + """ + Extracts feature activation bits from the signal bits, as a list where each index corresponds to the bit + position. LSB is on the left. + """ assert self.signal_bits <= 0xFF, 'signal_bits must be one byte at most' bitmask = self._get_feature_activation_bitmask() + bits = self.signal_bits & bitmask + + bit_list = get_bit_list(bits, min_size=settings.FEATURE_ACTIVATION.max_signal_bits) - return self.signal_bits & bitmask + return bit_list @classmethod def _get_feature_activation_bitmask(cls) -> int: diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 77bb54fab..c9425d421 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -44,6 +44,11 @@ class TransactionMetadata: # metadata (that does not have this calculated, from a tx with a new format that does have this calculated) min_height: int + # A list of feature activation bit counts. Must only be used by Blocks, is None otherwise. + # Each list index corresponds to a bit position, and its respective value is the rolling count of active bits from + # the previous boundary block up to this block, including it. LSB is on the left. + feature_activation_bit_counts: Optional[list[int]] + # It must be a weakref. _tx_ref: Optional['ReferenceType[BaseTransaction]'] @@ -51,8 +56,16 @@ class TransactionMetadata: _last_voided_by_hash: Optional[int] _last_spent_by_hash: Optional[int] - def __init__(self, spent_outputs: Optional[dict[int, list[bytes]]] = None, hash: Optional[bytes] = None, - accumulated_weight: float = 0, score: float = 0, height: int = 0, min_height: int = 0) -> None: + def __init__( + self, + spent_outputs: Optional[dict[int, list[bytes]]] = None, + hash: Optional[bytes] = None, + accumulated_weight: float = 0, + score: float = 0, + height: int = 0, + min_height: int = 0, + feature_activation_bit_counts: Optional[list[int]] = None + ) -> None: from hathor.transaction.genesis import is_genesis # Hash of the transaction. @@ -107,6 +120,8 @@ def __init__(self, spent_outputs: Optional[dict[int, list[bytes]]] = None, hash: # Validation self.validation = ValidationState.INITIAL + self.feature_activation_bit_counts = feature_activation_bit_counts + # Genesis specific: if hash is not None and is_genesis(hash): self.validation = ValidationState.FULL @@ -168,7 +183,7 @@ def __eq__(self, other: Any) -> bool: return False for field in ['hash', 'conflict_with', 'voided_by', 'received_by', 'children', 'accumulated_weight', 'twins', 'score', - 'first_block', 'validation', 'min_height']: + 'first_block', 'validation', 'min_height', 'feature_activation_bit_counts']: if (getattr(self, field) or None) != (getattr(other, field) or None): return False @@ -203,6 +218,7 @@ def to_json(self) -> dict[str, Any]: data['score'] = self.score data['height'] = self.height data['min_height'] = self.min_height + data['feature_activation_bit_counts'] = self.feature_activation_bit_counts if self.first_block is not None: data['first_block'] = self.first_block.hex() else: @@ -252,6 +268,7 @@ def create_from_json(cls, data: dict[str, Any]) -> 'TransactionMetadata': meta.score = data.get('score', 0) meta.height = data.get('height', 0) # XXX: should we calculate the height if it's not defined? meta.min_height = data.get('min_height', 0) + meta.feature_activation_bit_counts = data.get('feature_activation_bit_counts', []) first_block_raw = data.get('first_block', None) if first_block_raw: diff --git a/hathor/utils/int.py b/hathor/utils/int.py new file mode 100644 index 000000000..1492abd82 --- /dev/null +++ b/hathor/utils/int.py @@ -0,0 +1,54 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + + +def get_bit_list(n: int, min_size: Optional[int] = None) -> list[int]: + """ + Returns a list of bits corresponding to a non-negative number, with LSB on the left. + + Args: + n: the number + min_size: if set, pads the returned list with zeroes until it reaches min_size + + >>> get_bit_list(0b0) + [] + >>> get_bit_list(0b1) + [1] + >>> get_bit_list(0b10) + [0, 1] + >>> get_bit_list(0b111001010) + [0, 1, 0, 1, 0, 0, 1, 1, 1] + >>> get_bit_list(0b0, min_size=4) + [0, 0, 0, 0] + >>> get_bit_list(0b1, min_size=3) + [1, 0, 0] + >>> get_bit_list(0b10, min_size=1) + [0, 1] + >>> get_bit_list(0b111001010, min_size=10) + [0, 1, 0, 1, 0, 0, 1, 1, 1, 0] + """ + assert n >= 0 + bits = [] + + while n > 0: + bits.append(n & 1) + n >>= 1 + + if min_size is not None: + while len(bits) < min_size: + bits.append(0) + + return bits diff --git a/tests/feature_activation/test_feature_service.py b/tests/feature_activation/test_feature_service.py index 849fa8b04..d31aad08d 100644 --- a/tests/feature_activation/test_feature_service.py +++ b/tests/feature_activation/test_feature_service.py @@ -16,6 +16,7 @@ import pytest +from hathor.conf import HathorSettings from hathor.feature_activation import feature_service from hathor.feature_activation.feature import Feature from hathor.feature_activation.feature_service import FeatureService @@ -24,13 +25,14 @@ from hathor.feature_activation.model.feature_state import FeatureState from hathor.feature_activation.settings import Settings from hathor.transaction import Block +from hathor.transaction.storage import TransactionStorage @pytest.fixture def block_mocks() -> list[Block]: - mocks: list[Block] = [] + blocks: list[Block] = [] feature_activation_bits = [ - 0b0110, # 0: boundary block + 0b0000, # 0: boundary block 0b0010, 0b0110, 0b0010, @@ -60,15 +62,20 @@ def block_mocks() -> list[Block]: ] for i, bits in enumerate(feature_activation_bits): - mock = Mock(spec_set=Block) - mocks.append(mock) + settings = HathorSettings() + genesis_hash = settings.GENESIS_BLOCK_HASH + block_hash = genesis_hash if i == 0 else b'some_hash' - mock.is_genesis = i == 0 - mock.get_height = Mock(return_value=i) - mock.get_block_parent = Mock(return_value=mocks[i - 1]) - mock.get_feature_activation_bits = Mock(return_value=bits) + storage = Mock(spec_set=TransactionStorage) + storage.get_metadata = Mock(return_value=None) - return mocks + block = Block(hash=block_hash, storage=storage, signal_bits=bits) + blocks.append(block) + + get_block_parent_mock = Mock(return_value=blocks[i - 1]) + setattr(block, 'get_block_parent', get_block_parent_mock) + + return blocks @pytest.fixture @@ -385,50 +392,6 @@ def get_state(self: FeatureService, *, block: Block, feature: Feature) -> Featur assert result == expected -def test_get_bit_count_genesis(block_mocks: list[Block], service: FeatureService) -> None: - block = block_mocks[0] - - with pytest.raises(AssertionError) as e: - service.get_bit_count(boundary_block=block, bit=Mock()) - - assert str(e.value) == 'cannot calculate bit count for genesis' - - -@pytest.mark.parametrize('block_height', [1, 2, 3, 5, 18, 21]) -def test_get_bit_count_invalid(block_mocks: list[Block], service: FeatureService, block_height: int) -> None: - block = block_mocks[block_height] - - with pytest.raises(AssertionError) as e: - service.get_bit_count(boundary_block=block, bit=Mock()) - - assert str(e.value) == 'cannot calculate bit count for a non-boundary block' - - -@pytest.mark.parametrize( - ['block_height', 'bit', 'expected_count'], - [ - (4, 0, 0), - (4, 1, 4), - (4, 2, 2), - - (8, 0, 4), - (8, 1, 3), - (8, 2, 0), - ] -) -def test_get_bit_count( - block_mocks: list[Block], - service: FeatureService, - block_height: int, - bit: int, - expected_count: int -) -> None: - block = block_mocks[block_height] - result = service.get_bit_count(boundary_block=block, bit=bit) - - assert result == expected_count - - @pytest.mark.parametrize( ['block_height', 'ancestor_height'], [ diff --git a/tests/resources/transaction/test_mining.py b/tests/resources/transaction/test_mining.py index 6b77c8ded..885abfebf 100644 --- a/tests/resources/transaction/test_mining.py +++ b/tests/resources/transaction/test_mining.py @@ -39,6 +39,7 @@ def test_get_block_template_with_address(self): 'height': 1, 'min_height': 0, 'first_block': None, + 'feature_activation_bit_counts': [0, 0, 0, 0] }, 'tokens': [], 'data': '', @@ -70,6 +71,7 @@ def test_get_block_template_without_address(self): 'height': 1, 'min_height': 0, 'first_block': None, + 'feature_activation_bit_counts': [0, 0, 0, 0] }, 'tokens': [], 'data': '', diff --git a/tests/tx/test_block.py b/tests/tx/test_block.py index 0b64827c8..3ef6ce990 100644 --- a/tests/tx/test_block.py +++ b/tests/tx/test_block.py @@ -16,10 +16,79 @@ import pytest +from hathor.conf import HathorSettings from hathor.transaction import Block, TransactionMetadata +from hathor.transaction.genesis import BLOCK_GENESIS from hathor.transaction.storage import TransactionStorage +def test_calculate_feature_activation_bit_counts_genesis(): + result = BLOCK_GENESIS.calculate_feature_activation_bit_counts() + + assert result == [0, 0, 0, 0] + + +@pytest.fixture +def block_mocks() -> list[Block]: + blocks: list[Block] = [] + feature_activation_bits = [ + 0b0000, # 0: boundary block + 0b1010, + 0b1110, + 0b1110, + + 0b0011, # 4: boundary block + 0b0111, + 0b1111, + 0b0101, + + 0b0000, # 8: boundary block + 0b0000, + ] + + for i, bits in enumerate(feature_activation_bits): + settings = HathorSettings() + genesis_hash = settings.GENESIS_BLOCK_HASH + block_hash = genesis_hash if i == 0 else b'some_hash' + + storage = Mock(spec_set=TransactionStorage) + storage.get_metadata = Mock(return_value=None) + + block = Block(hash=block_hash, storage=storage, signal_bits=bits) + blocks.append(block) + + get_block_parent_mock = Mock(return_value=blocks[i - 1]) + setattr(block, 'get_block_parent', get_block_parent_mock) + + return blocks + + +@pytest.mark.parametrize( + ['block_height', 'expected_counts'], + [ + (0, [0, 0, 0, 0]), + (1, [0, 1, 0, 1]), + (2, [0, 2, 1, 2]), + (3, [0, 3, 2, 3]), + (4, [1, 1, 0, 0]), + (5, [2, 2, 1, 0]), + (6, [3, 3, 2, 1]), + (7, [4, 3, 3, 1]), + (8, [0, 0, 0, 0]), + (9, [0, 0, 0, 0]), + ] +) +def test_calculate_feature_activation_bit_counts( + block_mocks: list[Block], + block_height: int, + expected_counts: list[int] +) -> None: + block = block_mocks[block_height] + result = block.calculate_feature_activation_bit_counts() + + assert result == expected_counts + + def test_get_height(): block_hash = b'some_hash' block_height = 10 @@ -34,18 +103,19 @@ def test_get_height(): @pytest.mark.parametrize( - ['signal_bits', 'expected_bits'], + ['signal_bits', 'expected_bit_list'], [ - (0x00, 0), - (0x01, 1), - (0xF1, 1), - (0x06, 6), - (0xF6, 6), - (0x0F, 0xF), - (0xFF, 0xF), + (0x00, [0, 0, 0, 0]), # 0 + (0x01, [1, 0, 0, 0]), # 1 + (0xF1, [1, 0, 0, 0]), # 1 + (0x07, [1, 1, 1, 0]), # 7 + (0xF7, [1, 1, 1, 0]), # 7 + (0x0F, [1, 1, 1, 1]), # 0xF + (0xFF, [1, 1, 1, 1]), # 0xF ] ) -def test_get_feature_activation_bits(signal_bits: int, expected_bits: int) -> None: +def test_get_feature_activation_bit_list(signal_bits: int, expected_bit_list: list[int]) -> None: block = Block(signal_bits=signal_bits) + result = block._get_feature_activation_bit_list() - assert block.get_feature_activation_bits() == expected_bits + assert result == expected_bit_list