feat(feature-activation): implement signal_bits field in txs

hathor/transaction/base_transaction.py

@@ -67,9 +67,8 @@
 # H = unsigned short (2 bytes), d = double(8), f = float(4), I = unsigned int (4),
 # Q = unsigned long long int (64), B = unsigned char (1 byte)
 
-# Version (H), inputs len (B), and outputs len (B), token uids len (B).
-# H = unsigned short (2 bytes)
-_SIGHASH_ALL_FORMAT_STRING = '!HBBB'
+# Signal bits (B), version (B), inputs len (B), and outputs len (B), token uids len (B).
+_SIGHASH_ALL_FORMAT_STRING = '!BBBBB'
 
 # Weight (d), timestamp (I), and parents len (B)
 _GRAPH_FORMAT_STRING = '!dIB'
@@ -82,6 +81,9 @@
 _BLOCK_PARENTS_TXS = 2
 _BLOCK_PARENTS_BLOCKS = 1
 
+# The int value of one byte
+_ONE_BYTE = 0xFF
+
 
 def sum_weights(w1: float, w2: float) -> float:
     return aux_calc_weight(w1, w2, 1)
@@ -111,14 +113,11 @@ class TxVersion(IntEnum):
     MERGE_MINED_BLOCK = 3
 
     @classmethod
-    def _missing_(cls, value: Any) -> 'TxVersion':
-        # version's first byte is reserved for future use, so we'll ignore it
-        assert isinstance(value, int)
-        version = value & 0xFF
-        if version == value:
-            # Prevent infinite recursion when starting TxVerion with wrong version
-            raise ValueError('Invalid version.')
-        return cls(version)
+    def _missing_(cls, value: Any) -> None:
+        assert isinstance(value, int), f"Value '{value}' must be an integer"
+        assert value <= _ONE_BYTE, f'Value {hex(value)} must not be larger than one byte'
+
+        raise ValueError(f'Invalid version: {value}')
 
     def get_cls(self) -> Type['BaseTransaction']:
         from hathor.transaction.block import Block
@@ -155,9 +154,16 @@ class BaseTransaction(ABC):
 
     _metadata: Optional[TransactionMetadata]
 
+    # Bits extracted from the first byte of the version field. They carry extra information that may be interpreted
+    # differently by each subclass of BaseTransaction.
+    # Currently only the Block subclass uses it, carrying information about Feature Activation bits and also extra
+    # bits reserved for future use, depending on the configuration.
+    signal_bits: int
+
     def __init__(self,
                  nonce: int = 0,
                  timestamp: Optional[int] = None,
+                 signal_bits: int = 0,
                  version: int = TxVersion.REGULAR_BLOCK,
                  weight: float = 0,
                  inputs: Optional[List['TxInput']] = None,
@@ -168,13 +174,18 @@ def __init__(self,
         """
             Nonce: nonce used for the proof-of-work
             Timestamp: moment of creation
+            Signal bits: bits used to carry extra information that may be interpreted differently by each subclass
             Version: version when it was created
             Weight: different for transactions and blocks
             Outputs: all outputs that are being created
             Parents: transactions you are confirming (2 transactions and 1 block - in case of a block only)
         """
+        assert signal_bits <= _ONE_BYTE, f'signal_bits {hex(signal_bits)} must not be larger than one byte'
+        assert version <= _ONE_BYTE, f'version {hex(version)} must not be larger than one byte'
+
         self.nonce = nonce
         self.timestamp = timestamp or int(time.time())
+        self.signal_bits = signal_bits
         self.version = version
         self.weight = weight
         self.inputs = inputs or []
@@ -1354,8 +1365,8 @@ def tx_or_block_from_bytes(data: bytes,
                            storage: Optional['TransactionStorage'] = None) -> BaseTransaction:
     """ Creates the correct tx subclass from a sequence of bytes
     """
-    # version field takes up the first 2 bytes
-    version = int.from_bytes(data[0:2], 'big')
+    # version field takes up the second byte only
+    version = data[1]
     try:
         tx_version = TxVersion(version)
         cls = tx_version.get_cls()

hathor/transaction/block.py

@@ -38,11 +38,11 @@
 settings = HathorSettings()
 cpu = get_cpu_profiler()
 
-# Version (H), outputs len (B)
-_FUNDS_FORMAT_STRING = '!HB'
+# Signal bits (B), version (B), outputs len (B)
+_FUNDS_FORMAT_STRING = '!BBB'
 
-# Version (H), inputs len (B) and outputs len (B)
-_SIGHASH_ALL_FORMAT_STRING = '!HBB'
+# Signal bits (B), version (B), inputs len (B) and outputs len (B)
+_SIGHASH_ALL_FORMAT_STRING = '!BBBB'
 
 
 class Block(BaseTransaction):
@@ -51,14 +51,15 @@ class Block(BaseTransaction):
     def __init__(self,
                  nonce: int = 0,
                  timestamp: Optional[int] = None,
+                 signal_bits: int = 0,
                  version: int = TxVersion.REGULAR_BLOCK,
                  weight: float = 0,
                  outputs: Optional[List[TxOutput]] = None,
                  parents: Optional[List[bytes]] = None,
                  hash: Optional[bytes] = None,
                  data: bytes = b'',
                  storage: Optional['TransactionStorage'] = None) -> None:
-        super().__init__(nonce=nonce, timestamp=timestamp, version=version, weight=weight,
+        super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight,
                          outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
         self.data = data
 
@@ -165,8 +166,9 @@ def get_funds_fields_from_struct(self, buf: bytes, *, verbose: VerboseCallback =
 
         :raises ValueError: when the sequence of bytes is incorect
         """
-        (self.version, outputs_len), buf = unpack(_FUNDS_FORMAT_STRING, buf)
+        (self.signal_bits, self.version, outputs_len), buf = unpack(_FUNDS_FORMAT_STRING, buf)
         if verbose:
+            verbose('signal_bits', self.signal_bits)
             verbose('version', self.version)
             verbose('outputs_len', outputs_len)
 
@@ -202,7 +204,7 @@ def get_funds_struct(self) -> bytes:
         :return: funds data serialization of the block
         :rtype: bytes
         """
-        struct_bytes = pack(_FUNDS_FORMAT_STRING, self.version, len(self.outputs))
+        struct_bytes = pack(_FUNDS_FORMAT_STRING, self.signal_bits, self.version, len(self.outputs))
 
         for tx_output in self.outputs:
             struct_bytes += bytes(tx_output)

hathor/transaction/merge_mined_block.py

@@ -27,6 +27,7 @@ class MergeMinedBlock(Block):
     def __init__(self,
                  nonce: int = 0,
                  timestamp: Optional[int] = None,
+                 signal_bits: int = 0,
                  version: int = TxVersion.MERGE_MINED_BLOCK,
                  weight: float = 0,
                  outputs: Optional[List[TxOutput]] = None,
@@ -35,8 +36,8 @@ def __init__(self,
                  data: bytes = b'',
                  aux_pow: Optional[BitcoinAuxPow] = None,
                  storage: Optional['TransactionStorage'] = None) -> None:
-        super().__init__(nonce=nonce, timestamp=timestamp, version=version, weight=weight, data=data,
-                         outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
+        super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight,
+                         data=data, outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
         self.aux_pow = aux_pow
 
     def _get_formatted_fields_dict(self, short: bool = True) -> Dict[str, str]:

hathor/transaction/token_creation_tx.py

@@ -24,11 +24,11 @@
 
 settings = HathorSettings()
 
-# Version (H), inputs len (B), outputs len (B)
-_FUNDS_FORMAT_STRING = '!HBB'
+# Signal bits (B), version (B), inputs len (B), outputs len (B)
+_FUNDS_FORMAT_STRING = '!BBBB'
 
-# Version (H), inputs len (B), outputs len (B)
-_SIGHASH_ALL_FORMAT_STRING = '!HBB'
+# Signal bist (B), version (B), inputs len (B), outputs len (B)
+_SIGHASH_ALL_FORMAT_STRING = '!BBBB'
 
 # used when (de)serializing token information
 # version 1 expects only token name and symbol
@@ -39,6 +39,7 @@ class TokenCreationTransaction(Transaction):
     def __init__(self,
                  nonce: int = 0,
                  timestamp: Optional[int] = None,
+                 signal_bits: int = 0,
                  version: int = TxVersion.TOKEN_CREATION_TRANSACTION,
                  weight: float = 0,
                  inputs: Optional[List[TxInput]] = None,
@@ -48,8 +49,8 @@ def __init__(self,
                  token_name: str = '',
                  token_symbol: str = '',
                  storage: Optional['TransactionStorage'] = None) -> None:
-        super().__init__(nonce=nonce, timestamp=timestamp, version=version, weight=weight, inputs=inputs,
-                         outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
+        super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight,
+                         inputs=inputs, outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
         self.token_name = token_name
         self.token_symbol = token_symbol
         # for this special tx, its own hash is used as the created token uid. We're artificially
@@ -85,8 +86,9 @@ def get_funds_fields_from_struct(self, buf: bytes, *, verbose: VerboseCallback =
 
         :raises ValueError: when the sequence of bytes is incorect
         """
-        (self.version, inputs_len, outputs_len), buf = unpack(_FUNDS_FORMAT_STRING, buf)
+        (self.signal_bits, self.version, inputs_len, outputs_len), buf = unpack(_FUNDS_FORMAT_STRING, buf)
         if verbose:
+            verbose('signal_bits', self.signal_bits)
             verbose('version', self.version)
             verbose('inputs_len', inputs_len)
             verbose('outputs_len', outputs_len)
@@ -110,7 +112,13 @@ def get_funds_struct(self) -> bytes:
         :return: funds data serialization of the transaction
         :rtype: bytes
         """
-        struct_bytes = pack(_FUNDS_FORMAT_STRING, self.version, len(self.inputs), len(self.outputs))
+        struct_bytes = pack(
+            _FUNDS_FORMAT_STRING,
+            self.signal_bits,
+            self.version,
+            len(self.inputs),
+            len(self.outputs)
+        )
 
         tx_inputs = []
         for tx_input in self.inputs:
@@ -135,7 +143,13 @@ def get_sighash_all(self) -> bytes:
         if self._sighash_cache:
             return self._sighash_cache
 
-        struct_bytes = pack(_SIGHASH_ALL_FORMAT_STRING, self.version, len(self.inputs), len(self.outputs))
+        struct_bytes = pack(
+            _SIGHASH_ALL_FORMAT_STRING,
+            self.signal_bits,
+            self.version,
+            len(self.inputs),
+            len(self.outputs)
+        )
 
         tx_inputs = []
         for tx_input in self.inputs:

hathor/transaction/transaction.py

@@ -49,11 +49,11 @@
 settings = HathorSettings()
 cpu = get_cpu_profiler()
 
-# Version (H), token uids len (B) and inputs len (B), outputs len (B).
-_FUNDS_FORMAT_STRING = '!HBBB'
+# Signal bits (B), version (B), token uids len (B) and inputs len (B), outputs len (B).
+_FUNDS_FORMAT_STRING = '!BBBBB'
 
-# Version (H), inputs len (B), and outputs len (B), token uids len (B).
-_SIGHASH_ALL_FORMAT_STRING = '!HBBB'
+# Signal bits (B), version (B), inputs len (B), and outputs len (B), token uids len (B).
+_SIGHASH_ALL_FORMAT_STRING = '!BBBBB'
 
 
 class TokenInfo(NamedTuple):
@@ -74,6 +74,7 @@ class Transaction(BaseTransaction):
     def __init__(self,
                  nonce: int = 0,
                  timestamp: Optional[int] = None,
+                 signal_bits: int = 0,
                  version: int = TxVersion.REGULAR_TRANSACTION,
                  weight: float = 0,
                  inputs: Optional[List[TxInput]] = None,
@@ -86,8 +87,8 @@ def __init__(self,
             Creating new init just to make sure inputs will always be empty array
             Inputs: all inputs that are being used (empty in case of a block)
         """
-        super().__init__(nonce=nonce, timestamp=timestamp, version=version, weight=weight, inputs=inputs
-                         or [], outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
+        super().__init__(nonce=nonce, timestamp=timestamp, signal_bits=signal_bits, version=version, weight=weight,
+                         inputs=inputs or [], outputs=outputs or [], parents=parents or [], hash=hash, storage=storage)
         self.tokens = tokens or []
         self._sighash_cache: Optional[bytes] = None
         self._sighash_data_cache: Optional[bytes] = None
@@ -166,8 +167,13 @@ def get_funds_fields_from_struct(self, buf: bytes, *, verbose: VerboseCallback =
 
         :raises ValueError: when the sequence of bytes is incorect
         """
-        (self.version, tokens_len, inputs_len, outputs_len), buf = unpack(_FUNDS_FORMAT_STRING, buf)
+        (self.signal_bits, self.version, tokens_len, inputs_len, outputs_len), buf = unpack(
+            _FUNDS_FORMAT_STRING,
+            buf
+        )
+
         if verbose:
+            verbose('signal_bits', self.signal_bits)
             verbose('version', self.version)
             verbose('tokens_len', tokens_len)
             verbose('inputs_len', inputs_len)
@@ -195,7 +201,14 @@ def get_funds_struct(self) -> bytes:
         :return: funds data serialization of the transaction
         :rtype: bytes
         """
-        struct_bytes = pack(_FUNDS_FORMAT_STRING, self.version, len(self.tokens), len(self.inputs), len(self.outputs))
+        struct_bytes = pack(
+            _FUNDS_FORMAT_STRING,
+            self.signal_bits,
+            self.version,
+            len(self.tokens),
+            len(self.inputs),
+            len(self.outputs)
+        )
 
         for token_uid in self.tokens:
             struct_bytes += token_uid
@@ -220,8 +233,16 @@ def get_sighash_all(self) -> bytes:
         if self._sighash_cache:
             return self._sighash_cache
 
-        struct_bytes = bytearray(pack(_SIGHASH_ALL_FORMAT_STRING, self.version, len(self.tokens), len(self.inputs),
-                                 len(self.outputs)))
+        struct_bytes = bytearray(
+            pack(
+                _SIGHASH_ALL_FORMAT_STRING,
+                self.signal_bits,
+                self.version,
+                len(self.tokens),
+                len(self.inputs),
+                len(self.outputs)
+            )
+        )
 
         for token_uid in self.tokens:
             struct_bytes += token_uid

tests/tx/test_tx.py

@@ -838,21 +838,52 @@ def test_output_value(self):
         with self.assertRaises(InvalidOutputValue):
             TxOutput(-1, script)
 
-    def test_tx_version(self):
+    def test_tx_version_and_signal_bits(self):
         from hathor.transaction.base_transaction import TxVersion
 
-        # test the 1st byte of version field is ignored
-        version = TxVersion(0xFF00)
+        # test invalid type
+        with self.assertRaises(AssertionError) as cm:
+            TxVersion('test')
+
+        self.assertEqual(str(cm.exception), "Value 'test' must be an integer")
+
+        # test one byte max value
+        with self.assertRaises(AssertionError) as cm:
+            TxVersion(0x100)
+
+        self.assertEqual(str(cm.exception), 'Value 0x100 must not be larger than one byte')
+
+        # test invalid version
+        with self.assertRaises(ValueError) as cm:
+            TxVersion(10)
+
+        self.assertEqual(str(cm.exception), 'Invalid version: 10')
+
+        # test get the correct class
+        version = TxVersion(0x00)
         self.assertEqual(version.get_cls(), Block)
-        version = TxVersion(0xFF01)
+        version = TxVersion(0x01)
         self.assertEqual(version.get_cls(), Transaction)
 
+        # test Block.__init__() fails
+        with self.assertRaises(AssertionError) as cm:
+            Block(signal_bits=0x100)
+
+        self.assertEqual(str(cm.exception), 'signal_bits 0x100 must not be larger than one byte')
+
+        with self.assertRaises(AssertionError) as cm:
+            Block(version=0x200)
+
+        self.assertEqual(str(cm.exception), 'version 0x200 must not be larger than one byte')
+
         # test serialization doesn't mess up with version
         block = Block(
-            version=0xFF00,
+            signal_bits=0xF0,
+            version=0x0F,
             nonce=100,
             weight=1)
         block2 = block.clone()
+        self.assertEqual(block.signal_bits, block2.signal_bits)
         self.assertEqual(block.version, block2.version)
 
     def test_output_sum_ignore_authority(self):

tests/tx/test_tx_deserialization.py

@@ -20,7 +20,7 @@ def verbose(key, value):
             tx = cls.create_from_struct(self.tx_bytes, verbose=verbose)
             tx.verify_without_storage()
 
-            key, version = v[0]
+            key, version = v[1]
             self.assertEqual(key, 'version')
 
             tx_version = TxVersion(version)