diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index 574d0c968..d1cc73494 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -14,14 +14,15 @@ import os from math import log -from typing import Any, Dict, List, NamedTuple, Optional, Union +from pathlib import Path +from typing import NamedTuple, Optional, Union import pydantic from hathor.checkpoint import Checkpoint from hathor.feature_activation.settings import Settings as FeatureActivationSettings from hathor.utils import yaml -from hathor.utils.pydantic import BaseModel +from hathor.utils.named_tuple import validated_named_tuple_from_dict DECIMAL_PLACES = 2 @@ -40,7 +41,7 @@ class HathorSettings(NamedTuple): NETWORK_NAME: str # Initial bootstrap servers - BOOTSTRAP_DNS: List[str] = [] + BOOTSTRAP_DNS: list[str] = [] # enable peer whitelist ENABLE_PEER_WHITELIST: bool = False @@ -243,7 +244,7 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: TOKEN_DEPOSIT_PERCENTAGE: float = 0.01 # Array with the settings parameters that are used when calculating the settings hash - P2P_SETTINGS_HASH_FIELDS: List[str] = [ + P2P_SETTINGS_HASH_FIELDS: list[str] = [ 'P2PKH_VERSION_BYTE', 'MULTISIG_VERSION_BYTE', 'MIN_BLOCK_WEIGHT', @@ -359,13 +360,13 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: METRICS_COLLECT_ROCKSDB_DATA_INTERVAL: int = 86400 # 1 day # Block checkpoints - CHECKPOINTS: List[Checkpoint] = [] + CHECKPOINTS: list[Checkpoint] = [] # Used on testing to enable slow asserts that help catch bugs but we don't want to run in production SLOW_ASSERTS: bool = False # List of soft voided transaction. - SOFT_VOIDED_TX_IDS: List[bytes] = [] + SOFT_VOIDED_TX_IDS: list[bytes] = [] # Identifier used in metadata's voided_by to mark a tx as soft-voided. SOFT_VOIDED_ID: bytes = b'tx-non-grata' @@ -394,30 +395,24 @@ def MAXIMUM_NUMBER_OF_HALVINGS(self) -> int: @classmethod def from_yaml(cls, *, filepath: str) -> 'HathorSettings': """Takes a filepath to a yaml file and returns a validated HathorSettings instance.""" - settings_dict = yaml.dict_from(filepath=filepath) + settings_dict = yaml.dict_from_extended_yaml(filepath=filepath, custom_root=Path(__file__).parent) - return HathorSettings.from_dict(settings_dict) - - @classmethod - def from_dict(cls, settings: Dict[str, Any]) -> 'HathorSettings': - """Takes a settings dict and returns a validated HathorSettings instance.""" - # This intermediate step shouldn't be necessary, but for some reason pydantic.create_model_from_namedtuple - # doesn't support default attribute values, so we do this to add them - all_settings = HathorSettings(**settings) - validated_settings = _ValidatedHathorSettings(**all_settings._asdict()) - - return HathorSettings(**validated_settings.dict()) + return validated_named_tuple_from_dict( + HathorSettings, + settings_dict, + validators=_VALIDATORS + ) -def _parse_checkpoints(checkpoints: Union[Dict[int, str], List[Checkpoint]]) -> List[Checkpoint]: +def _parse_checkpoints(checkpoints: Union[dict[int, str], list[Checkpoint]]) -> list[Checkpoint]: """Parse a dictionary of raw checkpoint data into a list of checkpoints.""" - if isinstance(checkpoints, Dict): + if isinstance(checkpoints, dict): return [ Checkpoint(height, bytes.fromhex(_hash)) for height, _hash in checkpoints.items() ] - if not isinstance(checkpoints, List): + if not isinstance(checkpoints, list): raise TypeError(f'expected \'Dict[int, str]\' or \'List[Checkpoint]\', got {checkpoints}') return checkpoints @@ -434,29 +429,25 @@ def _parse_hex_str(hex_str: Union[str, bytes]) -> bytes: return hex_str -_ValidatedHathorSettings = pydantic.create_model_from_namedtuple( - HathorSettings, - __base__=BaseModel, - __validators__=dict( - _parse_hex_str=pydantic.validator( - 'P2PKH_VERSION_BYTE', - 'MULTISIG_VERSION_BYTE', - 'GENESIS_OUTPUT_SCRIPT', - 'GENESIS_BLOCK_HASH', - 'GENESIS_TX1_HASH', - 'GENESIS_TX2_HASH', - pre=True, - allow_reuse=True - )(_parse_hex_str), - _parse_soft_voided_tx_id=pydantic.validator( - 'SOFT_VOIDED_TX_IDS', - pre=True, - allow_reuse=True, - each_item=True - )(_parse_hex_str), - _parse_checkpoints=pydantic.validator( - 'CHECKPOINTS', - pre=True - )(_parse_checkpoints) - ) +_VALIDATORS = dict( + _parse_hex_str=pydantic.validator( + 'P2PKH_VERSION_BYTE', + 'MULTISIG_VERSION_BYTE', + 'GENESIS_OUTPUT_SCRIPT', + 'GENESIS_BLOCK_HASH', + 'GENESIS_TX1_HASH', + 'GENESIS_TX2_HASH', + pre=True, + allow_reuse=True + )(_parse_hex_str), + _parse_soft_voided_tx_id=pydantic.validator( + 'SOFT_VOIDED_TX_IDS', + pre=True, + allow_reuse=True, + each_item=True + )(_parse_hex_str), + _parse_checkpoints=pydantic.validator( + 'CHECKPOINTS', + pre=True + )(_parse_checkpoints) ) diff --git a/hathor/utils/dict.py b/hathor/utils/dict.py new file mode 100644 index 000000000..b3c988150 --- /dev/null +++ b/hathor/utils/dict.py @@ -0,0 +1,45 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy + + +def deep_merge(first_dict: dict, second_dict: dict) -> dict: + """ + Recursively merges two dicts, returning a new one with the merged values. Keeps both input dicts intact. + + Note: will raise RecursionError if there's a circular reference in both dicts. + + >>> dict1 = dict(a=1, b=dict(c=2, d=3), e=dict(f=4)) + >>> dict2 = dict(b=dict(d=5, e=6), e=7) + >>> result = deep_merge(dict1, dict2) + >>> result == dict(a=1, b=dict(c=2, d=5, e=6), e=7) + True + >>> dict1 == dict(a=1, b=dict(c=2, d=3), e=dict(f=4)) + True + >>> dict2 == dict(b=dict(d=5, e=6), e=7) + True + """ + merged = deepcopy(first_dict) + + def do_deep_merge(first: dict, second: dict) -> dict: + for key in second: + if key in first and isinstance(first[key], dict) and isinstance(second[key], dict): + do_deep_merge(first[key], second[key]) + else: + first[key] = second[key] + + return first + + return do_deep_merge(merged, second_dict) diff --git a/hathor/utils/named_tuple.py b/hathor/utils/named_tuple.py new file mode 100644 index 000000000..2745cf3d9 --- /dev/null +++ b/hathor/utils/named_tuple.py @@ -0,0 +1,53 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, NamedTuple, Optional, Type, TypeVar + +import pydantic + +from hathor.utils.pydantic import BaseModel + +T = TypeVar('T', bound=NamedTuple) + + +def validated_named_tuple_from_dict( + named_tuple_type: Type[T], + attributes_dict: dict[str, Any], + *, + validators: Optional[dict[str, classmethod]] = None +) -> T: + """ + Takes an attributes dict and returns a validated instance of the specified NamedTuple subclass. + Performs validation using pydantic. + + Args: + named_tuple_type: the NamedTuple subclass to create an instance from + attributes_dict: a dict with all required attributes for the NamedTuple subclass + validators: custom pydantic validators (read https://docs.pydantic.dev/latest/usage/validators) + + Returns: a validated instance of the specified NamedTuple subclass + """ + model = pydantic.create_model_from_namedtuple( + named_tuple_type, + __base__=BaseModel, + __validators__=validators + ) + + # This intermediate step shouldn't be necessary, but for some reason pydantic.create_model_from_namedtuple + # doesn't support default attribute values, so we do this to add them + all_attributes = named_tuple_type(**attributes_dict) + validated_attributes = model(**all_attributes._asdict()) + validated_attributes_dict = {k: v for k, v in validated_attributes} + + return named_tuple_type(**validated_attributes_dict) diff --git a/hathor/utils/yaml.py b/hathor/utils/yaml.py index ffaef57a1..3c1794af8 100644 --- a/hathor/utils/yaml.py +++ b/hathor/utils/yaml.py @@ -12,12 +12,61 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict +import os +from pathlib import Path +from typing import Any, Optional, Union import yaml +from hathor.utils.dict import deep_merge -def dict_from(*, filepath: str) -> Dict[str, Any]: +_EXTENDS_KEY = 'extends' + + +def dict_from_yaml(*, filepath: Union[Path, str]) -> dict[str, Any]: """Takes a filepath to a yaml file and returns a dictionary with its contents.""" + if not os.path.isfile(filepath): + raise ValueError(f"'{filepath}' is not a file") + with open(filepath, 'r') as file: - return yaml.safe_load(file) + contents = yaml.safe_load(file) + + if contents is None: + return {} + + if not isinstance(contents, dict): + raise ValueError(f"'{filepath}' cannot be parsed as a dictionary") + + return contents + + +def dict_from_extended_yaml(*, filepath: Union[Path, str], custom_root: Optional[Path] = None) -> dict[str, Any]: + """ + Takes a filepath to a yaml file and returns a dictionary with its contents. + + Supports extending another yaml file via the 'extends' key in the file. The 'extends' value can be an absolute path + to a yaml file, or a path relative to the base yaml file. The custom_root arg can be provided to set a custom root + for relative paths, taking lower precedence. + + Note: the 'extends' key is reserved and will not be present in the returned dictionary. + To opt-out of the extension feature, use dict_from_yaml(). + """ + extension_dict = dict_from_yaml(filepath=filepath) + file_to_extend = extension_dict.pop(_EXTENDS_KEY, None) + + if not file_to_extend: + return extension_dict + + filepath_to_extend = Path(filepath).parent / str(file_to_extend) + + if not os.path.isfile(filepath_to_extend) and custom_root: + filepath_to_extend = custom_root / str(file_to_extend) + + try: + dict_to_extend = dict_from_extended_yaml(filepath=filepath_to_extend, custom_root=custom_root) + except RecursionError as e: + raise ValueError('Cannot parse yaml with recursive extensions.') from e + + extended_dict = deep_merge(dict_to_extend, extension_dict) + + return extended_dict diff --git a/tests/utils_modules/__init__.py b/tests/utils_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils_modules/fixtures/empty.yml b/tests/utils_modules/fixtures/empty.yml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils_modules/fixtures/empty_extends.yml b/tests/utils_modules/fixtures/empty_extends.yml new file mode 100644 index 000000000..48a29feec --- /dev/null +++ b/tests/utils_modules/fixtures/empty_extends.yml @@ -0,0 +1,6 @@ +extends: + +a: aa +b: + d: dd + e: ee diff --git a/tests/utils_modules/fixtures/invalid_extends.yml b/tests/utils_modules/fixtures/invalid_extends.yml new file mode 100644 index 000000000..2046795e5 --- /dev/null +++ b/tests/utils_modules/fixtures/invalid_extends.yml @@ -0,0 +1,6 @@ +extends: ./unknown_file.yml + +a: aa +b: + d: dd + e: ee diff --git a/tests/utils_modules/fixtures/mainnet_extends.yml b/tests/utils_modules/fixtures/mainnet_extends.yml new file mode 100644 index 000000000..0b1dbbb63 --- /dev/null +++ b/tests/utils_modules/fixtures/mainnet_extends.yml @@ -0,0 +1,6 @@ +extends: mainnet.yml + +a: aa +b: + d: dd + e: ee diff --git a/tests/utils_modules/fixtures/number.yml b/tests/utils_modules/fixtures/number.yml new file mode 100644 index 000000000..190a18037 --- /dev/null +++ b/tests/utils_modules/fixtures/number.yml @@ -0,0 +1 @@ +123 diff --git a/tests/utils_modules/fixtures/self_extends.yml b/tests/utils_modules/fixtures/self_extends.yml new file mode 100644 index 000000000..d6b41e538 --- /dev/null +++ b/tests/utils_modules/fixtures/self_extends.yml @@ -0,0 +1,6 @@ +extends: self_extends.yml + +a: aa +b: + d: dd + e: ee diff --git a/tests/utils_modules/fixtures/valid.yml b/tests/utils_modules/fixtures/valid.yml new file mode 100644 index 000000000..c54a94253 --- /dev/null +++ b/tests/utils_modules/fixtures/valid.yml @@ -0,0 +1,4 @@ +a: 1 +b: + c: 2 + d: 3 diff --git a/tests/utils_modules/fixtures/valid_extends.yml b/tests/utils_modules/fixtures/valid_extends.yml new file mode 100644 index 000000000..bd73de21e --- /dev/null +++ b/tests/utils_modules/fixtures/valid_extends.yml @@ -0,0 +1,6 @@ +extends: valid.yml + +a: aa +b: + d: dd + e: ee diff --git a/tests/utils_modules/test_named_tuple.py b/tests/utils_modules/test_named_tuple.py new file mode 100644 index 000000000..e56aaf8f4 --- /dev/null +++ b/tests/utils_modules/test_named_tuple.py @@ -0,0 +1,89 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import NamedTuple + +import pydantic +import pytest +from pydantic import ValidationError + +from hathor.utils.named_tuple import validated_named_tuple_from_dict +from hathor.utils.pydantic import BaseModel + + +class InnerTuple(NamedTuple): + x: str + + +class InnerModel(BaseModel): + y: str + + +class OuterTuple(NamedTuple): + a: int + b: InnerTuple + c: InnerModel + + @classmethod + def validate_a(cls, a: int) -> int: + if a > 10: + raise ValueError('"a" cannot be greater than 10') + + return a + + +VALIDATORS = dict( + validate_a=pydantic.validator('a')(OuterTuple.validate_a) +) + + +@pytest.mark.parametrize( + ['attributes', 'expected'], + [ + ( + dict(a=0, b=('b',), c=dict(y='c')), + OuterTuple(a=0, b=InnerTuple(x='b'), c=InnerModel(y='c')) + ), + ( + dict(a=5, b=('bb',), c=dict(y='cc')), + OuterTuple(a=5, b=InnerTuple(x='bb'), c=InnerModel(y='cc')) + ), + ( + dict(a=10, b=('bbb',), c=dict(y='ccc')), + OuterTuple(a=10, b=InnerTuple(x='bbb'), c=InnerModel(y='ccc')) + ), + ] +) +def test_validated_named_tuple_from_dict(attributes, expected): + result = validated_named_tuple_from_dict(OuterTuple, attributes, validators=VALIDATORS) + + assert isinstance(result.b, InnerTuple) + assert isinstance(result.c, InnerModel) + assert result == expected + + +@pytest.mark.parametrize( + 'attributes', + [ + dict(a=11, b=('b',), c=dict(y='c')), + dict(a=50, b=('bb',), c=dict(y='cc')), + dict(a=100, b=('bbb',), c=dict(y='ccc')), + ] +) +def test_validated_named_tuple_from_dict_error(attributes): + with pytest.raises(ValidationError) as e: + validated_named_tuple_from_dict(OuterTuple, attributes, validators=VALIDATORS) + + errors = e.value.errors() + assert errors[0]['msg'] == '"a" cannot be greater than 10' diff --git a/tests/utils_modules/test_yaml.py b/tests/utils_modules/test_yaml.py new file mode 100644 index 000000000..7ef925b1d --- /dev/null +++ b/tests/utils_modules/test_yaml.py @@ -0,0 +1,127 @@ +# Copyright 2023 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from hathor.utils.yaml import dict_from_extended_yaml, dict_from_yaml + + +def _get_absolute_filepath(filepath: str) -> Path: + parent_dir = Path(__file__).parent + + return parent_dir / filepath + + +def test_dict_from_yaml_invalid_filepath(): + with pytest.raises(ValueError) as e: + dict_from_yaml(filepath='fake_file.yml') + + assert str(e.value) == "'fake_file.yml' is not a file" + + +def test_dict_from_yaml_empty(): + filepath = _get_absolute_filepath('fixtures/empty.yml') + result = dict_from_yaml(filepath=filepath) + + assert result == {} + + +def test_dict_from_yaml_invalid_contents(): + filepath = _get_absolute_filepath('fixtures/number.yml') + + with pytest.raises(ValueError) as e: + dict_from_yaml(filepath=filepath) + + assert str(e.value) == f"'{filepath}' cannot be parsed as a dictionary" + + +def test_dict_from_yaml_valid(): + filepath = _get_absolute_filepath('fixtures/valid.yml') + result = dict_from_yaml(filepath=filepath) + + assert result == dict(a=1, b=dict(c=2, d=3)) + + +def test_dict_from_extended_yaml_invalid_filepath(): + with pytest.raises(ValueError) as e: + dict_from_extended_yaml(filepath='fake_file.yml') + + assert str(e.value) == "'fake_file.yml' is not a file" + + +def test_dict_from_extended_yaml_empty(): + filepath = _get_absolute_filepath('fixtures/empty.yml') + result = dict_from_extended_yaml(filepath=filepath) + + assert result == {} + + +def test_dict_from_extended_yaml_invalid_contents(): + filepath = _get_absolute_filepath('fixtures/number.yml') + + with pytest.raises(ValueError) as e: + dict_from_extended_yaml(filepath=filepath) + + assert str(e.value) == f"'{filepath}' cannot be parsed as a dictionary" + + +def test_dict_from_extended_yaml_valid(): + filepath = _get_absolute_filepath('fixtures/valid.yml') + result = dict_from_extended_yaml(filepath=filepath) + + assert result == dict(a=1, b=dict(c=2, d=3)) + + +def test_dict_from_extended_yaml_empty_extends(): + filepath = _get_absolute_filepath('fixtures/empty_extends.yml') + result = dict_from_extended_yaml(filepath=filepath) + + assert result == dict(a='aa', b=dict(d='dd', e='ee')) + + +def test_dict_from_extended_yaml_invalid_extends(): + filepath = _get_absolute_filepath('fixtures/invalid_extends.yml') + + with pytest.raises(ValueError) as e: + dict_from_extended_yaml(filepath=filepath) + + assert "unknown_file.yml' is not a file" in str(e.value) + + +def test_dict_from_extended_yaml_recursive_extends(): + filepath = _get_absolute_filepath('fixtures/self_extends.yml') + + with pytest.raises(ValueError) as e: + dict_from_extended_yaml(filepath=filepath) + + assert str(e.value) == 'Cannot parse yaml with recursive extensions.' + + +def test_dict_from_extended_yaml_valid_extends(): + filepath = _get_absolute_filepath('fixtures/valid_extends.yml') + result = dict_from_extended_yaml(filepath=filepath) + + assert result == dict(a='aa', b=dict(c=2, d='dd', e='ee')) + + +def test_dict_from_yaml_mainnet_extends(): + from hathor.conf import MAINNET_SETTINGS_FILEPATH + + filepath = _get_absolute_filepath('fixtures/mainnet_extends.yml') + mainnet_dict = dict_from_yaml(filepath=MAINNET_SETTINGS_FILEPATH) + result = dict_from_extended_yaml(filepath=filepath, custom_root=Path(MAINNET_SETTINGS_FILEPATH).parent) + + assert result == dict(**mainnet_dict, a='aa', b=dict(d='dd', e='ee'))