From 9819aebd68fb2ee6085d9745cf4773a27980fed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 27 Sep 2024 15:23:28 -0700 Subject: [PATCH] True TOML config support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- src/tox/config/loader/toml.py | 147 +++++++++++++++++++ src/tox/config/source/discover.py | 8 +- src/tox/config/source/toml.py | 107 ++++++++++++++ tests/config/loader/toml/test_toml_loader.py | 15 ++ tests/config/source/test_toml.py | 34 +++++ 5 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 src/tox/config/loader/toml.py create mode 100644 src/tox/config/source/toml.py create mode 100644 tests/config/loader/toml/test_toml_loader.py create mode 100644 tests/config/source/test_toml.py diff --git a/src/tox/config/loader/toml.py b/src/tox/config/loader/toml.py new file mode 100644 index 0000000000..0576eb9c06 --- /dev/null +++ b/src/tox/config/loader/toml.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Literal, + Mapping, + Set, + TypeAlias, + TypeVar, + Union, + cast, +) + +from tox.config.loader.api import Loader, Override +from tox.config.types import Command, EnvList +from tox.report import HandledError + +if TYPE_CHECKING: + from tox.config.loader.section import Section + from tox.config.main import Config + +if sys.version_info >= (3, 11): # pragma: no cover (py311+) + from typing import TypeGuard +else: # pragma: no cover (py311+) + from typing_extensions import TypeGuard +if sys.version_info >= (3, 10): # pragma: no cover (py310+) + from typing import TypeGuard +else: # pragma: no cover (py310+) + from typing_extensions import TypeAlias + +TomlTypes: TypeAlias = Dict[str, "TomlTypes"] | list["TomlTypes"] | str | int | float | bool | None + + +class TomlLoader(Loader[TomlTypes]): + """Load configuration from a pyproject.toml file.""" + + def __init__( + self, + section: Section, + overrides: list[Override], + content: Mapping[str, TomlTypes], + ) -> None: + if not isinstance(content, Mapping): + msg = f"tox.{section.key} must be a mapping" + raise HandledError(msg) + self.content = content + super().__init__(section, overrides) + + def load_raw(self, key: str, conf: Config | None, env_name: str | None) -> TomlTypes: # noqa: ARG002 + return self.content[key] + + def found_keys(self) -> set[str]: + return set(self.content.keys()) + + @staticmethod + def to_str(value: TomlTypes) -> str: + return _ensure_type_correct(value, str) # type: ignore[return-value] # no mypy support + + @staticmethod + def to_bool(value: TomlTypes) -> bool: + return _ensure_type_correct(value, bool) + + @staticmethod + def to_list(value: TomlTypes, of_type: type[Any]) -> Iterator[_T]: + of = List[of_type] # type: ignore[valid-type] # no mypy support + return iter(_ensure_type_correct(value, of)) # type: ignore[call-overload,no-any-return] + + @staticmethod + def to_set(value: TomlTypes, of_type: type[Any]) -> Iterator[_T]: + of = Set[of_type] # type: ignore[valid-type] # no mypy support + return iter(_ensure_type_correct(value, of)) # type: ignore[call-overload,no-any-return] + + @staticmethod + def to_dict(value: TomlTypes, of_type: tuple[type[Any], type[Any]]) -> Iterator[tuple[_T, _T]]: + of = Mapping[of_type[0], of_type[1]] # type: ignore[valid-type] # no mypy support + return _ensure_type_correct(value, of).items() # type: ignore[type-abstract,attr-defined,no-any-return] + + @staticmethod + def to_path(value: TomlTypes) -> Path: + return Path(TomlLoader.to_str(value)) + + @staticmethod + def to_command(value: TomlTypes) -> Command: + return Command(args=cast(list[str], value)) # validated during load in _ensure_type_correct + + @staticmethod + def to_env_list(value: TomlTypes) -> EnvList: + return EnvList(envs=list(TomlLoader.to_list(value, str))) + + +_T = TypeVar("_T") + + +def _ensure_type_correct(val: TomlTypes, of_type: type[_T]) -> TypeGuard[_T]: # noqa: C901, PLR0912 + casting_to = getattr(of_type, "__origin__", of_type.__class__) + msg = "" + if casting_to in {list, List}: + entry_type = of_type.__args__[0] # type: ignore[attr-defined] + if not (isinstance(val, list) and all(_ensure_type_correct(v, entry_type) for v in val)): + msg = f"{val} is not list" + elif issubclass(of_type, Command): + # first we cast it to list then create commands, so for now just validate is a nested list + _ensure_type_correct(val, list[str]) + elif casting_to in {set, Set}: + entry_type = of_type.__args__[0] # type: ignore[attr-defined] + if not (isinstance(val, set) and all(_ensure_type_correct(v, entry_type) for v in val)): + msg = f"{val} is not set" + elif casting_to in {dict, Dict}: + key_type, value_type = of_type.__args__[0], of_type.__args__[1] # type: ignore[attr-defined] + if not ( + isinstance(val, dict) + and all( + _ensure_type_correct(dict_key, key_type) and _ensure_type_correct(dict_value, value_type) + for dict_key, dict_value in val.items() + ) + ): + msg = f"{val} is not dictionary" + elif casting_to == Union: # handle Optional values + args: list[type[Any]] = of_type.__args__ # type: ignore[attr-defined] + for arg in args: + try: + _ensure_type_correct(val, arg) + break + except TypeError: + pass + else: + msg = f"{val} is not union of {args}" + elif casting_to in {Literal, type(Literal)}: + choice = of_type.__args__ # type: ignore[attr-defined] + if val not in choice: + msg = f"{val} is not one of literal {choice}" + elif not isinstance(val, of_type): + msg = f"{val} is not one of {of_type}" + if msg: + raise TypeError(msg) + return cast(_T, val) # type: ignore[return-value] # logic too complicated for mypy + + +__all__ = [ + "TomlLoader", +] diff --git a/src/tox/config/source/discover.py b/src/tox/config/source/discover.py index 7fcac23cd7..b94b392415 100644 --- a/src/tox/config/source/discover.py +++ b/src/tox/config/source/discover.py @@ -9,12 +9,18 @@ from .legacy_toml import LegacyToml from .setup_cfg import SetupCfg +from .toml import Toml from .tox_ini import ToxIni if TYPE_CHECKING: from .api import Source -SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, SetupCfg, LegacyToml) +SOURCE_TYPES: tuple[type[Source], ...] = ( + ToxIni, + SetupCfg, + LegacyToml, + Toml, +) def discover_source(config_file: Path | None, root_dir: Path | None) -> Source: diff --git a/src/tox/config/source/toml.py b/src/tox/config/source/toml.py new file mode 100644 index 0000000000..15fb268e22 --- /dev/null +++ b/src/tox/config/source/toml.py @@ -0,0 +1,107 @@ +"""Load.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast + +from tox.config.loader.section import Section +from tox.config.loader.toml import TomlLoader + +from .api import Source + +if sys.version_info >= (3, 11): # pragma: no cover (py311+) + import tomllib +else: # pragma: no cover (py311+) + import tomli as tomllib + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + from tox.config.loader.api import Loader, OverrideMap + from tox.config.sets import CoreConfigSet + +TEST_ENV_PREFIX = "env" + + +class TomlSection(Section): + SEP = "." + + @classmethod + def test_env(cls, name: str) -> TomlSection: + return cls(f"tox{cls.SEP}{name}", name) + + @property + def is_test_env(self) -> bool: + return self.prefix == TEST_ENV_PREFIX + + @property + def keys(self) -> Iterable[str]: + return self.key.split(self.SEP) + + +class Toml(Source): + """Configuration sourced from a pyproject.toml files.""" + + FILENAME = "pyproject.toml" + + def __init__(self, path: Path) -> None: + if path.name != self.FILENAME or not path.exists(): + raise ValueError + with path.open("rb") as file_handler: + toml_content = tomllib.load(file_handler) + try: + content: Mapping[str, Any] = toml_content["tool"]["tox"] + if "legacy_tox_ini" in content: + msg = "legacy_tox_ini" + raise KeyError(msg) # noqa: TRY301 + self._content = content + except KeyError as exc: + raise ValueError(path) from exc + super().__init__(path) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.path!r})" + + def get_core_section(self) -> Section: # noqa: PLR6301 + return TomlSection(prefix=None, name="tox") + + def transform_section(self, section: Section) -> Section: # noqa: PLR6301 + return TomlSection(section.prefix, section.name) + + def get_loader(self, section: Section, override_map: OverrideMap) -> Loader[Any] | None: + current = self._content + for at, key in enumerate(cast(TomlSection, section).keys): + if at == 0: + if key != "tox": + msg = "Internal error, first key is not tox" + raise RuntimeError(msg) + elif key in current: + current = current[key] + else: + return None + return TomlLoader( + section=section, + overrides=override_map.get(section.key, []), + content=current, + ) + + def envs(self, core_conf: CoreConfigSet) -> Iterator[str]: + yield from core_conf["env_list"] + yield from [i.key for i in self.sections()] + + def sections(self) -> Iterator[Section]: + for env_name in self._content.get("env", {}): + yield TomlSection.from_key(env_name) + + def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Section]: # noqa: PLR6301, ARG002 + yield from [TomlSection.from_key(b) for b in base] + + def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]: # noqa: PLR6301 + return TomlSection.test_env(item), ["tox.env_base"], ["tox.pkgenv"] + + +__all__ = [ + "Toml", +] diff --git a/tests/config/loader/toml/test_toml_loader.py b/tests/config/loader/toml/test_toml_loader.py new file mode 100644 index 0000000000..885f9b6bb5 --- /dev/null +++ b/tests/config/loader/toml/test_toml_loader.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable + +from tox.config.loader.ini import IniLoader +from tox.config.source.ini_section import IniSection + +if TYPE_CHECKING: + from configparser import ConfigParser + + +def test_ini_loader_keys(mk_ini_conf: Callable[[str], ConfigParser]) -> None: + core = IniSection(None, "tox") + loader = IniLoader(core, mk_ini_conf("\n[tox]\n\na=b\nc=d\n\n"), [], core_section=core) + assert loader.found_keys() == {"a", "c"} diff --git a/tests/config/source/test_toml.py b/tests/config/source/test_toml.py new file mode 100644 index 0000000000..9bf12bd4f1 --- /dev/null +++ b/tests/config/source/test_toml.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tox.pytest import ToxProjectCreator + + +def test_conf_in_legacy_toml(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox] + env_list = [ "A", "B"] + + [tool.tox.env_base] + description = "Do magical things" + commands = [ + ["python", "--version"], + ["python", "-c", "import sys; print(sys.executable)"] + ] + + [tool.tox.env.C] + description = "Do magical things in C" + commands = [ + ["python", "--version"] + ] + """ + }) + + outcome = project.run("c", "--core", "-k", "commands") + outcome.assert_success() + + outcome = project.run("c", "-e", "C,3.13") + outcome.assert_success()