[WIP] Native TOML config

src/tox/config/loader/convert.py

@@ -4,9 +4,11 @@
 from abc import ABC, abstractmethod
 from collections import OrderedDict
 from inspect import isclass
+from itertools import chain
 from pathlib import Path
-from typing import Any, Callable, Dict, Generic, Iterator, List, Literal, Optional, Set, TypeVar, Union, cast
+from typing import Any, Callable, Dict, Generic, Iterable, Iterator, List, Literal, Optional, Set, TypeVar, Union, cast
 
+from tox.config.loader.ini.factor import find_factor_groups
 from tox.config.types import Command, EnvList
 
 _NO_MAPPING = object()
@@ -16,6 +18,44 @@
 Factory = Optional[Callable[[object], T]]  # note the argument is anything, due e.g. memory loader can inject anything
 
 
+class ConditionalValue(Generic[T]):
+    """Value with a condition."""
+
+    def __init__(self, value: T, condition: str | None) -> None:
+        self.value = value
+        self.condition = condition
+        self._groups = tuple(find_factor_groups(condition)) if condition is not None else ()
+
+    def matches(self, env_name: str) -> bool:
+        """Return whether the value matches the environment name."""
+        if self.condition is None:
+            return True
+
+        # Split env_name to factors.
+        env_factors = set(chain.from_iterable([(i for i, _ in a) for a in find_factor_groups(env_name)]))
+
+        matches = []
+        for group in self._groups:
+            group_matches = []
+            for factor, negate in group:
+                group_matches.append((factor in env_factors) ^ negate)
+            matches.append(all(group_matches))
+        return any(matches)
+
+
+class ConditionalSetting(Generic[T]):
+    """Setting whose value depends on various conditions."""
+
+    def __init__(self, values: typing.Iterable[ConditionalValue[T]]) -> None:
+        self.values = tuple(values)
+
+    def filter(self, env_name: str) -> Iterable[T]:
+        """Filter values for the environment."""
+        for value in self.values:
+            if value.matches(env_name):
+                yield value.value
+
+
 class Convert(ABC, Generic[T]):
     """A class that converts a raw type to a given tox (python) type."""
 

src/tox/config/loader/ini/factor.py

@@ -60,7 +60,11 @@ def expand_factors(value: str) -> Iterator[tuple[list[list[tuple[str, bool]]] |
 
 
 def find_factor_groups(value: str) -> Iterator[list[tuple[str, bool]]]:
-    """Transform '{py,!pi}-{a,b},c' to [{'py', 'a'}, {'py', 'b'}, {'pi', 'a'}, {'pi', 'b'}, {'c'}]."""
+    """Transform '{py,!pi}-{a,b},c' to [[('py', False), ('a', False)],
+    [('py', False), ('b', False)],
+    [('pi', True), ('a', False)],
+    [('pi', False), ('b', True)],
+    [('c', False)]]."""
     for env in expand_env_with_negation(value):
         result = [name_with_negate(f) for f in env.split("-")]
         yield result

src/tox/config/loader/memory.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterator
+from typing import TYPE_CHECKING, Any, Iterator, Sequence
 
 from tox.config.types import Command, EnvList
 
-from .api import Loader
+from .api import Loader, Override
 from .section import Section
 from .str_convert import StrConvert
 
@@ -14,9 +14,19 @@
 
 
 class MemoryLoader(Loader[Any]):
-    def __init__(self, **kwargs: Any) -> None:
-        super().__init__(Section(prefix="<memory>", name=str(id(self))), [])
-        self.raw: dict[str, Any] = {**kwargs}
+    def __init__(
+        self,
+        raw: dict[str, Any],
+        *,
+        section: Section | None = None,
+        overrides: list[Override] | None = None,
+    ) -> None:
+        section = section or Section(prefix="<memory>", name=str(id(self)))
+        super().__init__(section, overrides or [])
+        self.raw = raw
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(section={self._section.key}, overrides={self.overrides!r})"
 
     def load_raw(self, key: Any, conf: Config | None, env_name: str | None) -> Any:  # noqa: ARG002
         return self.raw[key]
@@ -62,4 +72,6 @@ def to_env_list(value: Any) -> EnvList:
             return value
         if isinstance(value, str):
             return StrConvert.to_env_list(value)
+        if isinstance(value, Sequence):
+            return EnvList(value)
         raise TypeError(value)

src/tox/config/loader/section.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Generic, TypeVar
 
 if TYPE_CHECKING:
     import sys
@@ -11,32 +12,30 @@
         from typing_extensions import Self
 
 
-class Section:  # noqa: PLW1641
-    """tox configuration section."""
+X = TypeVar("X")
+
+
+class BaseSection(ABC, Generic[X]):
+    """Base class for tox configuration section."""
 
     SEP = ":"  #: string used to separate the prefix and the section in the key
 
-    def __init__(self, prefix: str | None, name: str) -> None:
+    def __init__(self, prefix: X, name: str) -> None:
         self._prefix = prefix
         self._name = name
 
     @classmethod
+    @abstractmethod
     def from_key(cls: type[Self], key: str) -> Self:
         """
         Create a section from a section key.
 
         :param key: the section key
         :return: the constructed section
         """
-        sep_at = key.find(cls.SEP)
-        if sep_at == -1:
-            prefix, name = None, key
-        else:
-            prefix, name = key[:sep_at], key[sep_at + 1 :]
-        return cls(prefix, name)
 
     @property
-    def prefix(self) -> str | None:
+    def prefix(self) -> X:
         """:return: the prefix of the section"""
         return self._prefix
 
@@ -46,9 +45,9 @@ def name(self) -> str:
         return self._name
 
     @property
+    @abstractmethod
     def key(self) -> str:
         """:return: the section key"""
-        return self.SEP.join(i for i in (self._prefix, self._name) if i is not None)
 
     def __str__(self) -> str:
         return self.key
@@ -63,6 +62,31 @@ def __eq__(self, other: object) -> bool:
         )
 
 
+# TODO: Merge this with IniSection?
+class Section(BaseSection[str | None]):
+    """tox configuration section."""
+
+    @classmethod
+    def from_key(cls: type[Self], key: str) -> Self:
+        """
+        Create a section from a section key.
+
+        :param key: the section key
+        :return: the constructed section
+        """
+        sep_at = key.find(cls.SEP)
+        if sep_at == -1:
+            prefix, name = None, key
+        else:
+            prefix, name = key[:sep_at], key[sep_at + 1 :]
+        return cls(prefix, name)
+
+    @property
+    def key(self) -> str:
+        """:return: the section key"""
+        return self.SEP.join(i for i in (self._prefix, self._name) if i is not None)
+
+
 __all__ = [
     "Section",
 ]

src/tox/config/source/api.py

@@ -14,6 +14,8 @@
     from tox.config.sets import ConfigSet, CoreConfigSet
 
 
+# TODO: Generic in Section class?
+# TODO: Use BaseSection instead of Section?
 class Source(ABC):
     """Source is able to return a configuration value (for either the core or per environment source)."""
 

src/tox/config/source/discover.py

@@ -9,12 +9,13 @@
 
 from .legacy_toml import LegacyToml
 from .setup_cfg import SetupCfg
+from .toml import PyProjectToml, ToxToml
 from .tox_ini import ToxIni
 
 if TYPE_CHECKING:
     from .api import Source
 
-SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, SetupCfg, LegacyToml)
+SOURCE_TYPES: tuple[type[Source], ...] = (ToxIni, ToxToml, PyProjectToml, SetupCfg, LegacyToml)
 
 
 def discover_source(config_file: Path | None, root_dir: Path | None) -> Source:

src/tox/config/source/toml.py

@@ -0,0 +1,164 @@
+"""Support for TOML config sources.
+
+This is experimental API! Expect things to be broken.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from itertools import chain
+from typing import TYPE_CHECKING, Any, Iterable, Iterator
+
+import tomllib
+
+from tox.config.loader.ini.factor import find_envs
+from tox.config.loader.memory import MemoryLoader
+
+from .api import Source
+from .toml_section import BASE_TEST_ENV, CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, TEST_ENV_ROOT, TomlSection
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from tox.config.loader.api import OverrideMap
+    from tox.config.loader.section import Section
+    from tox.config.sets import ConfigSet
+
+
+def _extract_section(raw: dict[str, Any], section: TomlSection) -> Any:
+    """Extract section from TOML decoded data."""
+    result = raw
+    for key in chain(section.prefix, (section.name,)):
+        if key in result:
+            result = result[key]
+        else:
+            return None
+    return result
+
+
+class TomlSource(Source):
+    """Configuration sourced from a toml file (such as tox.toml).
+
+    This is experimental API! Expect things to be broken.
+    """
+
+    CORE_SECTION = CORE
+    ROOT_KEY: str | None = None
+
+    def __init__(self, path: Path, content: str | None = None) -> None:
+        super().__init__(path)
+        if content is None:
+            if not path.exists():
+                msg = f"Path {path} does not exist."
+                raise ValueError(msg)
+            content = path.read_text()
+        data = tomllib.loads(content)
+        if self.ROOT_KEY:
+            if self.ROOT_KEY not in data:
+                msg = f"Section {self.ROOT_KEY} not found in {path}."
+                raise ValueError(msg)
+            data = data[self.ROOT_KEY]
+        self._raw = data
+        self._section_mapping: defaultdict[str, list[str]] = defaultdict(list)
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(path={self.path})"
+
+    def transform_section(self, section: Section) -> Section:
+        return TomlSection(section.prefix, section.name)
+
+    def get_loader(self, section: TomlSection, override_map: OverrideMap) -> MemoryLoader | None:
+        result = _extract_section(self._raw, section)
+        if result is None:
+            return None
+
+        return MemoryLoader(
+            result,
+            section=section,
+            overrides=override_map.get(section.key, []),
+        )
+
+    def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Section]:  # noqa: PLR6301
+        for a_base in base:
+            yield TomlSection(in_section.prefix, a_base)
+
+    def sections(self) -> Iterator[TomlSection]:
+        # TODO: just return core section and any `tox.env.XXX` sections which exist directly.
+        for key in self._raw:
+            section = TomlSection.from_key(key)
+            yield section
+            if section == self.CORE_SECTION:
+                test_env_data = _extract_section(self._raw, TEST_ENV_ROOT)
+                for env_name in test_env_data or {}:
+                    yield TomlSection(TEST_ENV_PREFIX, env_name)
+
+    def envs(self, core_config: ConfigSet) -> Iterator[str]:
+        seen = set()
+        for name in self._discover_tox_envs(core_config):
+            if name not in seen:
+                seen.add(name)
+                yield name
+
+    def _discover_tox_envs(self, core_config: ConfigSet) -> Iterator[str]:
+        def register_factors(envs: Iterable[str]) -> None:
+            known_factors.update(chain.from_iterable(e.split("-") for e in envs))
+
+        explicit = list(core_config["env_list"])
+        yield from explicit
+        known_factors: set[str] = set()
+        register_factors(explicit)
+
+        # discover all additional defined environments, including generative section headers
+        for section in self.sections():
+            if section.is_test_env:
+                register_factors(section.names)
+                for name in section.names:
+                    self._section_mapping[name].append(section.key)
+                    yield name
+        # add all conditional markers that are not part of the explicitly defined sections
+        for section in self.sections():
+            yield from self._discover_from_section(section, known_factors)
+
+    def _discover_from_section(self, section: TomlSection, known_factors: set[str]) -> Iterator[str]:
+        section_data = _extract_section(self._raw, section)
+        for value in (section_data or {}).values():
+            if isinstance(value, bool):
+                # It's not a value with env definition.
+                continue
+            # XXX: We munch the value to multiline string to parse it by the library utils.
+            merged_value = "\n".join(str(v) for v in tuple(value))
+            for env in find_envs(merged_value):
+                if set(env.split("-")) - known_factors:
+                    yield env
+
+    def get_tox_env_section(self, item: str) -> tuple[TomlSection, list[str], list[str]]:  # noqa: PLR6301
+        return TomlSection.test_env(item), [BASE_TEST_ENV], [PKG_ENV_PREFIX]
+
+    def get_core_section(self) -> TomlSection:
+        return self.CORE_SECTION
+
+
+class ToxToml(TomlSource):
+    """Configuration sourced from a tox.toml file.
+
+    This is experimental API! Expect things to be broken.
+    """
+
+    FILENAME = "tox.toml"
+
+
+class PyProjectToml(TomlSource):
+    """Configuration sourced from a pyproject.toml file.
+
+    This is experimental API! Expect things to be broken.
+    """
+
+    FILENAME = "pyproject.toml"
+    ROOT_KEY = "tool"
+
+    def __init__(self, path: Path, content: str | None = None) -> None:
+        super().__init__(path, content)
+        core_data = _extract_section(self._raw, self.CORE_SECTION)
+        if core_data is not None and tuple(core_data.keys()) == ("legacy_tox_ini",):
+            msg = "pyproject.toml is in the legacy mode."
+            raise ValueError(msg)