From 57a4faec04926823e069bc405fc5cd704172b397 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 10:33:16 +0100 Subject: [PATCH 1/9] Vendor copy of tomli-w 1.2.0 --- easybuild/tools/__init__.py | 8 + easybuild/tools/_toml_writer/README.md | 12 ++ easybuild/tools/_toml_writer/__init__.py | 4 + easybuild/tools/_toml_writer/_writer.py | 229 +++++++++++++++++++++++ setup.py | 3 +- test/framework/general.py | 6 +- 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 easybuild/tools/_toml_writer/README.md create mode 100644 easybuild/tools/_toml_writer/__init__.py create mode 100644 easybuild/tools/_toml_writer/_writer.py diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index 60a2671221..b5a274a407 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -38,3 +38,11 @@ from easybuild.tools.loose_version import LooseVersion # noqa(F401) + + +def dump_toml(o: dict) -> str: + """Convert the input directory to a string in TOML format""" + # Import and redirect here on purpose as the conversion method (or Python package) + # is an implementation detail that can be changed at any time + import easybuild.tools._toml_writer + return easybuild.tools._toml_writer.dumps(o) diff --git a/easybuild/tools/_toml_writer/README.md b/easybuild/tools/_toml_writer/README.md new file mode 100644 index 0000000000..a5633274a1 --- /dev/null +++ b/easybuild/tools/_toml_writer/README.md @@ -0,0 +1,12 @@ +# TOML writer library + +This is an implementation of a library for writing TOML files. +To be able to update, change and remove it without breaking backwards compatibility in EasyBuild +it is in a "private" sub-package and should not be used directly: **It can be changed without warning!** + +The stable interface is `easybuild.tools.dump_toml`. + +## Implementation + +Currently, a copy of [`tomli-w`](https://github.com/hukkin/tomli-w) is vendored in this folder. +The used version is 1.2. diff --git a/easybuild/tools/_toml_writer/__init__.py b/easybuild/tools/_toml_writer/__init__.py new file mode 100644 index 0000000000..4013d575f0 --- /dev/null +++ b/easybuild/tools/_toml_writer/__init__.py @@ -0,0 +1,4 @@ +__all__ = ("dumps", "dump") +__version__ = "1.2.0" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from tomli_w._writer import dump, dumps diff --git a/easybuild/tools/_toml_writer/_writer.py b/easybuild/tools/_toml_writer/_writer.py new file mode 100644 index 0000000000..b1acd3f26b --- /dev/null +++ b/easybuild/tools/_toml_writer/_writer.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +from collections.abc import Mapping +from datetime import date, datetime, time +from types import MappingProxyType + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Generator + from decimal import Decimal + from typing import IO, Any, Final + +ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) +ILLEGAL_BASIC_STR_CHARS = frozenset('"\\') | ASCII_CTRL - frozenset("\t") +BARE_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_" +) +ARRAY_TYPES = (list, tuple) +MAX_LINE_LENGTH = 100 + +COMPACT_ESCAPES = MappingProxyType( + { + "\u0008": "\\b", # backspace + "\u000A": "\\n", # linefeed + "\u000C": "\\f", # form feed + "\u000D": "\\r", # carriage return + "\u0022": '\\"', # quote + "\u005C": "\\\\", # backslash + } +) + + +class Context: + def __init__(self, allow_multiline: bool, indent: int): + if indent < 0: + raise ValueError("Indent width must be non-negative") + self.allow_multiline: Final = allow_multiline + # cache rendered inline tables (mapping from object id to rendered inline table) + self.inline_table_cache: Final[dict[int, str]] = {} + self.indent_str: Final = " " * indent + + +def dump( + obj: Mapping[str, Any], + fp: IO[bytes], + /, + *, + multiline_strings: bool = False, + indent: int = 4, +) -> None: + ctx = Context(multiline_strings, indent) + for chunk in gen_table_chunks(obj, ctx, name=""): + fp.write(chunk.encode()) + + +def dumps( + obj: Mapping[str, Any], /, *, multiline_strings: bool = False, indent: int = 4 +) -> str: + ctx = Context(multiline_strings, indent) + return "".join(gen_table_chunks(obj, ctx, name="")) + + +def gen_table_chunks( + table: Mapping[str, Any], + ctx: Context, + *, + name: str, + inside_aot: bool = False, +) -> Generator[str, None, None]: + yielded = False + literals = [] + tables: list[tuple[str, Any, bool]] = [] # => [(key, value, inside_aot)] + for k, v in table.items(): + if isinstance(v, Mapping): + tables.append((k, v, False)) + elif is_aot(v) and not all(is_suitable_inline_table(t, ctx) for t in v): + tables.extend((k, t, True) for t in v) + else: + literals.append((k, v)) + + if inside_aot or name and (literals or not tables): + yielded = True + yield f"[[{name}]]\n" if inside_aot else f"[{name}]\n" + + if literals: + yielded = True + for k, v in literals: + yield f"{format_key_part(k)} = {format_literal(v, ctx)}\n" + + for k, v, in_aot in tables: + if yielded: + yield "\n" + else: + yielded = True + key_part = format_key_part(k) + display_name = f"{name}.{key_part}" if name else key_part + yield from gen_table_chunks(v, ctx, name=display_name, inside_aot=in_aot) + + +def format_literal(obj: object, ctx: Context, *, nest_level: int = 0) -> str: + if isinstance(obj, bool): + return "true" if obj else "false" + if isinstance(obj, (int, float, date, datetime)): + return str(obj) + if isinstance(obj, time): + if obj.tzinfo: + raise ValueError("TOML does not support offset times") + return str(obj) + if isinstance(obj, str): + return format_string(obj, allow_multiline=ctx.allow_multiline) + if isinstance(obj, ARRAY_TYPES): + return format_inline_array(obj, ctx, nest_level) + if isinstance(obj, Mapping): + return format_inline_table(obj, ctx) + + # Lazy import to improve module import time + from decimal import Decimal + + if isinstance(obj, Decimal): + return format_decimal(obj) + raise TypeError( + f"Object of type '{type(obj).__qualname__}' is not TOML serializable" + ) + + +def format_decimal(obj: Decimal) -> str: + if obj.is_nan(): + return "nan" + if obj.is_infinite(): + return "-inf" if obj.is_signed() else "inf" + dec_str = str(obj).lower() + return dec_str if "." in dec_str or "e" in dec_str else dec_str + ".0" + + +def format_inline_table(obj: Mapping, ctx: Context) -> str: + # check cache first + obj_id = id(obj) + if obj_id in ctx.inline_table_cache: + return ctx.inline_table_cache[obj_id] + + if not obj: + rendered = "{}" + else: + rendered = ( + "{ " + + ", ".join( + f"{format_key_part(k)} = {format_literal(v, ctx)}" + for k, v in obj.items() + ) + + " }" + ) + ctx.inline_table_cache[obj_id] = rendered + return rendered + + +def format_inline_array(obj: tuple | list, ctx: Context, nest_level: int) -> str: + if not obj: + return "[]" + item_indent = ctx.indent_str * (1 + nest_level) + closing_bracket_indent = ctx.indent_str * nest_level + return ( + "[\n" + + ",\n".join( + item_indent + format_literal(item, ctx, nest_level=nest_level + 1) + for item in obj + ) + + f",\n{closing_bracket_indent}]" + ) + + +def format_key_part(part: str) -> str: + try: + only_bare_key_chars = BARE_KEY_CHARS.issuperset(part) + except TypeError: + raise TypeError( + f"Invalid mapping key '{part}' of type '{type(part).__qualname__}'." + " A string is required." + ) from None + + if part and only_bare_key_chars: + return part + return format_string(part, allow_multiline=False) + + +def format_string(s: str, *, allow_multiline: bool) -> str: + do_multiline = allow_multiline and "\n" in s + if do_multiline: + result = '"""\n' + s = s.replace("\r\n", "\n") + else: + result = '"' + + pos = seq_start = 0 + while True: + try: + char = s[pos] + except IndexError: + result += s[seq_start:pos] + if do_multiline: + return result + '"""' + return result + '"' + if char in ILLEGAL_BASIC_STR_CHARS: + result += s[seq_start:pos] + if char in COMPACT_ESCAPES: + if do_multiline and char == "\n": + result += "\n" + else: + result += COMPACT_ESCAPES[char] + else: + result += "\\u" + hex(ord(char))[2:].rjust(4, "0") + seq_start = pos + 1 + pos += 1 + + +def is_aot(obj: Any) -> bool: + """Decides if an object behaves as an array of tables (i.e. a nonempty list + of dicts).""" + return bool( + isinstance(obj, ARRAY_TYPES) + and obj + and all(isinstance(v, Mapping) for v in obj) + ) + + +def is_suitable_inline_table(obj: Mapping, ctx: Context) -> bool: + """Use heuristics to decide if the inline-style representation is a good + choice for a given table.""" + rendered_inline = f"{ctx.indent_str}{format_inline_table(obj, ctx)}," + return len(rendered_inline) <= MAX_LINE_LENGTH and "\n" not in rendered_inline diff --git a/setup.py b/setup.py index 303f34974a..6cdff71f56 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,8 @@ def find_rel_test(): "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", "easybuild.tools.containers", "easybuild.tools.deprecated", "easybuild.tools.job", "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", "easybuild.tools.package", "easybuild.tools.package.package_naming_scheme", - "easybuild.tools.py2vs3", "easybuild.tools.repository", "easybuild.tools.tomllib", "easybuild.tools.tomllib.tomli", + "easybuild.tools.py2vs3", "easybuild.tools.repository", + "easybuild.tools.tomllib", "easybuild.tools.tomllib.tomli", "easybuild.tools._toml_writer", "test.framework", "test", ] diff --git a/test/framework/general.py b/test/framework/general.py index 0b5f5cd153..491d8f4ac4 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -156,7 +156,7 @@ def test_import_available_modules(self): def test_vendored_packages(self): """Smoke-test for vendored packages""" - from easybuild.tools import tomllib + from easybuild.tools import tomllib, dump_toml # Example from toml.io res = tomllib.loads(""" title = "TOML Example" @@ -180,6 +180,10 @@ def test_vendored_packages(self): role = "backend" """) self.assertTrue(res) # Should just be non-empty + # Dumping should round-trip + res_str = dump_toml(res) + res_parsed = tomllib.loads(res_str) + self.assertEqual(res_parsed, res) def suite(loader=None): From 33a8e9c7bec8829c677d30f15c639c393202b529 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 10:38:06 +0100 Subject: [PATCH 2/9] tomli-w: Allow importing sub-package --- easybuild/tools/_toml_writer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/_toml_writer/__init__.py b/easybuild/tools/_toml_writer/__init__.py index 4013d575f0..b7ebd048af 100644 --- a/easybuild/tools/_toml_writer/__init__.py +++ b/easybuild/tools/_toml_writer/__init__.py @@ -1,4 +1,4 @@ __all__ = ("dumps", "dump") __version__ = "1.2.0" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT -from tomli_w._writer import dump, dumps +from ._writer import dump, dumps From 5d08e21566f4e609e524795e1683fc91f03f3d8d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 10:45:42 +0100 Subject: [PATCH 3/9] Add test with "Z" timezone to TOML --- test/framework/general.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/general.py b/test/framework/general.py index 491d8f4ac4..d98f6950a9 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -164,6 +164,7 @@ def test_vendored_packages(self): [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 + odt1 = 1979-05-27T07:32:00Z [database] enabled = true From b914c2d2678ed09161090a97c38881868ad39751 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 11:00:13 +0100 Subject: [PATCH 4/9] tmli-w: Make compatible with Python 3.6 --- easybuild/tools/_toml_writer/README.md | 2 ++ easybuild/tools/_toml_writer/_writer.py | 21 +++++++++------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/_toml_writer/README.md b/easybuild/tools/_toml_writer/README.md index a5633274a1..cd191e03f0 100644 --- a/easybuild/tools/_toml_writer/README.md +++ b/easybuild/tools/_toml_writer/README.md @@ -10,3 +10,5 @@ The stable interface is `easybuild.tools.dump_toml`. Currently, a copy of [`tomli-w`](https://github.com/hukkin/tomli-w) is vendored in this folder. The used version is 1.2. + +Minor modifications to make it compatible with Python 3.6. diff --git a/easybuild/tools/_toml_writer/_writer.py b/easybuild/tools/_toml_writer/_writer.py index b1acd3f26b..ee93aa2989 100644 --- a/easybuild/tools/_toml_writer/_writer.py +++ b/easybuild/tools/_toml_writer/_writer.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from collections.abc import Mapping from datetime import date, datetime, time from types import MappingProxyType @@ -8,7 +6,7 @@ if TYPE_CHECKING: from collections.abc import Generator from decimal import Decimal - from typing import IO, Any, Final + from typing import IO, Any, Union ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) ILLEGAL_BASIC_STR_CHARS = frozenset('"\\') | ASCII_CTRL - frozenset("\t") @@ -34,16 +32,15 @@ class Context: def __init__(self, allow_multiline: bool, indent: int): if indent < 0: raise ValueError("Indent width must be non-negative") - self.allow_multiline: Final = allow_multiline + self.allow_multiline = allow_multiline # cache rendered inline tables (mapping from object id to rendered inline table) - self.inline_table_cache: Final[dict[int, str]] = {} - self.indent_str: Final = " " * indent + self.inline_table_cache: dict[int, str] = {} + self.indent_str = " " * indent def dump( - obj: Mapping[str, Any], + obj: Mapping, fp: IO[bytes], - /, *, multiline_strings: bool = False, indent: int = 4, @@ -54,19 +51,19 @@ def dump( def dumps( - obj: Mapping[str, Any], /, *, multiline_strings: bool = False, indent: int = 4 + obj: Mapping, *, multiline_strings: bool = False, indent: int = 4 ) -> str: ctx = Context(multiline_strings, indent) return "".join(gen_table_chunks(obj, ctx, name="")) def gen_table_chunks( - table: Mapping[str, Any], + table: Mapping, ctx: Context, *, name: str, inside_aot: bool = False, -) -> Generator[str, None, None]: +) -> Generator: yielded = False literals = [] tables: list[tuple[str, Any, bool]] = [] # => [(key, value, inside_aot)] @@ -153,7 +150,7 @@ def format_inline_table(obj: Mapping, ctx: Context) -> str: return rendered -def format_inline_array(obj: tuple | list, ctx: Context, nest_level: int) -> str: +def format_inline_array(obj: Union[tuple, list], ctx: Context, nest_level: int) -> str: if not obj: return "[]" item_indent = ctx.indent_str * (1 + nest_level) From 619c88b4a5e4fe3c0a8328f157d936b8f972897d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 11:00:35 +0100 Subject: [PATCH 5/9] tomli-w: Remove TYPE_CHECKING guard This confuses pylint --- easybuild/tools/_toml_writer/_writer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/_toml_writer/_writer.py b/easybuild/tools/_toml_writer/_writer.py index ee93aa2989..8c276a796c 100644 --- a/easybuild/tools/_toml_writer/_writer.py +++ b/easybuild/tools/_toml_writer/_writer.py @@ -2,11 +2,9 @@ from datetime import date, datetime, time from types import MappingProxyType -TYPE_CHECKING = False -if TYPE_CHECKING: - from collections.abc import Generator - from decimal import Decimal - from typing import IO, Any, Union +from collections.abc import Generator +from decimal import Decimal +from typing import IO, Any, Union ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) ILLEGAL_BASIC_STR_CHARS = frozenset('"\\') | ASCII_CTRL - frozenset("\t") From 1cfea61dc8c259c31979bbda9263990a005cc10b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Dec 2025 14:37:48 +0100 Subject: [PATCH 6/9] Use more features of the TOML format Trigger more code paths of the TOML parser and writer to ensure it works reliably. --- test/framework/general.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/framework/general.py b/test/framework/general.py index d98f6950a9..6ca6c01a40 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -158,8 +158,11 @@ def test_vendored_packages(self): """Smoke-test for vendored packages""" from easybuild.tools import tomllib, dump_toml # Example from toml.io - res = tomllib.loads(""" + res = tomllib.loads(''' title = "TOML Example" + description = """Test + this\u1234 + """ [owner] name = "Tom Preston-Werner" @@ -170,16 +173,20 @@ def test_vendored_packages(self): enabled = true ports = [ 8000, 8001, 8002 ] data = [ ["delta", "phi"], [3.14] ] - temp_targets = { cpu = 79.5, case = 72.0 } + temp_targets = { cpu = { x = 79.5 }, case = -inf } [servers] - [servers.alpha] role = "frontend" - [servers.beta] role = "backend" - """) + + [[products]] + name = "Hammer" + [[products]] + [[products]] + name = "Nail" + ''') self.assertTrue(res) # Should just be non-empty # Dumping should round-trip res_str = dump_toml(res) From 8b01413ae2c130baebf7d3be0572510ec1290c78 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Dec 2025 09:26:00 +0100 Subject: [PATCH 7/9] Show commit hash of tomli-w Co-authored-by: Kenneth Hoste --- easybuild/tools/_toml_writer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/_toml_writer/README.md b/easybuild/tools/_toml_writer/README.md index cd191e03f0..a11694e8b1 100644 --- a/easybuild/tools/_toml_writer/README.md +++ b/easybuild/tools/_toml_writer/README.md @@ -9,6 +9,6 @@ The stable interface is `easybuild.tools.dump_toml`. ## Implementation Currently, a copy of [`tomli-w`](https://github.com/hukkin/tomli-w) is vendored in this folder. -The used version is 1.2. +The used version is 1.2.0 (https://github.com/hukkin/tomli-w/releases/tag/1.2.0, commit a8f80172ba16fe694e37f6e07e6352ecee384c58). Minor modifications to make it compatible with Python 3.6. From a267bfc7de9666e9daebb349c8881d1c3f0ccf9c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Dec 2025 09:28:10 +0100 Subject: [PATCH 8/9] Also show commit hash for vendored tomli --- easybuild/tools/tomllib/tomli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/tomllib/tomli/README.md b/easybuild/tools/tomllib/tomli/README.md index ebc8edf853..23d13c0715 100644 --- a/easybuild/tools/tomllib/tomli/README.md +++ b/easybuild/tools/tomllib/tomli/README.md @@ -1,6 +1,6 @@ # Tomllib -Vendored `tomli` from https://github.com/hukkin/tomli version 2.3.0. +Vendored [`tomli`](https://github.com/hukkin/tomli) version 2.3.0, commit 3fccd16. PEP 680 added a version of it as `tomllib` to Python 3.11. Patched to remove features not available in Python 3.6, mostly type hints. From bc35f08b2faec8570bf1f86f077a0c34b7a4c585 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 12 Dec 2025 13:05:55 +0100 Subject: [PATCH 9/9] Move `dump_toml` to `easybuild.tools.filetools` --- easybuild/tools/__init__.py | 8 -------- easybuild/tools/_toml_writer/README.md | 2 +- easybuild/tools/filetools.py | 8 ++++++++ test/framework/general.py | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index b5a274a407..60a2671221 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -38,11 +38,3 @@ from easybuild.tools.loose_version import LooseVersion # noqa(F401) - - -def dump_toml(o: dict) -> str: - """Convert the input directory to a string in TOML format""" - # Import and redirect here on purpose as the conversion method (or Python package) - # is an implementation detail that can be changed at any time - import easybuild.tools._toml_writer - return easybuild.tools._toml_writer.dumps(o) diff --git a/easybuild/tools/_toml_writer/README.md b/easybuild/tools/_toml_writer/README.md index a11694e8b1..9020d38ff8 100644 --- a/easybuild/tools/_toml_writer/README.md +++ b/easybuild/tools/_toml_writer/README.md @@ -4,7 +4,7 @@ This is an implementation of a library for writing TOML files. To be able to update, change and remove it without breaking backwards compatibility in EasyBuild it is in a "private" sub-package and should not be used directly: **It can be changed without warning!** -The stable interface is `easybuild.tools.dump_toml`. +The stable interface is `easybuild.tools.filetools.dump_toml`. ## Implementation diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5c8f9cfbcc..21c3e70e47 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -3249,3 +3249,11 @@ def create_non_existing_paths(paths, max_tries=10000): set_gid_sticky_bits(path, recursive=True) return non_existing_paths + + +def dump_toml(o: dict) -> str: + """Convert the input directory to a string in TOML format""" + # Import and redirect here on purpose as the conversion method (or Python package) + # is an implementation detail that can be changed at any time + import easybuild.tools._toml_writer + return easybuild.tools._toml_writer.dumps(o) diff --git a/test/framework/general.py b/test/framework/general.py index 6ca6c01a40..d4f48ce8ad 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -156,7 +156,8 @@ def test_import_available_modules(self): def test_vendored_packages(self): """Smoke-test for vendored packages""" - from easybuild.tools import tomllib, dump_toml + from easybuild.tools import tomllib + from easybuild.tools.filetools import dump_toml # Example from toml.io res = tomllib.loads(''' title = "TOML Example"