Vendor copy of tomli-w 1.2.0, use it to implement dump_toml function in easybuild.tools.filetools

easybuild/tools/_toml_writer/README.md

@@ -0,0 +1,14 @@
+# TOML writer library
+
+This is an implementation of a library for writing TOML files.  
+To be able to update, change and remove it without breaking backwards compatibility in EasyBuild
+it is in a "private" sub-package and should not be used directly: **It can be changed without warning!**
+
+The stable interface is `easybuild.tools.filetools.dump_toml`.
+
+## Implementation
+
+Currently, a copy of [`tomli-w`](https://github.com/hukkin/tomli-w) is vendored in this folder.
+The used version is 1.2.0 (https://github.com/hukkin/tomli-w/releases/tag/1.2.0, commit a8f80172ba16fe694e37f6e07e6352ecee384c58).
+
+Minor modifications to make it compatible with Python 3.6.

easybuild/tools/_toml_writer/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ("dumps", "dump")
+__version__ = "1.2.0"  # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT
+
+from ._writer import dump, dumps

easybuild/tools/_toml_writer/_writer.py

@@ -0,0 +1,224 @@
+from collections.abc import Mapping
+from datetime import date, datetime, time
+from types import MappingProxyType
+
+from collections.abc import Generator
+from decimal import Decimal
+from typing import IO, Any, Union
+
+ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
+ILLEGAL_BASIC_STR_CHARS = frozenset('"\\') | ASCII_CTRL - frozenset("\t")
+BARE_KEY_CHARS = frozenset(
+    "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_"
+)
+ARRAY_TYPES = (list, tuple)
+MAX_LINE_LENGTH = 100
+
+COMPACT_ESCAPES = MappingProxyType(
+    {
+        "\u0008": "\\b",  # backspace
+        "\u000A": "\\n",  # linefeed
+        "\u000C": "\\f",  # form feed
+        "\u000D": "\\r",  # carriage return
+        "\u0022": '\\"',  # quote
+        "\u005C": "\\\\",  # backslash
+    }
+)
+
+
+class Context:
+    def __init__(self, allow_multiline: bool, indent: int):
+        if indent < 0:
+            raise ValueError("Indent width must be non-negative")
+        self.allow_multiline = allow_multiline
+        # cache rendered inline tables (mapping from object id to rendered inline table)
+        self.inline_table_cache: dict[int, str] = {}
+        self.indent_str = " " * indent
+
+
+def dump(
+    obj: Mapping,
+    fp: IO[bytes],
+    *,
+    multiline_strings: bool = False,
+    indent: int = 4,
+) -> None:
+    ctx = Context(multiline_strings, indent)
+    for chunk in gen_table_chunks(obj, ctx, name=""):
+        fp.write(chunk.encode())
+
+
+def dumps(
+    obj: Mapping, *, multiline_strings: bool = False, indent: int = 4
+) -> str:
+    ctx = Context(multiline_strings, indent)
+    return "".join(gen_table_chunks(obj, ctx, name=""))
+
+
+def gen_table_chunks(
+    table: Mapping,
+    ctx: Context,
+    *,
+    name: str,
+    inside_aot: bool = False,
+) -> Generator:
+    yielded = False
+    literals = []
+    tables: list[tuple[str, Any, bool]] = []  # => [(key, value, inside_aot)]
+    for k, v in table.items():
+        if isinstance(v, Mapping):
+            tables.append((k, v, False))
+        elif is_aot(v) and not all(is_suitable_inline_table(t, ctx) for t in v):
+            tables.extend((k, t, True) for t in v)
+        else:
+            literals.append((k, v))
+
+    if inside_aot or name and (literals or not tables):
+        yielded = True
+        yield f"[[{name}]]\n" if inside_aot else f"[{name}]\n"
+
+    if literals:
+        yielded = True
+        for k, v in literals:
+            yield f"{format_key_part(k)} = {format_literal(v, ctx)}\n"
+
+    for k, v, in_aot in tables:
+        if yielded:
+            yield "\n"
+        else:
+            yielded = True
+        key_part = format_key_part(k)
+        display_name = f"{name}.{key_part}" if name else key_part
+        yield from gen_table_chunks(v, ctx, name=display_name, inside_aot=in_aot)
+
+
+def format_literal(obj: object, ctx: Context, *, nest_level: int = 0) -> str:
+    if isinstance(obj, bool):
+        return "true" if obj else "false"
+    if isinstance(obj, (int, float, date, datetime)):
+        return str(obj)
+    if isinstance(obj, time):
+        if obj.tzinfo:
+            raise ValueError("TOML does not support offset times")
+        return str(obj)
+    if isinstance(obj, str):
+        return format_string(obj, allow_multiline=ctx.allow_multiline)
+    if isinstance(obj, ARRAY_TYPES):
+        return format_inline_array(obj, ctx, nest_level)
+    if isinstance(obj, Mapping):
+        return format_inline_table(obj, ctx)
+
+    # Lazy import to improve module import time
+    from decimal import Decimal
+
+    if isinstance(obj, Decimal):
+        return format_decimal(obj)
+    raise TypeError(
+        f"Object of type '{type(obj).__qualname__}' is not TOML serializable"
+    )
+
+
+def format_decimal(obj: Decimal) -> str:
+    if obj.is_nan():
+        return "nan"
+    if obj.is_infinite():
+        return "-inf" if obj.is_signed() else "inf"
+    dec_str = str(obj).lower()
+    return dec_str if "." in dec_str or "e" in dec_str else dec_str + ".0"
+
+
+def format_inline_table(obj: Mapping, ctx: Context) -> str:
+    # check cache first
+    obj_id = id(obj)
+    if obj_id in ctx.inline_table_cache:
+        return ctx.inline_table_cache[obj_id]
+
+    if not obj:
+        rendered = "{}"
+    else:
+        rendered = (
+            "{ "
+            + ", ".join(
+                f"{format_key_part(k)} = {format_literal(v, ctx)}"
+                for k, v in obj.items()
+            )
+            + " }"
+        )
+    ctx.inline_table_cache[obj_id] = rendered
+    return rendered
+
+
+def format_inline_array(obj: Union[tuple, list], ctx: Context, nest_level: int) -> str:
+    if not obj:
+        return "[]"
+    item_indent = ctx.indent_str * (1 + nest_level)
+    closing_bracket_indent = ctx.indent_str * nest_level
+    return (
+        "[\n"
+        + ",\n".join(
+            item_indent + format_literal(item, ctx, nest_level=nest_level + 1)
+            for item in obj
+        )
+        + f",\n{closing_bracket_indent}]"
+    )
+
+
+def format_key_part(part: str) -> str:
+    try:
+        only_bare_key_chars = BARE_KEY_CHARS.issuperset(part)
+    except TypeError:
+        raise TypeError(
+            f"Invalid mapping key '{part}' of type '{type(part).__qualname__}'."
+            " A string is required."
+        ) from None
+
+    if part and only_bare_key_chars:
+        return part
+    return format_string(part, allow_multiline=False)
+
+
+def format_string(s: str, *, allow_multiline: bool) -> str:
+    do_multiline = allow_multiline and "\n" in s
+    if do_multiline:
+        result = '"""\n'
+        s = s.replace("\r\n", "\n")
+    else:
+        result = '"'
+
+    pos = seq_start = 0
+    while True:
+        try:
+            char = s[pos]
+        except IndexError:
+            result += s[seq_start:pos]
+            if do_multiline:
+                return result + '"""'
+            return result + '"'
+        if char in ILLEGAL_BASIC_STR_CHARS:
+            result += s[seq_start:pos]
+            if char in COMPACT_ESCAPES:
+                if do_multiline and char == "\n":
+                    result += "\n"
+                else:
+                    result += COMPACT_ESCAPES[char]
+            else:
+                result += "\\u" + hex(ord(char))[2:].rjust(4, "0")
+            seq_start = pos + 1
+        pos += 1
+
+
+def is_aot(obj: Any) -> bool:
+    """Decides if an object behaves as an array of tables (i.e. a nonempty list
+    of dicts)."""
+    return bool(
+        isinstance(obj, ARRAY_TYPES)
+        and obj
+        and all(isinstance(v, Mapping) for v in obj)
+    )
+
+
+def is_suitable_inline_table(obj: Mapping, ctx: Context) -> bool:
+    """Use heuristics to decide if the inline-style representation is a good
+    choice for a given table."""
+    rendered_inline = f"{ctx.indent_str}{format_inline_table(obj, ctx)},"
+    return len(rendered_inline) <= MAX_LINE_LENGTH and "\n" not in rendered_inline

easybuild/tools/filetools.py

@@ -3249,3 +3249,11 @@ def create_non_existing_paths(paths, max_tries=10000):
         set_gid_sticky_bits(path, recursive=True)
 
     return non_existing_paths
+
+
+def dump_toml(o: dict) -> str:
+    """Convert the input directory to a string in TOML format"""
+    # Import and redirect here on purpose as the conversion method (or Python package)
+    # is an implementation detail that can be changed at any time
+    import easybuild.tools._toml_writer
+    return easybuild.tools._toml_writer.dumps(o)

easybuild/tools/tomllib/tomli/README.md

@@ -1,6 +1,6 @@
 # Tomllib
 
-Vendored `tomli` from https://github.com/hukkin/tomli version 2.3.0.
+Vendored [`tomli`](https://github.com/hukkin/tomli) version 2.3.0, commit 3fccd16.
 PEP 680 added a version of it as `tomllib` to Python 3.11.
 
 Patched to remove features not available in Python 3.6, mostly type hints.

setup.py

@@ -75,7 +75,8 @@ def find_rel_test():
     "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", "easybuild.tools.containers",
     "easybuild.tools.deprecated", "easybuild.tools.job", "easybuild.tools.toolchain",
     "easybuild.tools.module_naming_scheme", "easybuild.tools.package", "easybuild.tools.package.package_naming_scheme",
-    "easybuild.tools.py2vs3", "easybuild.tools.repository", "easybuild.tools.tomllib", "easybuild.tools.tomllib.tomli",
+    "easybuild.tools.py2vs3", "easybuild.tools.repository",
+    "easybuild.tools.tomllib", "easybuild.tools.tomllib.tomli", "easybuild.tools._toml_writer",
     "test.framework", "test",
 ]
 

test/framework/general.py

@@ -157,29 +157,42 @@ def test_import_available_modules(self):
     def test_vendored_packages(self):
         """Smoke-test for vendored packages"""
         from easybuild.tools import tomllib
+        from easybuild.tools.filetools import dump_toml
         # Example from toml.io
-        res = tomllib.loads("""
+        res = tomllib.loads('''
             title = "TOML Example"
+            description = """Test
+                this\u1234
+            """
 
             [owner]
             name = "Tom Preston-Werner"
             dob = 1979-05-27T07:32:00-08:00
+            odt1 = 1979-05-27T07:32:00Z
 
             [database]
             enabled = true
             ports = [ 8000, 8001, 8002 ]
             data = [ ["delta", "phi"], [3.14] ]
-            temp_targets = { cpu = 79.5, case = 72.0 }
+            temp_targets = { cpu = { x = 79.5 }, case = -inf }
 
             [servers]
-
             [servers.alpha]
             role = "frontend"
-
             [servers.beta]
             role = "backend"
-        """)
+
+            [[products]]
+            name = "Hammer"
+            [[products]]
+            [[products]]
+            name = "Nail"
+        ''')
         self.assertTrue(res)  # Should just be non-empty
+        # Dumping should round-trip
+        res_str = dump_toml(res)
+        res_parsed = tomllib.loads(res_str)
+        self.assertEqual(res_parsed, res)
 
 
 def suite(loader=None):