[python] Move from toml library to tomli + tomli-w

`toml` Python library (for parsing and dumping TOML files) is buggy and
doesn't support the full spec of TOML 1.0.0. This commit replaces it
with more robust `tomli` (for parsing) and `tomli-w` (for dumping).

Signed-off-by: Dmitrii Kuvaiskii <[email protected]>

Author: Dmitrii Kuvaiskii

Diff for: .ci/ubuntu18.04.dockerfile

@@ -86,7 +86,8 @@ RUN git clone https://github.com/giltene/wrk2.git \
 RUN python3 -m pip install -U \
     'Sphinx==1.8' \
     'sphinx_rtd_theme<1' \
-    'toml>=0.10' \
+    'tomli>=1.1.0' \
+    'tomli-w>=1.0.0' \
     'meson>=0.56,<0.57' \
     'docutils>=0.17,<0.18'
 

Diff for: .ci/ubuntu20.04.dockerfile

@@ -64,7 +64,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3-pytest-xdist \
     python3-scipy \
     python3-sphinx-rtd-theme \
-    python3-toml \
     shellcheck \
     sphinx-doc \
     sqlite3 \
@@ -87,6 +86,8 @@ RUN git clone https://github.com/giltene/wrk2.git \
 # the earliest supported minor version (pip implicitly installs latest version satisfying the
 # specification)
 RUN python3 -m pip install -U \
+    'tomli>=1.1.0' \
+    'tomli-w>=1.0.0' \
     'meson>=0.56,<0.57' \
     'docutils>=0.17,<0.18'
 

Diff for: Documentation/devel/building.rst

@@ -38,11 +38,11 @@ Run the following command on Ubuntu LTS to install dependencies::
     sudo apt-get install -y build-essential \
         autoconf bison gawk nasm ninja-build python3 python3-click \
         python3-jinja2 python3-pyelftools wget
-    sudo python3 -m pip install 'meson>=0.56' 'toml>=0.10'
+    sudo python3 -m pip install 'meson>=0.56' 'tomli>=1.1.0' 'tomli-w>=1.0.0'
 
-You can also install Meson and python3-toml from apt instead of pip, but only if
-your distro is new enough to have Meson >= 0.56 and python3-toml >= 0.10 (Debian
-11, Ubuntu 20.10).
+You can also install Meson, python3-tomli and python3-tomli-w from apt instead
+of pip, but only if your distro is new enough to have Meson >= 0.56,
+python3-tomli >= 1.1.0 and python3-tomli-w >= 1.0.0 (e.g. Ubuntu 22.04).
 
 For GDB support and to run all tests locally you also need to install::
 

Diff for: Documentation/requirements.txt

@@ -1,7 +1,7 @@
 sphinx==1.8.0
 breathe<4.13.0
 sphinx_rtd_theme<1
-toml>=0.10
+tomli>=1.1.0
 
 # Work around Sphinx/docutils incompatibility, see https://github.com/sphinx-doc/sphinx/issues/9727.
 # TODO: This shouldn't be necessary once we upgrade to newer Sphinx (Sphinx 4.2.0 is not yet

Diff for: libos/test/regression/file_check_policy_strict.manifest.template

@@ -21,14 +21,14 @@ sgx.file_check_policy = "strict"
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",
   "file:{{ gramine.runtimedir(libc) }}/",
-]
 
-# below entry in sgx.trusted_files is to test TOML-table syntax without `sha256`
-[[sgx.trusted_files]]
-uri = "file:{{ binary_dir }}/{{ entrypoint }}"
+  # test TOML-table syntax without `sha256`
+  { uri = "file:{{ binary_dir }}/{{ entrypoint }}" },
+
+  # test TOML-table syntax with `sha256` (trusted_testfile has hard-coded contents, so we can use
+  # pre-calculated SHA256 hash)
+  { uri = "file:trusted_testfile", sha256 = "41dacdf1e6d0481d3b1ab1a91f93139db02b96f29cfdd3fb0b819ba1e33cafc4" },
 
-# below entry in sgx.trusted_files is for testing purposes (trusted_testfile has
-# hard-coded contents, so we can use pre-calculated SHA256 hash)
-[[sgx.trusted_files]]
-uri = "file:trusted_testfile"
-sha256 = "41dacdf1e6d0481d3b1ab1a91f93139db02b96f29cfdd3fb0b819ba1e33cafc4"
+  # test correct parsing of `\\x2d` sequence (previously used `toml` Python parser had a bug)
+  { uri = "file:nonexisting\\x2dfile", sha256 = "0123456789012345678901234567890123456789012345678901234567890123" },
+]

Diff for: python/gramine-gen-depend

@@ -10,12 +10,12 @@ import click
 from graminelibos import Manifest, _CONFIG_PKGLIBDIR
 
 @click.command()
-@click.option('--manifest', '-m', 'manifest_file', type=click.File('r', encoding='utf-8'),
-              required=True, help='Input .manifest file')
+@click.option('--manifest', '-m', 'manifest_file', type=click.File('rb'), required=True,
+              help='Input .manifest file')
 @click.option('--libpal', '-l', type=click.Path(exists=True, dir_okay=False),
               default=os.path.join(_CONFIG_PKGLIBDIR, 'sgx/libpal.so'), help='Input libpal file',
               show_default=True)
-@click.option('--output', '-o', type=click.File('w', encoding='utf-8'), required=True,
+@click.option('--output', '-o', type=click.File('wb'), required=True,
               help='Output .manifest.d file')
 def main(manifest_file, libpal, output):
     manifest = Manifest.load(manifest_file)

Diff for: python/gramine-manifest

@@ -21,7 +21,7 @@ def validate_define(_ctx, _param, values):
 @click.option('--string', '-c')
 @click.option('--define', '-D', multiple=True, callback=validate_define)
 @click.argument('infile', type=click.File('r'), required=False)
-@click.argument('outfile', type=click.File('w'), default='-')
+@click.argument('outfile', type=click.File('wb'), default='-')
 def main(string, define, infile, outfile):
     if not bool(string) ^ bool(infile):
         click.get_current_context().fail('specify exactly one of (infile, -c)')

Diff for: python/gramine-sgx-sign

@@ -33,7 +33,7 @@ def main(output, libpal, key, manifest_file, sigfile, depfile, verbose):
 
     expanded = manifest.expand_all_trusted_files()
 
-    with open(output, 'w', encoding='utf-8') as f:
+    with open(output, 'wb') as f:
         manifest.dump(f)
 
     if not sigfile:

Diff for: python/graminelibos/manifest.py

@@ -12,7 +12,8 @@
 import os
 import pathlib
 
-import toml
+import tomli
+import tomli_w
 
 from . import _env
 
@@ -38,7 +39,10 @@ def uri2path(uri):
     return pathlib.Path(uri[len('file:'):])
 
 def append_tf(trusted_files, uri, hash_):
-    trusted_files.append({'uri': uri, 'sha256': hash_})
+    if hash_ is not None:
+        trusted_files.append({'uri': uri, 'sha256': hash_})
+    else:
+        trusted_files.append({'uri': uri})
 
 def append_trusted_dir_or_file(trusted_files, val, expanded):
     if isinstance(val, dict):
@@ -81,7 +85,7 @@ class Manifest:
     """
 
     def __init__(self, manifest_str):
-        manifest = toml.loads(manifest_str)
+        manifest = tomli.loads(manifest_str)
 
         sgx = manifest.setdefault('sgx', {})
         sgx.setdefault('trusted_files', [])
@@ -104,7 +108,6 @@ def __init__(self, manifest_str):
             raise ValueError("Unsupported trusted files syntax, more info: " +
                   "https://gramine.readthedocs.io/en/latest/manifest-syntax.html#trusted-files")
 
-        # Current toml versions (< 1.0) do not support non-homogeneous arrays
         trusted_files = []
         for tf in sgx['trusted_files']:
             if isinstance(tf, dict) and 'uri' in tf:
@@ -150,10 +153,10 @@ def load(cls, f):
         return cls.loads(f.read())
 
     def dumps(self):
-        return toml.dumps(self._manifest)
+        return tomli_w.dumps(self._manifest)
 
     def dump(self, f):
-        toml.dump(self._manifest, f)
+        tomli_w.dump(self._manifest, f)
 
     def expand_all_trusted_files(self):
         """Expand all trusted files entries.

Diff for: python/graminelibos/util_tests.py

@@ -8,7 +8,7 @@
 import subprocess
 import sys
 
-import toml
+import tomli
 
 from . import ninja_syntax, _CONFIG_SYSLIBDIR, _CONFIG_PKGLIBDIR
 
@@ -47,7 +47,8 @@ class TestConfig:
     def __init__(self, path):
         self.config_path = path
 
-        data = toml.load(path)
+        with open(path, "rb") as f:
+            data = tomli.load(f)
 
         self.manifests = self.get_manifests(data)
         arch = platform.machine()