migrate pyuavcan.dsdl generation core to lang/py

src/nunavut/_utilities.py

@@ -93,7 +93,9 @@ def test_truth(cls, ynd_value: "YesNoDefault", default_value: bool) -> bool:
     DEFAULT = 2
 
 
-def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pathlib.Path, None, None]:
+def iter_package_resources(
+        pkg_name: str, 
+        *suffix_filters: str) -> Generator[tuple[pathlib.Path, pathlib.Path], None, None]:
     """
     >>> from nunavut._utilities import iter_package_resources
     >>> rs = [x for x in iter_package_resources("nunavut.lang", ".py") if x.name == "__init__.py"]
@@ -103,6 +105,7 @@ def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pat
     '__init__.py'
 
     """
-    for resource in importlib_resources.files(pkg_name).iterdir():
+    root = importlib_resources.files(pkg_name)
+    for resource in root.rglob("*"):
         if any(suffix == resource.suffix for suffix in suffix_filters):
-            yield resource
+            yield (root, resource)

src/nunavut/jinja/__init__.py

@@ -792,7 +792,7 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any):
     # +-----------------------------------------------------------------------+
     # | AbstractGenerator
     # +-----------------------------------------------------------------------+
-    def get_templates(self) -> typing.Iterable[pathlib.Path]:
+    def get_templates(self) -> typing.Iterable[tuple[pathlib.Path, pathlib.Path]]:
         files = []
         target_language = self.language_context.get_target_language()
 
@@ -815,7 +815,7 @@ def generate_all(self, is_dryrun: bool = False, allow_overwrite: bool = True) ->
 
     def _generate_all(
         self, target_language: nunavut.lang.Language, sub_folders: pathlib.Path, is_dryrun: bool, allow_overwrite: bool
-    ) -> typing.Iterable[pathlib.Path]:
+    ) -> typing.Iterable[tuple[pathlib.Path, pathlib.Path]]:
         target_path = pathlib.Path(self.namespace.get_support_output_folder()) / sub_folders
 
         line_pps = []  # type: typing.List['nunavut.postprocessors.LinePostProcessor']
@@ -830,8 +830,8 @@ def _generate_all(
                     raise ValueError("PostProcessor type {} is unknown.".format(type(pp)))
 
         generated = []  # type: typing.List[pathlib.Path]
-        for resource in self.get_templates():
-            target = (target_path / resource.name).with_suffix(target_language.extension)
+        for root, resource in self.get_templates():
+            target = (target_path / resource.relative_to(root)).with_suffix(target_language.extension)
             logger.info("Generating support file: %s", target)
             if not self._support_enabled:
                 self._remove_header(target, is_dryrun, allow_overwrite)

src/nunavut/lang/__init__.py

@@ -51,7 +51,7 @@ def load_language_module(cls, language_name: str) -> "types.ModuleType":
     @classmethod
     def _load_config(cls, *additional_config_files: pathlib.Path) -> LanguageConfig:
         parser = LanguageConfig()
-        for resource in iter_package_resources(__name__, ".yaml"):
+        for root, resource in iter_package_resources(__name__, ".yaml"):
             ini_string = resource.read_text()
             parser.read_string(ini_string)
         for additional_path in additional_config_files:

src/nunavut/lang/_common.py

@@ -33,8 +33,8 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t
             for namespace_part in self._language.support_namespace:
                 namespace_path = namespace_path / pathlib.Path(namespace_part)
             path_list += [
-                (namespace_path / pathlib.Path(p.name).with_suffix(output_extension)).as_posix()
-                for p in self._language.support_files
+                (namespace_path / p.relative_to(r).with_suffix(output_extension)).as_posix()
+                for r, p in self._language.support_files
             ]
 
         prefer_system_includes = self._language.get_config_value_as_bool("prefer_system_includes", False)

src/nunavut/lang/py/__init__.py

@@ -7,10 +7,15 @@
     Filters for generating python. All filters in this
     module will be available in the template's global namespace as ``py``.
 """
+import base64
 import builtins
 import functools
+import gzip
+import itertools
 import keyword
+import pickle
 import typing
+from typing import Iterable, Sequence
 
 import pydsdl
 
@@ -332,3 +337,51 @@ def filter_longest_id_length(language: Language, attributes: typing.List[pydsdl.
         return max(map(len, map(functools.partial(filter_id, language), attributes)))
     else:
         return max(map(len, attributes))
+
+
+@template_language_filter(__name__)
+def filter_pickle(language: Language, instance: typing.Any) -> str:
+    pickled: str = base64.b85encode(gzip.compress(pickle.dumps(instance, protocol=4))).decode().strip()
+    segment_gen = map("".join, itertools.zip_longest(*([iter(pickled)] * 100), fillvalue=""))
+    return "\n".join(repr(x) for x in segment_gen)
+
+
+@template_language_filter(__name__)
+def filter_numpy_scalar_type(language: Language, t: pydsdl.Any) -> str:
+    def pick_width(w: int) -> int:
+        for o in [8, 16, 32, 64]:
+            if w <= o:
+                return o
+        raise ValueError(f"Invalid bit width: {w}")  # pragma: no cover
+
+    if isinstance(t, pydsdl.BooleanType):
+        return "bool"  # TODO: numpy.bool is deprecated in v1.20
+    if isinstance(t, pydsdl.SignedIntegerType):
+        return f"_np_.int{pick_width(t.bit_length)}"
+    if isinstance(t, pydsdl.UnsignedIntegerType):
+        return f"_np_.uint{pick_width(t.bit_length)}"
+    if isinstance(t, pydsdl.FloatType):
+        return f"_np_.float{pick_width(t.bit_length)}"
+    assert not isinstance(t, pydsdl.PrimitiveType), "Forgot to handle some primitive types"
+    return "object"  # TODO: numpy.object is deprecated in v1.20
+
+
+@template_language_filter(__name__)
+def filter_newest_minor_version_aliases(
+        language: Language,
+        tys: Iterable[pydsdl.CompositeType],
+) -> Sequence[tuple[str, pydsdl.CompositeType]]:
+    """
+    Implementation of https://github.com/UAVCAN/nunavut/issues/193
+    """
+    tys = list(tys)
+    return [
+        (
+            f"{name}_{major}",
+            max(
+                (t for t in tys if t.short_name == name and t.version.major == major),
+                key=lambda x: int(x.version.minor),
+            ),
+        )
+        for name, major in sorted({(x.short_name, x.version.major) for x in tys})
+    ]

src/nunavut/lang/py/support/__init__.py

@@ -0,0 +1,63 @@
+#
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright (C) 2018-2021  UAVCAN Development Team  <uavcan.org>
+# This software is distributed under the terms of the MIT License.
+#
+"""
+Contains supporting Python routines to distribute with generated types.
+"""
+import pathlib
+import typing
+from nunavut._utilities import iter_package_resources
+
+# Re-export support utilities for use in pyuavcan.dsdl
+# Need to suppress flake8 warning for these "unused" imports
+from .nunavut_support import serialize as serialize # noqa:F401
+from .nunavut_support import deserialize as deserialize # noqa:F401
+
+from .nunavut_support import CompositeObject as CompositeObject # noqa:F401
+from .nunavut_support import ServiceObject as ServiceObject # noqa:F401
+
+from .nunavut_support import CompositeObjectTypeVar as CompositeObjectTypeVar # noqa:F401
+
+from .nunavut_support import FixedPortObject as FixedPortObject # noqa:F401
+from .nunavut_support import FixedPortCompositeObject as FixedPortCompositeObject # noqa:F401
+from .nunavut_support import FixedPortServiceObject as FixedPortServiceObject # noqa:F401
+
+from .nunavut_support import get_fixed_port_id as get_fixed_port_id # noqa:F401
+from .nunavut_support import get_model as get_model # noqa:F401
+from .nunavut_support import get_class as get_class # noqa:F401
+from .nunavut_support import get_extent_bytes as get_extent_bytes # noqa:F401
+
+from .nunavut_support import get_attribute as get_attribute # noqa:F401
+from .nunavut_support import set_attribute as set_attribute # noqa:F401
+
+
+__version__ = "1.0.0"
+"""Version of the Python support routines."""
+
+
+def list_support_files() -> typing.Generator[pathlib.Path, None, None]:
+    """
+    Get a list of Python support routines embedded in this package.
+
+    .. invisible-code-block: python
+
+        from nunavut.lang.py.support import list_support_files
+        import pathlib
+        support_file_count = 0
+
+    .. code-block:: python
+
+        for path in list_support_files():
+            support_file_count += 1
+            assert path.parent.stem == 'support'
+            assert (path.suffix == '.py' or path.suffix == '.j2')
+
+    .. invisible-code-block: python
+
+        assert support_file_count > 0
+
+    :return: A list of Python support routine resources.
+    """
+    return iter_package_resources(__name__, ".py", ".j2")