Add yaml output (#41)

Adds yaml output. 

* Write yaml if output file ends in yaml or yml extension
* Writes yaml if a "--yaml" cmdline arg was provided
* Adds test for yaml output
* Adds ci tests with a matrix of extra dependencies.
* Adds yaml dependency: `pyyaml` as an extra dependency in pyproject.toml
* Adds yaml dependeny:  `types-pyyaml` as a lint dev dependency.
* Makes the indentation of .json output consistent

Co-authored-by: yfprojects <[email protected]>
Co-authored-by: real-yfprojects <[email protected]>

.github/workflows/ci.yml

@@ -23,8 +23,41 @@ jobs:
       - name: Run linters
         run: make lint
 
+  generate_test_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+      - name: setup python
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        with:
+          python-version: "3.11"
+
+      - name: Extract extras from `pyproject.toml`
+        id: set-matrix
+        shell: python
+        run: |
+          import tomllib
+          import os
+          import json
+          with open('pyproject.toml', 'rb') as f:
+              manifest = tomllib.load(f)
+          yaml = { 'include' : [{ 'extras' : extra} for extra in [''] + list(manifest['tool']['poetry']['extras'])]}
+          out = json.dumps(yaml)
+          print(out)
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+            f.write('matrix=' + out)
+
   test:
+    name: test ${{ matrix.extras && 'with' || '' }} ${{ matrix.extras }}
     runs-on: ubuntu-latest
+    needs: generate_test_matrix
+    strategy:
+      matrix: ${{ fromJson(needs.generate_test_matrix.outputs.matrix) }}
+      fail-fast: false
     steps:
       - name: Checkout
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -33,7 +66,7 @@ jobs:
         id: setup
         uses: ./.github/actions/setup
         with:
-          install-options: --without lint
+          install-options: --without lint ${{ matrix.extras && format('--extras "{0}"', matrix.extras) || '' }}
 
       - name: Run Tests
         run: make test

pyproject.toml

@@ -1,24 +1,26 @@
 [build-system]
-requires = ["poetry-core"]
+requires      = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-name = "req2flatpak"
-version = "0.1.0"
+name        = "req2flatpak"
+version     = "0.1.0"
 description = "Generates a flatpak-builder build module for installing python packages defined in requirements.txt files."
-authors = ["johannesjh <[email protected]>"]
-license = "MIT"
-readme = "README.rst"
+authors     = ["johannesjh <[email protected]>"]
+license     = "MIT"
+readme      = "README.rst"
 
 [tool.poetry.scripts]
 req2flatpak = 'req2flatpak:main'
 
 [tool.poetry.dependencies]
-python = "^3.7.2"
+python    = "^3.7.2"
 packaging = { version = "^21.3", optional = true }
+pyyaml    = { version = "^6.0", optional = true }
 
 [tool.poetry.extras]
 packaging = ["packaging"]
+yaml      = ["pyyaml"]
 
 [tool.poetry.group.lint.dependencies]
 pylama = { extras = [
@@ -28,7 +30,10 @@ pylama = { extras = [
     "toml",
 ], version = "^8.4.1" }
 bandit = { extras = ["toml"], version = "^1.7.4" }
-types-setuptools = "^65.5.0.2" # type stubs for mypy linting
+
+# type stubs for mypy linting
+types-setuptools = "^65.5.0.2"
+types-pyyaml     = "^6.0.12.8"
 
 # [tool.poetry.group.tests.dependencies]
 pydocstyle = "<6.2"
@@ -37,18 +42,18 @@ pydocstyle = "<6.2"
 optional = true
 
 [tool.poetry.group.docs.dependencies]
-sphinx = "^5.3.0"
-sphinx-argparse = "^0.3.2"
+sphinx                           = "^5.3.0"
+sphinx-argparse                  = "^0.3.2"
 sphinx-rtd-theme-github-versions = "^1.1"
 
 [tool.isort]
-profile = "black"
+profile        = "black"
 skip_gitignore = true
 
 [tool.pylama]
 max_line_length = 88
-concurrent = true
-linters = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy"
+concurrent      = true
+linters         = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy"
 
 [tool.pylama.linter.pycodestyle]
 ignore = "W503,E203,E501"
@@ -63,8 +68,8 @@ ignore = ["D202", "D203", "D205", "D401", "D212"]
 
 [tool.pylint]
 format.max-line-length = 88
-main.disable = ["C0301"]            # ignore line-too-long
-basic.good-names = ["e", "f", "py"]
+main.disable           = ["C0301"]        # ignore line-too-long
+basic.good-names       = ["e", "f", "py"]
 
 [tool.bandit]
 exclude_dirs = ['tests']

req2flatpak.py

@@ -64,6 +64,10 @@
 
 logger = logging.getLogger(__name__)
 
+try:
+    import yaml
+except ImportError:
+    yaml = None  # type: ignore
 
 # =============================================================================
 # Helper functions / semi vendored code
@@ -73,7 +77,6 @@
     # added with py 3.8
     from functools import cached_property  # type: ignore[attr-defined]
 except ImportError:
-
     # Inspired by the implementation in the standard library
     # pylint: disable=invalid-name,too-few-public-methods
     class cached_property:  # type: ignore[no-redef]
@@ -551,9 +554,7 @@ def downloads(
         preferred downloads are returned first.
         """
         cache = set()
-        for (platform_tag, download) in product(
-            platform.python_tags, release.downloads
-        ):
+        for platform_tag, download in product(platform.python_tags, release.downloads):
             if download in cache:
                 continue
             if wheels_only and not download.is_wheel:
@@ -634,12 +635,30 @@ def sources(downloads: Iterable[Download]) -> List[dict]:
     @classmethod
     def build_module_as_str(cls, *args, **kwargs) -> str:
         """
-        Generates a build module for inclusion in a flatpak-builder build manifest.
+        Generate JSON build module for inclusion in a flatpak-builder build manifest.
+
+        The args and kwargs are the same as in
+        :py:meth:`~req2flatpak.FlatpakGenerator.build_module`
+        """
+        return json.dumps(cls.build_module(*args, **kwargs), indent=4)
+
+    @classmethod
+    def build_module_as_yaml_str(cls, *args, **kwargs) -> str:
+        """
+        Generate YAML build module for inclusion in a flatpak-builder build manifest.
 
         The args and kwargs are the same as in
         :py:meth:`~req2flatpak.FlatpakGenerator.build_module`
         """
-        return json.dumps(cls.build_module(*args, **kwargs), indent=2)
+        # optional dependency, not imported at top
+        if not yaml:
+            raise ImportError(
+                "Package `pyyaml` has to be installed for the yaml format."
+            )
+
+        return yaml.dump(
+            cls.build_module(*args, **kwargs), default_flow_style=False, sort_keys=False
+        )
 
 
 # =============================================================================
@@ -676,12 +695,22 @@ def cli_parser() -> argparse.ArgumentParser:
         default=False,
         help="Uses a persistent cache when querying pypi.",
     )
+    parser.add_argument(
+        "--yaml",
+        action="store_true",
+        help="Write YAML instead of the default JSON.  Needs the 'pyyaml' package.",
+    )
+
     parser.add_argument(
         "--outfile",
         "-o",
         nargs="?",
         type=argparse.FileType("w"),
         default=sys.stdout,
+        help="""
+            By default, writes JSON but specify a '.yaml' extension and YAML
+            will be written instead, provided you have the 'pyyaml' package.
+        """,
     )
     parser.add_argument(
         "--platform-info",
@@ -698,21 +727,33 @@ def cli_parser() -> argparse.ArgumentParser:
     return parser
 
 
-def main():
+def main():  # pylint: disable=too-many-branches
     """Main function that provides req2flatpak's commandline interface."""
 
     # process commandline arguments
     parser = cli_parser()
     options = parser.parse_args()
 
     # stream output to a file or to stdout
-    output_stream = options.outfile if hasattr(options.outfile, "write") else sys.stdout
+    if hasattr(options.outfile, "write"):
+        output_stream = options.outfile
+        if pathlib.Path(output_stream.name).suffix.casefold() in (".yaml", ".yml"):
+            options.yaml = True
+    else:
+        output_stream = sys.stdout
+
+    if options.yaml and not yaml:
+        parser.error(
+            "Outputing YAML requires 'pyyaml' package: try 'pip install pyyaml'"
+        )
 
     # print platform info if requested, and exit
     if options.platform_info:
-        json.dump(
-            asdict(PlatformFactory.from_current_interpreter()), output_stream, indent=4
-        )
+        info = asdict(PlatformFactory.from_current_interpreter())
+        if options.yaml:
+            yaml.dump(info, output_stream, default_flow_style=False, sort_keys=False)
+        else:
+            json.dump(info, output_stream, indent=4)
         parser.exit()
 
     # print installed packages if requested, and exit
@@ -770,10 +811,16 @@ def main():
     }
 
     # generate flatpak-builder build module
-    build_module = FlatpakGenerator.build_module(requirements, downloads)
-
-    # write output
-    json.dump(build_module, output_stream, indent=4)
+    if options.yaml:
+        # write yaml
+        output_stream.write(
+            FlatpakGenerator.build_module_as_yaml_str(requirements, downloads)
+        )
+    else:
+        # write json
+        output_stream.write(
+            FlatpakGenerator.build_module_as_str(requirements, downloads)
+        )
 
 
 if __name__ == "__main__":

tests/test_req2flatpak.py

@@ -4,11 +4,18 @@
 import tempfile
 import unittest
 from abc import ABC
+from contextlib import contextmanager
 from itertools import product
 from pathlib import Path
-from typing import List
+from typing import Generator, List
+from unittest import skipUnless
 from unittest.mock import patch
 
+try:
+    import yaml
+except ImportError:
+    yaml = None  # type: ignore
+
 from req2flatpak import (
     DownloadChooser,
     FlatpakGenerator,
@@ -81,6 +88,17 @@ def validate_build_module(self, build_module: dict) -> None:
         """To be implemented by subclasses."""
         raise NotImplementedError
 
+    @contextmanager
+    def requirements_file(
+        self,
+    ) -> Generator[tempfile._TemporaryFileWrapper, None, None]:
+        """Create a temporary requirements file."""
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file:
+            req_file.write("\n".join(self.requirements))
+            req_file.flush()
+            req_file.seek(0)
+            yield req_file
+
     def _run_r2f(self, args: List[str]) -> subprocess.CompletedProcess:
         """Runs req2flatpak's cli in a subprocess."""
         cwd = Path(__file__).parent / ".."
@@ -95,18 +113,36 @@ def test_cli_with_reqs_as_args(self):
         build_module = json.loads(result.stdout)
         self.validate_build_module(build_module)
 
+    @skipUnless(yaml, "The yaml extra dependency is needed for this feature.")
+    def test_cli_with_reqs_as_args_yaml(self):
+        """Runs req2flatpak in yaml mode by passing requirements as cmdline arg."""
+        args = ["--requirements"] + self.requirements
+        args += ["--target-platforms"] + self.target_platforms
+        args += ["--yaml"]
+        result = self._run_r2f(args)
+        build_module = yaml.safe_load(result.stdout)
+        self.validate_build_module(build_module)
+
     def test_cli_with_reqs_as_file(self):
         """Runs req2flatpak by passing requirements as requirements.txt file."""
-        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file:
-            req_file.write("\n".join(self.requirements))
-            req_file.flush()
-            req_file.seek(0)
+        with self.requirements_file() as req_file:
             args = ["--requirements-file", req_file.name]
             args += ["--target-platforms"] + self.target_platforms
             result = self._run_r2f(args)
             build_module = json.loads(result.stdout)
             self.validate_build_module(build_module)
 
+    @skipUnless(yaml, "The yaml extra dependency is needed for this feature.")
+    def test_cli_with_reqs_as_file_yaml(self):
+        """Runs req2flatpak by passing requirements as requirements.txt file."""
+        with self.requirements_file() as req_file:
+            args = ["--requirements-file", req_file.name]
+            args += ["--target-platforms"] + self.target_platforms
+            args += ["--yaml"]
+            result = self._run_r2f(args)
+            build_module = yaml.safe_load(result.stdout)
+            self.validate_build_module(build_module)
+
     def test_api(self):
         """Runs req2flatpak by calling its python api."""
         platforms = [