diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 37cb6ead..aa0923e3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -38,6 +38,12 @@ ## New Features +- Add support for linting code examples found in *docstrings*. + + A new module `frequenz.repo.config.pytest.examples` is added with an utility function to be able to easily collect and lint code examples in *docstrings*. + + There is also a new optional dependency `extra-lint-examples` to easily pull the dependencies needed to do this linting. Please have a look at the documentation in the `frequenz.repo.config` package for more details. + ### Cookiecutter template - Add a new GitHub workflow to check that release notes were updated. diff --git a/mkdocs.yml b/mkdocs.yml index c54d2d7f..fdc6d1db 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -109,6 +109,7 @@ plugins: - https://nox.thea.codes/en/stable/objects.inv - https://oprypin.github.io/mkdocs-gen-files/objects.inv - https://setuptools.pypa.io/en/latest/objects.inv + - https://sybil.readthedocs.io/en/stable/objects.inv - https://typing-extensions.readthedocs.io/en/stable/objects.inv - search - section-index diff --git a/pyproject.toml b/pyproject.toml index 3df574fe..6c26790c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,11 @@ api = [ app = [] lib = [] model = [] +extra-lint-examples = [ + "pylint >= 2.17.3, < 3", + "pytest >= 7.3.0, < 8", + "sybil >= 5.0.3, < 6", +] dev-docstrings = [ "pydocstyle == 6.3.0", "darglint == 1.8.1", @@ -124,7 +129,7 @@ disable = [ ] [[tool.mypy.overrides]] -module = ["cookiecutter", "cookiecutter.*"] +module = ["cookiecutter", "cookiecutter.*", "sybil", "sybil.*"] ignore_missing_imports = true [tool.pytest.ini_options] diff --git a/src/frequenz/repo/config/__init__.py b/src/frequenz/repo/config/__init__.py index 39d1ff54..0af02df7 100644 --- a/src/frequenz/repo/config/__init__.py +++ b/src/frequenz/repo/config/__init__.py @@ -207,6 +207,77 @@ - path/to/my/custom/script.py ``` +## `pytest` (running tests) + +### Linting examples in the source code's *docstrings* + +To make sure the examples included in your source code's *docstrings* are valid, you can +use [`pytest`](https://pypi.org/project/pytest/) to automatically collect all the +examples wrapped in triple backticks (````python`) within our docstrings and validate +them using [`pylint`](https://pypi.org/project/pylint/). + +To do so there is some setup that's needed: + +1. Add a `conftest.py` file to the root directory containing your source code with the + following contents: + + ```python + from frequenz.repo.config.pytest import examples + from sybil import Sybil + + pytest_collect_file = Sybil(**examples.get_sybil_arguments()).pytest() + ``` + + Unfortunately, because of how Sybil works, the [`Sybil`][sybil.Sybil] class needs to + be instantiated in the `conftest.py` file. To easily do this, the convenience + function + [`get_sybil_arguments()`][frequenz.repo.config.pytest.examples.get_sybil_arguments] + is provided to get the arguments to pass to the `Sybil()` constructor to be able to + collect and lint the examples. + +2. Add the following configuration to your `pyproject.toml` file (see + the [`nox` section](#pyprojecttoml-configuration) for details on how to configure + dependencies to play nicely with `nox`): + + ```toml + [project.optional-dependencies] + # ... + dev-pytest = [ + # ... + "frequenz-repo-config[extra-lint-examples] == 0.5.0", + ] + # ... + [[tool.mypy.overrides]] + module = [ + # ... + "sybil", + "sybil.*", + ] + ignore_missing_imports = true + # ... + [tool.pytest.ini_options] + testpaths = [ + # ... + "src", + ] + ``` + + This will make sure that you have the appropriate dependencies installed to run the + the tests linting and that `mypy` doesn't complain about the `sybil` module not being + typed. + +3. Exclude the `src/conftest.py` file from the distribution package, as it shouldn't be + shipped with the code, it is only for delelopment purposes. To do so, add the + following line to the `MANIFEST.in` file: + + ``` + # ... + exclude src/conftest.py + ``` + +Now you should be able to run `nox -s pytest` (or `pytest` directly) and see the tests +linting the examples in your code's *docstrings*. + # APIs ## Protobuf configuation diff --git a/src/frequenz/repo/config/pytest/__init__.py b/src/frequenz/repo/config/pytest/__init__.py new file mode 100644 index 00000000..bea80332 --- /dev/null +++ b/src/frequenz/repo/config/pytest/__init__.py @@ -0,0 +1,12 @@ +# License: MIT +# Copyright © 2023 Frequenz Energy-as-a-Service GmbH + +"""Pytest utilities. + +This package contains utilities for testing with [`pytest`](https://pypi.org/project/pytest/). + +The following modules are available: + +- [`examples`][frequenz.repo.config.pytest.examples]: Utilities to enable linting of + code examples in docstrings. +""" diff --git a/src/frequenz/repo/config/pytest/examples.py b/src/frequenz/repo/config/pytest/examples.py new file mode 100644 index 00000000..b465ce54 --- /dev/null +++ b/src/frequenz/repo/config/pytest/examples.py @@ -0,0 +1,234 @@ +# License: MIT +# Copyright © 2023 Frequenz Energy-as-a-Service GmbH + +"""Utility to enable linting of code examples in docstrings. + +Code examples are often wrapped in triple backticks (````python`) within our docstrings. +This plugin extracts these code examples and validates them using pylint. + +The main utility function is +[`get_sybil_arguments()`][frequenz.repo.config.pytest.examples.get_sybil_arguments], +which returns a dictionary that can be used to pass to the [`Sybil()`][sybil.Sybil] +constructor. + +You still need to create a `conftest.py` file in the root of your project's sources, +typically `src/conftest.py`, with the following contents: + +```python +from frequenz.repo.config.pytest import examples +from sybil import Sybil + +pytest_collect_file = Sybil(**examples.get_sybil_arguments()).pytest() +``` +""" + +import ast +import os +import subprocess +from pathlib import Path +from typing import Any + +from sybil import Example +from sybil.evaluators.python import pad +from sybil.parsers.abstract.lexers import textwrap +from sybil.parsers.myst import CodeBlockParser + +_PYLINT_DISABLE_COMMENT = ( + "# pylint: {}=unused-import,wildcard-import,unused-wildcard-import" +) + +_FORMAT_STRING = """ +# Generated auto-imports for code example +{disable_pylint} +{imports} +{enable_pylint} + +{code}""" + + +def get_sybil_arguments() -> dict[str, Any]: + """Get the arguments to pass when instantiating the Sybil object to lint docs examples. + + Returns: + The arguments to pass when instantiating the Sybil object. + """ + return { + "parsers": [_CustomPythonCodeBlockParser()], + "patterns": ["*.py"], + } + + +def _get_import_statements(code: str) -> list[str]: + """Get all import statements from a given code string. + + Args: + code: The code to extract import statements from. + + Returns: + A list of import statements. + """ + tree = ast.parse(code) + import_statements: list[str] = [] + + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + import_statement = ast.get_source_segment(code, node) + assert import_statement is not None + import_statements.append(import_statement) + + return import_statements + + +def _path_to_import_statement(path: Path) -> str: + """Convert a path to a Python file to an import statement. + + Args: + path: The path to convert. + + Returns: + The import statement. + + Raises: + ValueError: If the path does not point to a Python file. + """ + # Make the path relative to the present working directory + if path.is_absolute(): + path = path.relative_to(Path.cwd()) + + # Check if the path is a Python file + if path.suffix != ".py": + raise ValueError("Path must point to a Python file (.py)") + + # Remove 'src' prefix if present + parts = path.parts + if parts[0] == "src": + parts = parts[1:] + + # Remove the '.py' extension and join parts with '.' + module_path = ".".join(parts)[:-3] + + # Create the import statement + import_statement = f"from {module_path} import *" + return import_statement + + +# We need to add the type ignore comment here because the Sybil library does not +# have type annotations. +class _CustomPythonCodeBlockParser(CodeBlockParser): # type: ignore[misc] + """Code block parser that validates extracted code examples using pylint. + + This parser is a modified version of the default Python code block parser + from the Sybil library. + It uses pylint to validate the extracted code examples. + + All code examples are preceded by the original file's import statements as + well as an wildcard import of the file itself. + This allows us to use the code examples as if they were part of the original + file. + + Additionally, the code example is padded with empty lines to make sure the + line numbers are correct. + + Pylint warnings which are unimportant for code examples are disabled. + """ + + def __init__(self) -> None: + """Initialize the parser.""" + super().__init__("python") + + def evaluate(self, example: Example) -> None | str: + """Validate the extracted code example using pylint. + + Args: + example: The extracted code example. + + Returns: + None if the code example is valid, otherwise the pylint output. + """ + # Get the import statements for the original file + import_header = _get_import_statements(example.document.text) + # Add a wildcard import of the original file + import_header.append( + _path_to_import_statement(Path(os.path.relpath(example.path))) + ) + imports_code = "\n".join(import_header) + + # Dedent the code example + # There is also example.parsed that is already prepared, but it has + # empty lines stripped and thus fucks up the line numbers. + example_code = textwrap.dedent( + example.document.text[example.start : example.end] + ) + # Remove first line (the line with the triple backticks) + example_code = example_code[example_code.find("\n") + 1 :] + + example_with_imports = _FORMAT_STRING.format( + disable_pylint=_PYLINT_DISABLE_COMMENT.format("disable"), + imports=imports_code, + enable_pylint=_PYLINT_DISABLE_COMMENT.format("enable"), + code=example_code, + ) + + # Make sure the line numbers are correct + source = pad( + example_with_imports, + example.line - imports_code.count("\n") - _FORMAT_STRING.count("\n"), + ) + + # pylint disable parameters + pylint_disable_params = [ + "missing-module-docstring", + "missing-class-docstring", + "missing-function-docstring", + "reimported", + "unused-variable", + "no-name-in-module", + "await-outside-async", + ] + + response = _validate_with_pylint(source, example.path, pylint_disable_params) + + if len(response) > 0: + return ( + f"Pylint validation failed for code example:\n" + f"{example_with_imports}\nOutput: " + "\n".join(response) + ) + + return None + + +def _validate_with_pylint( + code_example: str, path: str, disable_params: list[str] +) -> list[str]: + """Validate a code example using pylint. + + Args: + code_example: The code example to validate. + path: The path to the original file. + disable_params: The pylint disable parameters. + + Returns: + A list of pylint messages. + """ + try: + pylint_command = [ + "pylint", + "--disable", + ",".join(disable_params), + "--from-stdin", + path, + ] + + subprocess.run( + pylint_command, + input=code_example, + text=True, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as exception: + output = exception.output + assert isinstance(output, str) + return output.splitlines() + + return [] diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index 8386f134..a9bfec45 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -19,6 +19,6 @@ def test_optional_dependencies() -> None: defined = { k for k in pyproject_toml["project"]["optional-dependencies"].keys() - if k != "dev" and not k.startswith("dev-") + if k != "dev" and not k.startswith("dev-") and not k.startswith("extra-") } assert defined == expected, utils.MSG_UNEXPECTED_REPO_TYPES