CHG: switch to ruff format instead of black entirely (#3435)

* CHG: replace `black` and `bandit` with `ruff` Signed-off-by: Simon Brugman <[email protected]> Co-authored-by: Merel Theisen <[email protected]> Co-authored-by: Sajid Alam <[email protected]>
kedro-org · Jan 17, 2024 · 42b526e · 42b526e
1 parent 9ffda6c
commit 42b526e
Show file tree

Hide file tree

Showing 45 changed files with 140 additions and 148 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ default_stages: [commit, manual]
 
 repos:
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.1.3
+      rev: v0.1.8
       hooks:
         - id: ruff
           name: "ruff on kedro/, tests/ and docs/"
@@ -32,14 +32,6 @@ repos:
           - id: requirements-txt-fixer # Sorts entries in requirements.txt
             exclude: "^kedro/templates/|^features/steps/test_starter/"
 
-    - repo: https://github.com/asottile/blacken-docs
-      rev: v1.12.1
-      hooks:
-          - id: blacken-docs
-            additional_dependencies:
-            - black~=23.0
-            entry: blacken-docs --skip-errors
-
     - repo: local
       hooks:
           - id: imports
@@ -56,9 +48,3 @@ repos:
             exclude: ^features/steps/test_starter
             pass_filenames: false
             entry: make secret-scan
-          - id: bandit
-            name: "Bandit security check"
-            language: system
-            types: [file, python]
-            exclude: ^kedro/templates/|^tests/|^features/steps/test_starter
-            entry: bandit -ll
diff --git a/README.md b/README.md
@@ -39,13 +39,13 @@ Our [Get Started guide](https://docs.kedro.org/en/stable/get_started/install.htm
 
 ## What are the main features of Kedro?
 
-| Feature              | What is this?                                                                                                                                                                                                                                                                                                                                                                                  |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Project Template     | A standard, modifiable and easy-to-use project template based on [Cookiecutter Data Science](https://github.com/drivendata/cookiecutter-data-science/).                                                                                                                                                                                                                                        |
-| Data Catalog         | A series of lightweight data connectors used to save and load data across many different file formats and file systems, including local and network file systems, cloud object stores, and HDFS. The Data Catalog also includes data and model versioning for file-based systems.                                                                                                              |
-| Pipeline Abstraction | Automatic resolution of dependencies between pure Python functions and data pipeline visualisation using [Kedro-Viz](https://github.com/kedro-org/kedro-viz).                                                                                                                                                                                                                                  |
-| Coding Standards     | Test-driven development using [`pytest`](https://github.com/pytest-dev/pytest), produce well-documented code using [Sphinx](http://www.sphinx-doc.org/en/master/), create linted code with support for [`flake8`](https://github.com/PyCQA/flake8), [`isort`](https://github.com/PyCQA/isort) and [`black`](https://github.com/psf/black) and make use of the standard Python logging library. |
-| Flexible Deployment  | Deployment strategies that include single or distributed-machine deployment as well as additional support for deploying on Argo, Prefect, Kubeflow, AWS Batch and Databricks.                                                                                                                                                                                                                  |
+| Feature              | What is this?                                                                                                                                                                                                                                                                                                                                                                                      |
+| -------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Project Template     | A standard, modifiable and easy-to-use project template based on [Cookiecutter Data Science](https://github.com/drivendata/cookiecutter-data-science/).                                                                                                                                                                                                                                            |
+| Data Catalog         | A series of lightweight data connectors used to save and load data across many different file formats and file systems, including local and network file systems, cloud object stores, and HDFS. The Data Catalog also includes data and model versioning for file-based systems.                                                                                                                  |
+| Pipeline Abstraction | Automatic resolution of dependencies between pure Python functions and data pipeline visualisation using [Kedro-Viz](https://github.com/kedro-org/kedro-viz).                                                                                                                                                                                                                                      |
+| Coding Standards     | Test-driven development using [`pytest`](https://github.com/pytest-dev/pytest), produce well-documented code using [Sphinx](http://www.sphinx-doc.org/en/master/), create linted code with support for [`ruff`](https://github.com/astral-sh/ruff) and make use of the standard Python logging library. |
+| Flexible Deployment  | Deployment strategies that include single or distributed-machine deployment as well as additional support for deploying on Argo, Prefect, Kubeflow, AWS Batch and Databricks.                                                                                                                                                                                                                      |
 
 ## How do I use Kedro?
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -8,6 +8,7 @@
 * Removed setuptools dependency
 * Added `source_dir` explicitly in `pyproject.toml` for non-src layout project.
 * `MemoryDataset` entries are now included in free outputs.
+* Removed black dependency and replaced it functionality with `ruff format`.
 
 ## Breaking changes to the API
 * Added logging about not using async mode in `SequentiallRunner` and `ParallelRunner`.

diff --git a/docs/source/deployment/prefect.md b/docs/source/deployment/prefect.md
@@ -163,7 +163,7 @@ def kedro_init(
     context = session.load_context()
     catalog = context.catalog
     logger.info("Registering datasets...")
-    unregistered_ds = pipeline.datasets() - set(catalog.list())  # NOQA
+    unregistered_ds = pipeline.datasets() - set(catalog.list())
     for ds_name in unregistered_ds:
         catalog.add(ds_name, MemoryDataset())
     return {"catalog": catalog, "sess_id": session.session_id}

diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md
@@ -9,34 +9,36 @@ Linting tools check your code for errors such as a missing bracket or line inden
 As a project grows and goes through various stages of development it becomes important to maintain code quality. Using a consistent format and linting your code ensures that it is consistent, readable, and easy to debug and maintain.
 
 ## Set up Python tools
-There are a variety of Python tools available to use with your Kedro projects. This guide shows you how to use [`black`](https://github.com/psf/black) and [`ruff`](https://beta.ruff.rs).
-- **`black`** is a [PEP 8](https://peps.python.org/pep-0008/) compliant opinionated Python code formatter. `black` can
-check for styling inconsistencies and reformat your files in place.
-[You can read more in the `black` documentation](https://black.readthedocs.io/en/stable/).
-- **`ruff`** is a fast linter that replaces `flake8`, `pylint`, `pyupgrade`, `isort` and [more](https://beta.ruff.rs/docs/rules/).
-  - It helps to make your code compliant to [`pep8`](https://pypi.org/project/pep8/).
-  - It reformats code by sorting imports alphabetically and automatically separating them into sections by
-type. [You can read more in the `isort` documentation](https://pycqa.github.io/isort/).
 
+There are a variety of Python tools available to use with your Kedro projects. This guide shows you how to use [`ruff`](https://beta.ruff.rs).
+
+**`ruff`** is a fast linter and formatter that replaces `flake8`, `pylint`, `pyupgrade`, `isort`, `black` and [more](https://beta.ruff.rs/docs/rules/).
+  - It helps to make your code compliant to [PEP 8](https://peps.python.org/pep-0008/).
+  - It reformats code and sorts imports alphabetically and automatically separating them into sections by
+type.
 
 ### Install the tools
-Install `black` and `ruff` by adding the following lines to your project's `requirements.txt`
+Install `ruff` by adding the following lines to your project's `requirements.txt`
 file:
 ```text
-black # Used for formatting code
 ruff # Used for linting, formatting and sorting module imports
-
 ```
+
 To install all the project-specific dependencies, including the linting tools, navigate to the root directory of the
 project and run:
+
 ```bash
 pip install -r requirements.txt
 ```
+
 Alternatively, you can individually install the linting tools using the following shell commands:
+
 ```bash
-pip install black ruff
+pip install ruff
 ```
+
 #### Configure `ruff`
+
 `ruff` read configurations from `pyproject.toml` within your project root. You can enable different rule sets within the `[tool.ruff]` section. For example, the rule set `F` is equivalent to `Pyflakes`.
 
 To start with `ruff`, we recommend adding this section to enable a few basic rules sets.
@@ -50,20 +52,20 @@ select = [
     "I",  # isort
     "PL", # Pylint
 ]
-ignore = ["E501"]  # Black take care off line-too-long
 ```
 
 ```{note}
 It is a good practice to [split your line when it is too long](https://beta.ruff.rs/docs/rules/line-too-long/), so it can be read easily even in a small screen. `ruff` treats this slightly different from `black`, when using together we recommend to disable this rule, i.e. `E501` to avoid conflicts.
 ```
 
 ### Run the tools
+
 Use the following commands to run lint checks:
 ```bash
-black --check <project_root>
+ruff format --check <project_root>
 ruff check <project_root>
 ```
-You can also have `black` automatically format your code by omitting the `--check` flag.
+You can also have `ruff format` automatically format your code by omitting the `--check` flag.
 
 ## Automated formatting and linting with `pre-commit` hooks
 
@@ -72,36 +74,44 @@ These hooks are run before committing your code to your repositories to automati
 making code reviews easier and less time-consuming.
 
 ### Install `pre-commit`
+
 You can install `pre-commit` along with other dependencies by including it in the `requirements.txt` file of your
 Kedro project by adding the following line:
+
 ```text
 pre-commit
 ```
 You can also install `pre-commit` using the following command:
+
 ```bash
 pip install pre-commit
 ```
+
 ### Add `pre-commit` configuration file
+
 Create a file named `.pre-commit-config.yaml` in your Kedro project root directory. You can add entries for the hooks
 you want to run before each `commit`.
-Below is a sample `YAML` file with entries for `ruff` and black`:
+Below is a sample `YAML` file with entries for `ruff`:
+
 ```yaml
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.270
+    rev: '' # Replace with latest version, for example 'v0.1.8'
     hooks:
       - id: ruff
-
-  - repo: https://github.com/psf/black
-    rev: 22.8.0
-    hooks:
-      - id: black
-        language_version: python3.9
+        args: [--fix]
+      - id: ruff-format
 ```
+
+See GitHub for [the latest configuration for ruff's pre-commit](https://github.com/astral-sh/ruff-pre-commit).
+
 ### Install git hook scripts
+
 Run the following command to complete installation:
+
 ```bash
 pre-commit install
 ```
+
 This enables `pre-commit` hooks to run automatically every time you execute `git commit`.
diff --git a/docs/source/get_started/new_project.md b/docs/source/get_started/new_project.md
@@ -31,7 +31,7 @@ Next, the CLI asks which tools you'd like to include in the project:
 
 ```text
 Tools
-1) Lint: Basic linting with Black and Ruff
+1) Lint: Basic linting with ruff
 2) Test: Basic testing with pytest
 3) Log: Additional, environment-specific logging options
 4) Docs: A Sphinx documentation setup
@@ -47,7 +47,6 @@ The options are described in more detail in the [documentation about the new pro
 
 Select the tools by number, or `all` or follow the default to add `none`.
 
-
 ### Project examples
 
 Finally, the CLI offers the option to include starter example code in the project:

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -47,9 +47,9 @@ Welcome to Kedro's award-winning documentation!
     :target: https://linen-slack.kedro.org/
     :alt: Kedro's Slack archive
 
-.. image:: https://img.shields.io/badge/code%20style-black-black.svg
-    :target: https://github.com/psf/black
-    :alt: Code style is Black
+.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
+    :target: https://github.com/astral-sh/ruff
+    :alt: Linted and Formatted with Ruff
 
 .. image:: https://bestpractices.coreinfrastructure.org/projects/6711/badge
     :target: https://bestpractices.coreinfrastructure.org/projects/6711

diff --git a/docs/source/starters/new_project_tools.md b/docs/source/starters/new_project_tools.md
@@ -190,7 +190,7 @@ With these installed, you can then make use of the following commands to format
 
 ```bash
 ruff format path/to/project/root
-black path/to/project/root --check
+ruff check path/to/project/root
 ```
 
 Though it has no impact on how your code works, linting is important for code quality because improves consistency, readability, debugging, and maintainability. To learn more about linting your Kedro projects, check our [linting documentation](../development/linting.md).

diff --git a/docs/source/tutorial/tutorial_template.md b/docs/source/tutorial/tutorial_template.md
@@ -32,10 +32,9 @@ The spaceflights project dependencies are stored in `requirements.txt`(you may f
 
 ```text
 # code quality packages
-black~=22.0
 ipython>=7.31.1, <8.0; python_version < '3.8'
 ipython~=8.10; python_version >= '3.8'
-ruff~=0.0.290
+ruff==0.1.8
 
 # notebook tooling
 jupyter~=1.0

diff --git a/features/steps/sh_run.py b/features/steps/sh_run.py
@@ -36,7 +36,7 @@ def run(
     """
     if isinstance(cmd, str) and split:
         cmd = shlex.split(cmd)
-    result = subprocess.run(cmd, input="", capture_output=True, **kwargs)  # noqa: PLW1510
+    result = subprocess.run(cmd, input="", capture_output=True, **kwargs)  # noqa: PLW1510, S603
     result.stdout = result.stdout.decode("utf-8")
     result.stderr = result.stderr.decode("utf-8")
     if print_output:
@@ -59,9 +59,9 @@ def check_run(cmd: list | str, print_output: bool = False) -> None:
         split_cmd = cmd
 
     if print_output:
-        subprocess.check_call(split_cmd)
+        subprocess.check_call(split_cmd)  # noqa: S603
     else:
-        subprocess.check_call(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        subprocess.check_call(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # noqa: S603
 
 
 class ChildTerminatingPopen(subprocess.Popen):

diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt
@@ -1,4 +1,4 @@
-black~=22.0
+ruff==0.1.8
 ipython>=8.10
 jupyterlab>=3.0
 notebook

diff --git a/kedro/framework/cli/catalog.py b/kedro/framework/cli/catalog.py
@@ -23,7 +23,6 @@ def _create_session(package_name: str, **kwargs: Any) -> KedroSession:
     return KedroSession.create(**kwargs)
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def catalog_cli() -> None:  # pragma: no cover
     pass
@@ -34,7 +33,6 @@ def catalog() -> None:
     """Commands for working with catalog."""
 
 
-# noqa: too-many-locals,protected-access
 @catalog.command("list")
 @env_option
 @click.option(
@@ -173,7 +171,7 @@ def create_catalog(metadata: ProjectMetadata, pipeline_name: str, env: str) -> N
 
     catalog_datasets = {
         ds_name
-        for ds_name in context.catalog._datasets.keys()  # noqa: protected-access
+        for ds_name in context.catalog._datasets.keys()
         if not ds_name.startswith("params:") and ds_name != "parameters"
     }
 

diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py
@@ -29,7 +29,7 @@
     _get_entry_points,
     load_entry_points,
 )
-from kedro.framework.project import LOGGING  # noqa # noqa: unused-import
+from kedro.framework.project import LOGGING  # noqa: F401
 from kedro.framework.startup import _is_project, bootstrap_project
 
 LOGO = rf"""
@@ -149,7 +149,6 @@ def global_groups(self) -> Sequence[click.MultiCommand]:
 
     @property
     def project_groups(self) -> Sequence[click.MultiCommand]:
-        # noqa: line-too-long
         """Property which loads all project command groups from the
         project and the plugins, then combines them with the built-in ones.
         Built-in commands can be overridden by plugins, which can be

diff --git a/kedro/framework/cli/hooks/manager.py b/kedro/framework/cli/hooks/manager.py
@@ -1,5 +1,4 @@
 """This module defines a dedicated hook manager for hooks that extends Kedro CLI behaviour."""
-# noqa: global-statement,invalid-name
 import logging
 
 from pluggy import PluginManager

diff --git a/kedro/framework/cli/jupyter.py b/kedro/framework/cli/jupyter.py
@@ -36,7 +36,6 @@ def list_commands(self, ctx: click.Context) -> list[str]:
         return ["setup", "notebook", "lab", "convert"]
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def jupyter_cli() -> None:  # pragma: no cover
     pass
@@ -156,7 +155,6 @@ def _create_kernel(kernel_name: str, display_name: str) -> str:
     """
     # These packages are required by jupyter lab and notebook, which we have already
     # checked are importable, so we don't run _check_module_importable on them.
-    # noqa: import-outside-toplevel
     from ipykernel.kernelspec import install
 
     try:

diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py
@@ -71,7 +71,6 @@ def _check_pipeline_name(ctx: click.Context, param: Any, value: str) -> str:  #
     return value
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def pipeline_cli() -> None:  # pragma: no cover
     pass
@@ -216,7 +215,6 @@ def _echo_deletion_warning(message: str, **paths: list[Path]) -> None:
 
 
 def _create_pipeline(name: str, template_path: Path, output_dir: Path) -> Path:
-    # noqa: import-outside-toplevel
     from cookiecutter.main import cookiecutter
 
     cookie_context = {"pipeline_name": name, "kedro_version": kedro.__version__}

diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py
@@ -61,7 +61,6 @@
 CONF_SOURCE_HELP = """Path of a directory where project configuration is stored."""
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def project_group() -> None:  # pragma: no cover
     pass
@@ -197,7 +196,7 @@ def package(metadata: ProjectMetadata) -> None:
     help=PARAMS_ARG_HELP,
     callback=_split_params,
 )
-def run(  # noqa: PLR0913,unused-argument,too-many-locals
+def run(  # noqa: PLR0913
     tags: str,
     env: str,
     runner: str,

diff --git a/kedro/framework/cli/registry.py b/kedro/framework/cli/registry.py
@@ -9,7 +9,6 @@
 from kedro.framework.startup import ProjectMetadata
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def registry_cli() -> None:  # pragma: no cover
     pass
@@ -31,7 +30,7 @@ def list_registered_pipelines() -> None:
 @click.pass_obj
 def describe_registered_pipeline(
     metadata: ProjectMetadata, /, name: str, **kwargs: Any
-) -> None:  # noqa: unused-argument, protected-access
+) -> None:
     """Describe a registered pipeline by providing a pipeline name.
     Defaults to the `__default__` pipeline.
     """