diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8401336..6595cbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,7 +93,6 @@ repos: types-setuptools ] pass_filenames: false - language_version: "3.9" - repo: https://github.com/mgedmin/check-manifest rev: "0.49" hooks: diff --git a/CHANGES.md b/CHANGES.md index 1591ce8..4c4d790 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel). +## 0.3.0 - 2023-xx-xx + +- {pull}`50` deprecates INI configurations and aligns the package with pytask v0.3. + ## 0.2.1 - 2022-08-19 - {pull}`43` adds docformatter. diff --git a/LICENSE b/LICENSE index 4c96cf3..f4c44ec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020-2021 Tobias Raabe +Copyright 2020 Tobias Raabe Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software diff --git a/MANIFEST.in b/MANIFEST.in index de254c6..1eb48e3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,5 @@ exclude tox.ini include README.md include LICENSE + +recursive-include src py.typed diff --git a/README.md b/README.md index 78159d5..298052b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![image](https://img.shields.io/conda/vn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel) [![image](https://img.shields.io/conda/pn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel) [![PyPI - License](https://img.shields.io/pypi/l/pytask-parallel)](https://pypi.org/project/pytask-parallel) -[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-parallel/Continuous%20Integration%20Workflow/main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain) +[![image](https://img.shields.io/github/actions/workflow/status/pytask-dev/pytask-parallel/main.yml?branch=main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain) [![image](https://codecov.io/gh/pytask-dev/pytask-parallel/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-parallel) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-parallel/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-parallel/main) [![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/environment.yml b/environment.yml index e6ab11d..dd1290c 100644 --- a/environment.yml +++ b/environment.yml @@ -16,7 +16,7 @@ dependencies: - conda-verify # Package dependencies - - pytask >=0.2.0 + - pytask >=0.3 - cloudpickle - loky - pybaum >=0.1.1 diff --git a/setup.cfg b/setup.cfg index a6484da..1152a9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ install_requires = cloudpickle loky pybaum>=0.1.1 - pytask>=0.2 + pytask>=0.3 python_requires = >=3.7 include_package_data = True package_dir = =src diff --git a/src/pytask_parallel/backends.py b/src/pytask_parallel/backends.py index 8828bef..4397224 100644 --- a/src/pytask_parallel/backends.py +++ b/src/pytask_parallel/backends.py @@ -3,19 +3,42 @@ from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ThreadPoolExecutor +from enum import Enum -PARALLEL_BACKENDS = { - "processes": ProcessPoolExecutor, - "threads": ThreadPoolExecutor, -} - -PARALLEL_BACKENDS_DEFAULT = "processes" - try: from loky import get_reusable_executor + except ImportError: - pass + + class ParallelBackendChoices(str, Enum): + PROCESSES = "processes" + THREADS = "threads" + + PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES + + PARALLEL_BACKENDS = { + ParallelBackendChoices.PROCESSES: ProcessPoolExecutor, + ParallelBackendChoices.THREADS: ThreadPoolExecutor, + } + else: - PARALLEL_BACKENDS["loky"] = get_reusable_executor - PARALLEL_BACKENDS_DEFAULT = "loky" + + class ParallelBackendChoices(str, Enum): # type: ignore[no-redef] + PROCESSES = "processes" + THREADS = "threads" + LOKY = "loky" + + PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES + + PARALLEL_BACKENDS = { + ParallelBackendChoices.PROCESSES: ProcessPoolExecutor, + ParallelBackendChoices.THREADS: ThreadPoolExecutor, + ParallelBackendChoices.LOKY: ( # type: ignore[attr-defined] + get_reusable_executor + ), + } + + PARALLEL_BACKENDS_DEFAULT = ( + ParallelBackendChoices.LOKY # type: ignore[attr-defined] + ) diff --git a/src/pytask_parallel/build.py b/src/pytask_parallel/build.py index d16a674..9bf403a 100644 --- a/src/pytask_parallel/build.py +++ b/src/pytask_parallel/build.py @@ -2,9 +2,10 @@ from __future__ import annotations import click +from pytask import EnumChoice from pytask import hookimpl -from pytask_parallel.backends import PARALLEL_BACKENDS from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT +from pytask_parallel.backends import ParallelBackendChoices @hookimpl @@ -15,19 +16,16 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ["-n", "--n-workers"], help=( "Max. number of pytask_parallel tasks. Integer >= 1 or 'auto' which is " - "os.cpu_count() - 1. [default: 1 (no parallelization)]" + "os.cpu_count() - 1." ), metavar="[INTEGER|auto]", - default=None, + default=1, ), click.Option( ["--parallel-backend"], - type=click.Choice(PARALLEL_BACKENDS), - help=( - "Backend for the parallelization. " - f"[default: {PARALLEL_BACKENDS_DEFAULT}]" - ), - default=None, + type=EnumChoice(ParallelBackendChoices), + help="Backend for the parallelization.", + default=PARALLEL_BACKENDS_DEFAULT, ), ] cli.commands["build"].params.extend(additional_parameters) diff --git a/src/pytask_parallel/config.py b/src/pytask_parallel/config.py index a329900..6c4af23 100644 --- a/src/pytask_parallel/config.py +++ b/src/pytask_parallel/config.py @@ -1,42 +1,34 @@ """Configure pytask.""" from __future__ import annotations +import enum import os from typing import Any -from typing import Callable from pytask import hookimpl -from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT -from pytask_parallel.callbacks import n_workers_callback -from pytask_parallel.callbacks import parallel_backend_callback +from pytask_parallel.backends import ParallelBackendChoices @hookimpl -def pytask_parse_config( - config: dict[str, Any], - config_from_cli: dict[str, Any], - config_from_file: dict[str, Any], -) -> None: +def pytask_parse_config(config: dict[str, Any]) -> None: """Parse the configuration.""" - config["n_workers"] = _get_first_non_none_value( - config_from_cli, - config_from_file, - key="n_workers", - default=1, - callback=n_workers_callback, - ) if config["n_workers"] == "auto": config["n_workers"] = max(os.cpu_count() - 1, 1) - config["delay"] = 0.1 + if ( + isinstance(config["parallel_backend"], str) + and config["parallel_backend"] in ParallelBackendChoices._value2member_map_ + ): + config["parallel_backend"] = ParallelBackendChoices(config["parallel_backend"]) + elif ( + isinstance(config["parallel_backend"], enum.Enum) + and config["parallel_backend"] in ParallelBackendChoices + ): + pass + else: + raise ValueError("Invalid value for 'parallel_backend'.") - config["parallel_backend"] = _get_first_non_none_value( - config_from_cli, - config_from_file, - key="parallel_backend", - default=PARALLEL_BACKENDS_DEFAULT, - callback=parallel_backend_callback, - ) + config["delay"] = 0.1 @hookimpl @@ -44,20 +36,3 @@ def pytask_post_parse(config: dict[str, Any]) -> None: """Disable parallelization if debugging is enabled.""" if config["pdb"] or config["trace"]: config["n_workers"] = 1 - - -def _get_first_non_none_value( - *configs: dict[str, Any], - key: str, - default: Any | None = None, - callback: Callable[..., Any] | None = None, -) -> Any: - """Get the first non-None value for a key from a list of dictionaries. - - This function allows to prioritize information from many configurations by changing - the order of the inputs while also providing a default. - - """ - callback = (lambda x: x) if callback is None else callback # noqa: E731 - processed_values = (callback(config.get(key)) for config in configs) - return next((value for value in processed_values if value is not None), default) diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index a614cc8..65d422f 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -19,31 +19,25 @@ from pytask import get_marks from pytask import hookimpl from pytask import Mark +from pytask import parse_warning_filter from pytask import remove_internal_traceback_frames_from_exc_info from pytask import Session from pytask import Task +from pytask import warning_record_to_str +from pytask import WarningReport from pytask_parallel.backends import PARALLEL_BACKENDS +from pytask_parallel.backends import ParallelBackendChoices from rich.console import ConsoleOptions from rich.traceback import Traceback -# Can be removed if pinned to pytask >= 0.2.6. -try: - from pytask import parse_warning_filter - from pytask import warning_record_to_str - from pytask import WarningReport -except ImportError: - from _pytask.warnings import parse_warning_filter - from _pytask.warnings import warning_record_to_str - from _pytask.warnings_utils import WarningReport - @hookimpl def pytask_post_parse(config: dict[str, Any]) -> None: """Register the parallel backend.""" - if config["parallel_backend"] in ("loky", "processes"): - config["pm"].register(ProcessesNameSpace) - elif config["parallel_backend"] in ("threads",): + if config["parallel_backend"] == ParallelBackendChoices.THREADS: config["pm"].register(DefaultBackendNameSpace) + else: + config["pm"].register(ProcessesNameSpace) @hookimpl(tryfirst=True) @@ -66,7 +60,7 @@ def pytask_execute_build(session: Session) -> bool | None: with parallel_backend(max_workers=session.config["n_workers"]) as executor: - session.executor = executor + session.config["_parallel_executor"] = executor sleeper = _Sleeper() while session.scheduler.is_active(): @@ -189,7 +183,7 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None: bytes_function = cloudpickle.dumps(task) bytes_kwargs = cloudpickle.dumps(kwargs) - return session.executor.submit( + return session.config["_parallel_executor"].submit( _unserialize_and_execute_task, bytes_function=bytes_function, bytes_kwargs=bytes_kwargs, @@ -285,7 +279,7 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None: """ if session.config["n_workers"] > 1: kwargs = _create_kwargs_for_task(task) - return session.executor.submit( + return session.config["_parallel_executor"].submit( _mock_processes_for_threads, func=task.execute, **kwargs ) else: diff --git a/src/pytask_parallel/py.typed b/src/pytask_parallel/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_config.py b/tests/test_config.py index b399030..95929a8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,7 @@ import pytest from pytask import ExitCode from pytask import main -from pytask_parallel.backends import PARALLEL_BACKENDS +from pytask_parallel.backends import ParallelBackendChoices @pytest.mark.end_to_end @@ -26,9 +26,6 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe @pytest.mark.end_to_end -@pytest.mark.parametrize( - "config_file", ["pytask.ini", "tox.ini", "setup.cfg", "pyproject.toml"] -) @pytest.mark.parametrize( "configuration_option, value, exit_code", [ @@ -38,31 +35,20 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe ("parallel_backend", "unknown_backend", ExitCode.CONFIGURATION_FAILED), ] + [ - ("parallel_backend", parallel_backend, ExitCode.OK) - for parallel_backend in PARALLEL_BACKENDS + ("parallel_backend", parallel_backend.value, ExitCode.OK) + for parallel_backend in ParallelBackendChoices ], ) def test_reading_values_from_config_file( - tmp_path, capsys, config_file, configuration_option, value, exit_code + tmp_path, configuration_option, value, exit_code ): - if config_file == "pyproject.toml": - config = f""" - [tool.pytask.ini_options] - {configuration_option} = {value!r} - """ - else: - config = f""" - [pytask] - {configuration_option} = {value} - """ - tmp_path.joinpath(config_file).write_text(textwrap.dedent(config)) + config = f""" + [tool.pytask.ini_options] + {configuration_option} = {value!r} + """ + tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(config)) session = main({"paths": tmp_path}) - captured = capsys.readouterr() - if config_file == "pyproject.toml": - assert "WARNING" not in captured.out - else: - assert "WARNING" in captured.out assert session.exit_code == exit_code if value == "auto": diff --git a/tests/test_execute.py b/tests/test_execute.py index 958e6e4..1af4048 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -126,7 +126,7 @@ def myfunc(): with PARALLEL_BACKENDS[parallel_backend]( max_workers=session.config["n_workers"] ) as executor: - session.executor = executor + session.config["_parallel_executor"] = executor backend_name_space = { "processes": ProcessesNameSpace, diff --git a/tox.ini b/tox.ini index d0e8914..8ac41cc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,14 @@ [tox] envlist = pytest -skipsdist = True -skip_missing_interpreters = True [testenv] -basepython = python +usedevelop = true [testenv:pytest] conda_deps = cloudpickle loky - pytask >=0.1.0 + pytask >=0.3.0 pytest pytest-cov pytest-xdist @@ -21,10 +19,6 @@ commands = pip install --no-deps -e . pytest {posargs} -[doc8] -ignore = D002, D004 -max-line-length = 89 - [flake8] docstring-convention = numpy ignore =