diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index 89fdd674..f4ef1853 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -43,10 +43,17 @@ jobs: - os: windows-latest python-version: "3.10" steps: - - uses: neuroinformatics-unit/actions/test@v2 + - name: Run tests + uses: neuroinformatics-unit/actions/test@v2 with: python-version: ${{ matrix.python-version }} + - name: Check benchmarks + run: | + pip install --upgrade pip + pip install asv + asv check -v -E existing + build_sdist_wheels: name: Build source distribution needs: [test] diff --git a/asv.conf.json b/asv.conf.json index f06e89b3..b5aa650c 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -4,7 +4,7 @@ "version": 1, // The name of the project being benchmarked - "project": "brainglobe_workflows", + "project": "brainglobe-workflows", // The project's homepage "project_url": "https://github.com/brainglobe/brainglobe-workflows", @@ -12,7 +12,7 @@ // The URL or local path of the source code repository for the // project being benchmarked // "repo": ".", - "repo": "https://github.com/brainglobe/brainglobe-workflows", + "repo": "https://github.com/brainglobe/brainglobe-workflows.git", // The Python project's subdirectory in your repo. If missing or // the empty string, the project is assumed to be located at the root @@ -40,14 +40,14 @@ // List of branches to benchmark. If not provided, defaults to "master" // (for git) or "default" (for mercurial). - "branches": ["smg/tests-refactor"], // for git + "branches": ["smg/asv-check-in-ci"], // for git // "branches": ["default"], // for mercurial // The DVCS being used. If not set, it will be automatically // determined from "repo" by looking at the protocol in the URL // (if remote), or by looking for special directories, such as // ".git" (if local). - "dvcs": "git", + // "dvcs": "git", // The tool to use to create environments. May be "conda", // "virtualenv", "mamba" (above 3.8) @@ -147,7 +147,7 @@ // The directory (relative to the current directory) that benchmarks are // stored in. If not provided, defaults to "benchmarks" - "benchmark_dir": "brainglobe_benchmarks", + "benchmark_dir": "benchmarks", // The directory (relative to the current directory) to cache the Python // environments in. If not provided, defaults to "env" @@ -155,11 +155,11 @@ // The directory (relative to the current directory) that raw benchmark // results are stored in. If not provided, defaults to "results". - "results_dir": "brainglobe_benchmarks/results", + "results_dir": "benchmarks/results", // The directory (relative to the current directory) that the html tree // should be written to. If not provided, defaults to "html". - "html_dir": "brainglobe_benchmarks/html", + "html_dir": "benchmarks/html", // The number of characters to retain in the commit hashes. // "hash_length": 8, diff --git a/benchmarks/cellfinder_core.py b/benchmarks/cellfinder_core.py index 41821d94..8f94824b 100644 --- a/benchmarks/cellfinder_core.py +++ b/benchmarks/cellfinder_core.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -import pooch from brainglobe_utils.IO.cells import save_cells from cellfinder.core.main import main as cellfinder_run from cellfinder.core.tools.IO import read_with_dask @@ -81,18 +80,18 @@ class TimeBenchmarkPrepGIN: # Custom attributes input_config_path = str(DEFAULT_JSON_CONFIG_PATH_CELLFINDER) - def setup_cache( - self, - ): + def setup_cache(self): """ Download the input data from the GIN repository to the local - directory specified in the default_config.json + directory specified in the default_config.json. Notes ----- The `setup_cache` method only performs the computations once per benchmark round and then caches the result to disk [1]_. It cannot - be parametrised [2]_. + be parametrised [2]_. Therefore, if we sweep across different input + JSON files, we need to ensure all data for all configs is made + available with this setup function. [1] https://asv.readthedocs.io/en/latest/writing_benchmarks.html#setup-and-teardown-functions @@ -108,15 +107,6 @@ def setup_cache( config_dict = json.load(cfg) config = CellfinderConfig(**config_dict) - # Download data with pooch - _ = pooch.retrieve( - url=config.data_url, - known_hash=config.data_hash, - path=config._install_path, - progressbar=True, - processor=pooch.Unzip(extract_dir=config.data_dir_relative), - ) - # Check paths to input data should now exist in config assert Path(config._signal_dir_path).exists() assert Path(config._background_dir_path).exists() @@ -129,12 +119,7 @@ def setup(self): """ # Run setup - cfg = setup_cellfinder_workflow( - [ - "--config", - self.input_config_path, - ] - ) + cfg = setup_cellfinder_workflow(self.input_config_path) # Save configuration as attribute self.cfg = cfg @@ -162,7 +147,7 @@ class TimeFullWorkflow(TimeBenchmarkPrepGIN): A base class for timing benchmarks for the cellfinder workflow. """ - def time_workflow_from_cellfinder_run(self): + def time_workflow(self): run_workflow_from_cellfinder_run(self.cfg) @@ -177,10 +162,10 @@ class TimeReadInputDask(TimeBenchmarkPrepGIN): """ def time_read_signal_with_dask(self): - read_with_dask(self.cfg._signal_dir_path) + read_with_dask(str(self.cfg._signal_dir_path)) def time_read_background_with_dask(self): - read_with_dask(self.cfg._background_dir_path) + read_with_dask(str(self.cfg._background_dir_path)) class TimeDetectCells(TimeBenchmarkPrepGIN): @@ -198,13 +183,37 @@ def setup(self): # basic setup TimeBenchmarkPrepGIN.setup(self) - # add input data as arrays to config - self.signal_array = read_with_dask(self.cfg._signal_dir_path) - self.background_array = read_with_dask(self.cfg._background_dir_path) + # add input data as arrays to the config + self.signal_array = read_with_dask(str(self.cfg._signal_dir_path)) + self.background_array = read_with_dask( + str(self.cfg._background_dir_path) + ) def time_cellfinder_run(self): cellfinder_run( - self.signal_array, self.background_array, self.cfg.voxel_sizes + self.signal_array, + self.background_array, + self.cfg.voxel_sizes, + self.cfg.start_plane, + self.cfg.end_plane, + self.cfg.trained_model, + self.cfg.model_weights, + self.cfg.model, + self.cfg.batch_size, + self.cfg.n_free_cpus, + self.cfg.network_voxel_sizes, + self.cfg.soma_diameter, + self.cfg.ball_xy_size, + self.cfg.ball_z_size, + self.cfg.ball_overlap_fraction, + self.cfg.log_sigma_size, + self.cfg.n_sds_above_mean_thresh, + self.cfg.soma_spread_factor, + self.cfg.max_cluster_size, + self.cfg.cube_width, + self.cfg.cube_height, + self.cfg.cube_depth, + self.cfg.network_depth, ) @@ -215,12 +224,36 @@ def setup(self): TimeBenchmarkPrepGIN.setup(self) # add input data as arrays to config - self.signal_array = read_with_dask(self.cfg._signal_dir_path) - self.background_array = read_with_dask(self.cfg._background_dir_path) + self.signal_array = read_with_dask(str(self.cfg._signal_dir_path)) + self.background_array = read_with_dask( + str(self.cfg._background_dir_path) + ) # detect cells self.detected_cells = cellfinder_run( - self.signal_array, self.background_array, self.cfg.voxel_sizes + self.signal_array, + self.background_array, + self.cfg.voxel_sizes, + self.cfg.start_plane, + self.cfg.end_plane, + self.cfg.trained_model, + self.cfg.model_weights, + self.cfg.model, + self.cfg.batch_size, + self.cfg.n_free_cpus, + self.cfg.network_voxel_sizes, + self.cfg.soma_diameter, + self.cfg.ball_xy_size, + self.cfg.ball_z_size, + self.cfg.ball_overlap_fraction, + self.cfg.log_sigma_size, + self.cfg.n_sds_above_mean_thresh, + self.cfg.soma_spread_factor, + self.cfg.max_cluster_size, + self.cfg.cube_width, + self.cfg.cube_height, + self.cfg.cube_depth, + self.cfg.network_depth, ) def time_save_cells(self): diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/benchmarks/test_cellfinder.py b/tests/benchmarks/test_cellfinder.py deleted file mode 100644 index c323aed0..00000000 --- a/tests/benchmarks/test_cellfinder.py +++ /dev/null @@ -1,100 +0,0 @@ -import json -import subprocess -from pathlib import Path - -import pytest -from asv import util - - -@pytest.fixture() -def asv_config_monkeypatched_path(tmp_path: Path) -> str: - """ - Create a monkeypatched asv.conf.json file - in a Pytest-generated temporary directory - and return its path - - Parameters - ---------- - tmp_path : Path - path to pytest-generated temporary directory - - Returns - ------- - str - Path to monkeypatched asv config file - """ - # read reference asv config - asv_original_path = Path(__file__).resolve().parents[3] / "asv.conf.json" - asv_monkeypatched_dict = util.load_json( - asv_original_path, js_comments=True - ) - - # change directories - for ky in ["env_dir", "results_dir", "html_dir"]: - asv_monkeypatched_dict[ky] = str( - Path(tmp_path) / asv_monkeypatched_dict[ky] - ) - - # change repo to URL rather than local - asv_monkeypatched_dict[ - "repo" - ] = "https://github.com/brainglobe/brainglobe-workflows.git" - - # define path to a temp json file to dump config data - asv_monkeypatched_path = tmp_path / "asv.conf.json" - - # save monkeypatched config data to json file - with open(asv_monkeypatched_path, "w") as js: - json.dump(asv_monkeypatched_dict, js) - - # check json file exists - assert asv_monkeypatched_path.is_file() - - return str(asv_monkeypatched_path) - - -@pytest.mark.skip(reason="focus of PR32") -def test_run_benchmarks(asv_config_monkeypatched_path): - # --- ideally monkeypatch an asv config so that results are in tmp_dir? - - # set up machine (env_dir, results_dir, html_dir) - asv_machine_output = subprocess.run( - [ - "asv", - "machine", - "--yes", - "--config", - asv_config_monkeypatched_path, - ] - ) - assert asv_machine_output.returncode == 0 - - # run benchmarks - asv_benchmark_output = subprocess.run( - [ - "asv", - "run", - "--config", - asv_config_monkeypatched_path, - # "--dry-run" - # # Do not save any results to disk? not truly testing then - ], - cwd=str( - Path(asv_config_monkeypatched_path).parent - ), # run from where asv config is - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - encoding="utf-8", - ) - # STDOUT: "· Cloning project\n· Fetching recent changes\n· - # Creating environments\n· No __init__.py file in 'benchmarks'\n" - - # check returncode - assert asv_benchmark_output.returncode == 0 - - # check logs? - - # delete directories? - # check teardown after yield: - # https://docs.pytest.org/en/6.2.x/fixture.html#yield-fixtures-recommended