Check asv benchmarks in CI

.github/workflows/test_and_deploy.yml

@@ -43,10 +43,17 @@ jobs:
         - os: windows-latest
           python-version: "3.10"
     steps:
-      - uses: neuroinformatics-unit/actions/test@v2
+      - name: Run tests
+        uses: neuroinformatics-unit/actions/test@v2
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Check benchmarks
+        run: |
+            pip install --upgrade pip
+            pip install asv
+            asv check -v -E existing
+
   build_sdist_wheels:
     name: Build source distribution
     needs: [test]

asv.conf.json

@@ -4,15 +4,15 @@
     "version": 1,
 
     // The name of the project being benchmarked
-    "project": "brainglobe_workflows",
+    "project": "brainglobe-workflows",
 
     // The project's homepage
     "project_url": "https://github.com/brainglobe/brainglobe-workflows",
 
     // The URL or local path of the source code repository for the
     // project being benchmarked
     // "repo": ".",
-    "repo": "https://github.com/brainglobe/brainglobe-workflows",
+    "repo": "https://github.com/brainglobe/brainglobe-workflows.git",
 
     // The Python project's subdirectory in your repo.  If missing or
     // the empty string, the project is assumed to be located at the root
@@ -40,14 +40,14 @@
 
     // List of branches to benchmark. If not provided, defaults to "master"
     // (for git) or "default" (for mercurial).
-    "branches": ["smg/tests-refactor"], // for git
+    "branches": ["smg/asv-check-in-ci"], // for git
     // "branches": ["default"],    // for mercurial
 
     // The DVCS being used.  If not set, it will be automatically
     // determined from "repo" by looking at the protocol in the URL
     // (if remote), or by looking for special directories, such as
     // ".git" (if local).
-    "dvcs": "git",
+    // "dvcs": "git",
 
     // The tool to use to create environments.  May be "conda",
     // "virtualenv", "mamba" (above 3.8)
@@ -147,19 +147,19 @@
 
     // The directory (relative to the current directory) that benchmarks are
     // stored in.  If not provided, defaults to "benchmarks"
-    "benchmark_dir": "brainglobe_benchmarks",
+    "benchmark_dir": "benchmarks",
 
     // The directory (relative to the current directory) to cache the Python
     // environments in.  If not provided, defaults to "env"
     "env_dir": ".asv/env",
 
     // The directory (relative to the current directory) that raw benchmark
     // results are stored in.  If not provided, defaults to "results".
-    "results_dir": "brainglobe_benchmarks/results",
+    "results_dir": "benchmarks/results",
 
     // The directory (relative to the current directory) that the html tree
     // should be written to.  If not provided, defaults to "html".
-    "html_dir": "brainglobe_benchmarks/html",
+    "html_dir": "benchmarks/html",
 
     // The number of characters to retain in the commit hashes.
     // "hash_length": 8,

benchmarks/cellfinder_core.py

@@ -2,7 +2,6 @@
 import shutil
 from pathlib import Path
 
-import pooch
 from brainglobe_utils.IO.cells import save_cells
 from cellfinder.core.main import main as cellfinder_run
 from cellfinder.core.tools.IO import read_with_dask
@@ -81,18 +80,18 @@ class TimeBenchmarkPrepGIN:
     # Custom attributes
     input_config_path = str(DEFAULT_JSON_CONFIG_PATH_CELLFINDER)
 
-    def setup_cache(
-        self,
-    ):
+    def setup_cache(self):
         """
         Download the input data from the GIN repository to the local
-        directory specified in the default_config.json
+        directory specified in the default_config.json.
 
         Notes
         -----
         The `setup_cache` method only performs the computations once
         per benchmark round and then caches the result to disk [1]_. It cannot
-        be parametrised [2]_.
+        be parametrised [2]_. Therefore, if we sweep across different input
+        JSON files, we need to ensure all data for all configs is made
+        available with this setup function.
 
 
         [1] https://asv.readthedocs.io/en/latest/writing_benchmarks.html#setup-and-teardown-functions
@@ -108,15 +107,6 @@ def setup_cache(
             config_dict = json.load(cfg)
         config = CellfinderConfig(**config_dict)
 
-        # Download data with pooch
-        _ = pooch.retrieve(
-            url=config.data_url,
-            known_hash=config.data_hash,
-            path=config._install_path,
-            progressbar=True,
-            processor=pooch.Unzip(extract_dir=config.data_dir_relative),
-        )
-
         # Check paths to input data should now exist in config
         assert Path(config._signal_dir_path).exists()
         assert Path(config._background_dir_path).exists()
@@ -129,12 +119,7 @@ def setup(self):
         """
 
         # Run setup
-        cfg = setup_cellfinder_workflow(
-            [
-                "--config",
-                self.input_config_path,
-            ]
-        )
+        cfg = setup_cellfinder_workflow(self.input_config_path)
 
         # Save configuration as attribute
         self.cfg = cfg
@@ -162,7 +147,7 @@ class TimeFullWorkflow(TimeBenchmarkPrepGIN):
         A base class for timing benchmarks for the cellfinder workflow.
     """
 
-    def time_workflow_from_cellfinder_run(self):
+    def time_workflow(self):
         run_workflow_from_cellfinder_run(self.cfg)
 
 
@@ -177,10 +162,10 @@ class TimeReadInputDask(TimeBenchmarkPrepGIN):
     """
 
     def time_read_signal_with_dask(self):
-        read_with_dask(self.cfg._signal_dir_path)
+        read_with_dask(str(self.cfg._signal_dir_path))
 
     def time_read_background_with_dask(self):
-        read_with_dask(self.cfg._background_dir_path)
+        read_with_dask(str(self.cfg._background_dir_path))
 
 
 class TimeDetectCells(TimeBenchmarkPrepGIN):
@@ -198,13 +183,37 @@ def setup(self):
         # basic setup
         TimeBenchmarkPrepGIN.setup(self)
 
-        # add input data as arrays to config
-        self.signal_array = read_with_dask(self.cfg._signal_dir_path)
-        self.background_array = read_with_dask(self.cfg._background_dir_path)
+        # add input data as arrays to the config
+        self.signal_array = read_with_dask(str(self.cfg._signal_dir_path))
+        self.background_array = read_with_dask(
+            str(self.cfg._background_dir_path)
+        )
 
     def time_cellfinder_run(self):
         cellfinder_run(
-            self.signal_array, self.background_array, self.cfg.voxel_sizes
+            self.signal_array,
+            self.background_array,
+            self.cfg.voxel_sizes,
+            self.cfg.start_plane,
+            self.cfg.end_plane,
+            self.cfg.trained_model,
+            self.cfg.model_weights,
+            self.cfg.model,
+            self.cfg.batch_size,
+            self.cfg.n_free_cpus,
+            self.cfg.network_voxel_sizes,
+            self.cfg.soma_diameter,
+            self.cfg.ball_xy_size,
+            self.cfg.ball_z_size,
+            self.cfg.ball_overlap_fraction,
+            self.cfg.log_sigma_size,
+            self.cfg.n_sds_above_mean_thresh,
+            self.cfg.soma_spread_factor,
+            self.cfg.max_cluster_size,
+            self.cfg.cube_width,
+            self.cfg.cube_height,
+            self.cfg.cube_depth,
+            self.cfg.network_depth,
         )
 
 
@@ -215,12 +224,36 @@ def setup(self):
         TimeBenchmarkPrepGIN.setup(self)
 
         # add input data as arrays to config
-        self.signal_array = read_with_dask(self.cfg._signal_dir_path)
-        self.background_array = read_with_dask(self.cfg._background_dir_path)
+        self.signal_array = read_with_dask(str(self.cfg._signal_dir_path))
+        self.background_array = read_with_dask(
+            str(self.cfg._background_dir_path)
+        )
 
         # detect cells
         self.detected_cells = cellfinder_run(
-            self.signal_array, self.background_array, self.cfg.voxel_sizes
+            self.signal_array,
+            self.background_array,
+            self.cfg.voxel_sizes,
+            self.cfg.start_plane,
+            self.cfg.end_plane,
+            self.cfg.trained_model,
+            self.cfg.model_weights,
+            self.cfg.model,
+            self.cfg.batch_size,
+            self.cfg.n_free_cpus,
+            self.cfg.network_voxel_sizes,
+            self.cfg.soma_diameter,
+            self.cfg.ball_xy_size,
+            self.cfg.ball_z_size,
+            self.cfg.ball_overlap_fraction,
+            self.cfg.log_sigma_size,
+            self.cfg.n_sds_above_mean_thresh,
+            self.cfg.soma_spread_factor,
+            self.cfg.max_cluster_size,
+            self.cfg.cube_width,
+            self.cfg.cube_height,
+            self.cfg.cube_depth,
+            self.cfg.network_depth,
         )
 
     def time_save_cells(self):