diff --git a/benchmarks/README.md b/benchmarks/README.md index 6ea53c3ae8..8dffd473f3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -30,7 +30,9 @@ used to generate benchmark test objects/files; see but will defer to any value already set in the shell. * `BENCHMARK_DATA` - optional - path to a directory for benchmark synthetic test data, which the benchmark scripts will create if it doesn't already -exist. Defaults to `/benchmarks/.data/` if not set. +exist. Defaults to `/benchmarks/.data/` if not set. Note that some of +the generated files, especially in the 'SPerf' suite, are many GB in size so +plan accordingly. * `ON_DEMAND_BENCHMARKS` - optional - when set (to any value): benchmarks decorated with `@on_demand_benchmark` are included in the ASV run. Usually coupled with the ASV `--bench` argument to only run the benchmark(s) of diff --git a/benchmarks/benchmarks/generate_data/__init__.py b/benchmarks/benchmarks/generate_data/__init__.py index 8874a2c214..78b971d9de 100644 --- a/benchmarks/benchmarks/generate_data/__init__.py +++ b/benchmarks/benchmarks/generate_data/__init__.py @@ -22,6 +22,7 @@ from pathlib import Path from subprocess import CalledProcessError, check_output, run from textwrap import dedent +from warnings import warn from iris._lazy_data import as_concrete_data from iris.fileformats import netcdf @@ -47,6 +48,11 @@ BENCHMARK_DATA = Path(environ.get("BENCHMARK_DATA", default_data_dir)) if BENCHMARK_DATA == default_data_dir: BENCHMARK_DATA.mkdir(exist_ok=True) + message = ( + f"No BENCHMARK_DATA env var, defaulting to {BENCHMARK_DATA}. " + "Note that some benchmark files are GB in size." + ) + warn(message) elif not BENCHMARK_DATA.is_dir(): message = f"Not a directory: {BENCHMARK_DATA} ." raise ValueError(message) diff --git a/benchmarks/benchmarks/generate_data/stock.py b/benchmarks/benchmarks/generate_data/stock.py index bbc7dc0a63..eaf46bb405 100644 --- a/benchmarks/benchmarks/generate_data/stock.py +++ b/benchmarks/benchmarks/generate_data/stock.py @@ -9,6 +9,8 @@ See :mod:`benchmarks.generate_data` for an explanation of this structure. """ +from hashlib import sha256 +import json from pathlib import Path from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh @@ -16,6 +18,14 @@ from . import BENCHMARK_DATA, REUSE_DATA, load_realised, run_function_elsewhere +def hash_args(*args, **kwargs): + """Convert arguments into a short hash - for preserving args in filenames.""" + arg_string = str(args) + kwarg_string = json.dumps(kwargs) + full_string = arg_string + kwarg_string + return sha256(full_string.encode()).hexdigest()[:10] + + def _create_file__xios_common(func_name, **kwargs): def _external(func_name_, temp_file_dir, **kwargs_): from iris.tests.stock import netcdf @@ -23,7 +33,7 @@ def _external(func_name_, temp_file_dir, **kwargs_): func = getattr(netcdf, func_name_) print(func(temp_file_dir, **kwargs_), end="") - args_hash = hash(str(kwargs)) + args_hash = hash_args(**kwargs) save_path = (BENCHMARK_DATA / f"{func_name}_{args_hash}").with_suffix( ".nc" ) @@ -95,7 +105,7 @@ def _external(*args, **kwargs): save_mesh(new_mesh, save_path_) arg_list = [n_nodes, n_faces, n_edges] - args_hash = hash(str(arg_list)) + args_hash = hash_args(*arg_list) save_path = (BENCHMARK_DATA / f"sample_mesh_{args_hash}").with_suffix( ".nc" ) @@ -139,7 +149,7 @@ def _external(sample_mesh_kwargs_, save_path_): new_meshcoord = sample_meshcoord(mesh=input_mesh) save_mesh(new_meshcoord.mesh, save_path_) - args_hash = hash(str(sample_mesh_kwargs)) + args_hash = hash_args(**sample_mesh_kwargs) save_path = ( BENCHMARK_DATA / f"sample_mesh_coord_{args_hash}" ).with_suffix(".nc") diff --git a/benchmarks/benchmarks/sperf/__init__.py b/benchmarks/benchmarks/sperf/__init__.py index 696c8ef4df..eccad56f6f 100644 --- a/benchmarks/benchmarks/sperf/__init__.py +++ b/benchmarks/benchmarks/sperf/__init__.py @@ -20,10 +20,14 @@ class FileMixin: """For use in any benchmark classes that work on a file.""" + # Allows time for large file generation. + timeout = 3600.0 + # Largest file with these params: ~90GB. + # Total disk space: ~410GB. params = [ [12, 384, 640, 960, 1280, 1668], [1, 36, 72], - [1, 3, 36, 72], + [1, 3, 10], ] param_names = ["cubesphere_C", "N levels", "N time steps"] # cubesphere_C: notation refers to faces per panel. diff --git a/benchmarks/benchmarks/sperf/combine_regions.py b/benchmarks/benchmarks/sperf/combine_regions.py index fd2c95c7fc..e68382c06e 100644 --- a/benchmarks/benchmarks/sperf/combine_regions.py +++ b/benchmarks/benchmarks/sperf/combine_regions.py @@ -16,20 +16,21 @@ from iris.experimental.ugrid.utils import recombine_submeshes from .. import TrackAddedMemoryAllocation, on_demand_benchmark -from ..generate_data.ugrid import make_cube_like_2d_cubesphere +from ..generate_data.ugrid import BENCHMARK_DATA, make_cube_like_2d_cubesphere class Mixin: # Characterise time taken + memory-allocated, for various stages of combine # operations on cubesphere-like test data. - timeout = 180.0 + timeout = 300.0 params = [100, 200, 300, 500, 1000, 1668] param_names = ["cubesphere_C"] # Fix result units for the tracking benchmarks. unit = "Mb" + temp_save_path = BENCHMARK_DATA / "tmp.nc" def _parametrised_cache_filename(self, n_cubesphere, content_name): - return f"cube_C{n_cubesphere}_{content_name}.nc" + return BENCHMARK_DATA / f"cube_C{n_cubesphere}_{content_name}.nc" def _make_region_cubes(self, full_mesh_cube): """Make a fixed number of region cubes from a full meshcube.""" @@ -139,6 +140,9 @@ def setup( # Fix dask usage mode for all the subsequent performance tests. self.fix_dask_settings() + def teardown(self, _): + self.temp_save_path.unlink(missing_ok=True) + def fix_dask_settings(self): """ Fix "standard" dask behaviour for time+space testing. @@ -165,6 +169,9 @@ def recombine(self): ) return result + def save_recombined_cube(self): + save(self.recombined_cube, self.temp_save_path) + @on_demand_benchmark class CreateCube(Mixin): @@ -215,15 +222,15 @@ class SaveData(Mixin): def time_save(self, n_cubesphere): # Save to disk, which must compute data + stream it to file. - save(self.recombined_cube, "tmp.nc") + self.save_recombined_cube() @TrackAddedMemoryAllocation.decorator() def track_addedmem_save(self, n_cubesphere): - save(self.recombined_cube, "tmp.nc") + self.save_recombined_cube() def track_filesize_saved(self, n_cubesphere): - save(self.recombined_cube, "tmp.nc") - return os.path.getsize("tmp.nc") * 1.0e-6 + self.save_recombined_cube() + return self.temp_save_path.stat().st_size * 1.0e-6 @on_demand_benchmark @@ -243,8 +250,8 @@ def setup( def time_stream_file2file(self, n_cubesphere): # Save to disk, which must compute data + stream it to file. - save(self.recombined_cube, "tmp.nc") + self.save_recombined_cube() @TrackAddedMemoryAllocation.decorator() def track_addedmem_stream_file2file(self, n_cubesphere): - save(self.recombined_cube, "tmp.nc") + self.save_recombined_cube() diff --git a/benchmarks/benchmarks/sperf/load.py b/benchmarks/benchmarks/sperf/load.py index c1d1db43a9..6a60355976 100644 --- a/benchmarks/benchmarks/sperf/load.py +++ b/benchmarks/benchmarks/sperf/load.py @@ -18,9 +18,6 @@ def time_load_cube(self, _, __, ___): @on_demand_benchmark class Realise(FileMixin): - # The larger files take a long time to realise. - timeout = 600.0 - def setup(self, c_size, n_levels, n_times): super().setup(c_size, n_levels, n_times) self.loaded_cube = self.load_cube() diff --git a/lib/iris/tests/stock/netcdf.py b/lib/iris/tests/stock/netcdf.py index 030e90a0f3..4e12850ef1 100644 --- a/lib/iris/tests/stock/netcdf.py +++ b/lib/iris/tests/stock/netcdf.py @@ -9,6 +9,8 @@ from string import Template import subprocess +import dask +from dask import array as da import netCDF4 import numpy as np @@ -79,11 +81,13 @@ def _add_standard_data(nc_path, unlimited_dim_size=0): # so it can be a dim-coord. data_size = np.prod(shape) data = np.arange(1, data_size + 1, dtype=var.dtype).reshape(shape) + var[:] = data else: # Fill with a plain value. But avoid zeros, so we can simulate # valid ugrid connectivities even when start_index=1. - data = np.ones(shape, dtype=var.dtype) # Do not use zero - var[:] = data + with dask.config.set({"array.chunk-size": "2048MiB"}): + data = da.ones(shape, dtype=var.dtype) # Do not use zero + da.store(data, var) ds.close() diff --git a/noxfile.py b/noxfile.py index 00a866f814..7919dfdfde 100755 --- a/noxfile.py +++ b/noxfile.py @@ -9,6 +9,7 @@ import hashlib import os from pathlib import Path +import re from tempfile import NamedTemporaryFile from typing import Literal @@ -314,7 +315,7 @@ def benchmarks( ---------- session: object A `nox.sessions.Session` object. - run_type: {"overnight", "branch", "custom"} + run_type: {"overnight", "branch", "cperf", "sperf", "custom"} * ``overnight``: benchmarks all commits between the input **first commit** to ``HEAD``, comparing each to its parent for performance shifts. If a commit causes shifts, the output is saved to a file: @@ -501,6 +502,11 @@ def asv_compare(*commits): asv_command = ( asv_harness.format(posargs=commit_range) + f" --bench={run_type}" ) + # C/SPerf benchmarks are much bigger than the CI ones: + # Don't fail the whole run if memory blows on 1 benchmark. + asv_command = asv_command.replace(" --strict", "") + # Only do a single round. + asv_command = re.sub(r"rounds=\d", "rounds=1", asv_command) session.run(*asv_command.split(" "), *asv_args) asv_command = f"asv publish {commit_range} --html-dir={publish_subdir}" @@ -511,7 +517,6 @@ def asv_compare(*commits): print( f'New ASV results for "{run_type}".\n' f'See "{publish_subdir}",' - f'\n html in "{location / "html"}".' f'\n or JSON files under "{location / "results"}".' )