diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6db4df455c..a57dac9675 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -532,9 +532,7 @@ def _get_ancestors(variable, config_user): logger.info("Using input files for variable %s of dataset %s:\n%s", variable['short_name'], variable['dataset'], '\n'.join(input_files)) - if (not config_user.get('skip-nonexistent') - or variable['dataset'] == variable.get('reference_dataset')): - check.data_availability(input_files, variable, dirnames, filenames) + check.data_availability(input_files, variable, dirnames, filenames) # Set up provenance tracking for i, filename in enumerate(input_files): @@ -673,6 +671,16 @@ def get_matching(attributes): return grouped_products +def _allow_skipping(ancestors, variable, config_user): + """Allow skipping of datasets.""" + allow_skipping = all([ + config_user.get('skip-nonexistent'), + not ancestors, + variable['dataset'] != variable.get('reference_dataset'), + ]) + return allow_skipping + + def _get_preprocessor_products(variables, profile, order, ancestor_products, config_user, name): """Get preprocessor product definitions for a set of datasets. @@ -714,7 +722,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, try: ancestors = _get_ancestors(variable, config_user) except RecipeError as ex: - if config_user.get('skip-nonexistent') and not ancestors: + if _allow_skipping(ancestors, variable, config_user): logger.info("Skipping: %s", ex.message) else: missing_vars.add(ex.message) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index f508f0fa4f..938e4b6f2e 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -173,6 +173,8 @@ def _get_concatenation_error(cubes): def concatenate(cubes): """Concatenate all cubes after fixing metadata.""" + if not cubes: + return cubes if len(cubes) == 1: return cubes[0] @@ -227,7 +229,15 @@ def save(cubes, filename, optimize_access='', compress=False, alias='', str filename + Raises + ------ + ValueError + cubes is empty. + """ + if not cubes: + raise ValueError(f"Cannot save empty cubes '{cubes}'") + # Rename some arguments kwargs['target'] = filename kwargs['zlib'] = compress diff --git a/tests/integration/preprocessor/_io/test_concatenate.py b/tests/integration/preprocessor/_io/test_concatenate.py index f8ef998f90..ce0d9529b1 100644 --- a/tests/integration/preprocessor/_io/test_concatenate.py +++ b/tests/integration/preprocessor/_io/test_concatenate.py @@ -1,7 +1,7 @@ """Integration tests for :func:`esmvalcore.preprocessor._io.concatenate`.""" -import warnings import unittest +import warnings from unittest.mock import call import numpy as np @@ -243,6 +243,12 @@ def test_concatenate(self): np.testing.assert_array_equal( concatenated.coord('time').points, np.array([1, 2, 3, 4, 5, 6])) + def test_concatenate_empty_cubes(self): + """Test concatenation with empty :class:`iris.cube.CubeList`.""" + empty_cubes = CubeList([]) + result = _io.concatenate(empty_cubes) + assert result is empty_cubes + def test_concatenate_noop(self): """Test concatenation of a single cube.""" concatenated = _io.concatenate([self.raw_cubes[0]]) diff --git a/tests/integration/preprocessor/_io/test_save.py b/tests/integration/preprocessor/_io/test_save.py index cc6c98364c..9ccb25efcf 100644 --- a/tests/integration/preprocessor/_io/test_save.py +++ b/tests/integration/preprocessor/_io/test_save.py @@ -8,7 +8,7 @@ import netCDF4 import numpy as np from iris.coords import DimCoord -from iris.cube import Cube +from iris.cube import Cube, CubeList from esmvalcore.preprocessor import save @@ -78,6 +78,13 @@ def test_save_zlib(self): self.assertEqual(sample_filters['complevel'], 4) handler.close() + def test_fail_empty_cubes(self): + """Test save fails if empty cubes is provided.""" + (_, filename) = self._create_sample_cube() + empty_cubes = CubeList([]) + with self.assertRaises(ValueError): + save(empty_cubes, filename) + def test_fail_without_filename(self): """Test save fails if filename is not provided.""" cube, _ = self._create_sample_cube() diff --git a/tests/unit/test_recipe.py b/tests/unit/test_recipe.py index 1ed1875926..885684cee2 100644 --- a/tests/unit/test_recipe.py +++ b/tests/unit/test_recipe.py @@ -1,6 +1,6 @@ import pytest -from esmvalcore._recipe import Recipe +from esmvalcore._recipe import Recipe, _allow_skipping from esmvalcore._recipe_checks import RecipeError @@ -40,3 +40,37 @@ def test_expand_ensemble_nolist(self): with pytest.raises(RecipeError): Recipe._expand_ensemble(datasets) + + +VAR_A = {'dataset': 'A'} +VAR_A_REF_A = {'dataset': 'A', 'reference_dataset': 'A'} +VAR_A_REF_B = {'dataset': 'A', 'reference_dataset': 'B'} + + +TEST_ALLOW_SKIPPING = [ + ([], VAR_A, {}, False), + ([], VAR_A, {'skip-nonexistent': False}, False), + ([], VAR_A, {'skip-nonexistent': True}, True), + ([], VAR_A_REF_A, {}, False), + ([], VAR_A_REF_A, {'skip-nonexistent': False}, False), + ([], VAR_A_REF_A, {'skip-nonexistent': True}, False), + ([], VAR_A_REF_B, {}, False), + ([], VAR_A_REF_B, {'skip-nonexistent': False}, False), + ([], VAR_A_REF_B, {'skip-nonexistent': True}, True), + (['A'], VAR_A, {}, False), + (['A'], VAR_A, {'skip-nonexistent': False}, False), + (['A'], VAR_A, {'skip-nonexistent': True}, False), + (['A'], VAR_A_REF_A, {}, False), + (['A'], VAR_A_REF_A, {'skip-nonexistent': False}, False), + (['A'], VAR_A_REF_A, {'skip-nonexistent': True}, False), + (['A'], VAR_A_REF_B, {}, False), + (['A'], VAR_A_REF_B, {'skip-nonexistent': False}, False), + (['A'], VAR_A_REF_B, {'skip-nonexistent': True}, False), +] + + +@pytest.mark.parametrize('ancestors,var,cfg,out', TEST_ALLOW_SKIPPING) +def test_allow_skipping(ancestors, var, cfg, out): + """Test ``_allow_skipping``.""" + result = _allow_skipping(ancestors, var, cfg) + assert result is out