Fixed --skip-nonexistent option

esmvalcore/_recipe.py

@@ -532,9 +532,7 @@ def _get_ancestors(variable, config_user):
     logger.info("Using input files for variable %s of dataset %s:\n%s",
                 variable['short_name'], variable['dataset'],
                 '\n'.join(input_files))
-    if (not config_user.get('skip-nonexistent')
-            or variable['dataset'] == variable.get('reference_dataset')):
-        check.data_availability(input_files, variable, dirnames, filenames)
+    check.data_availability(input_files, variable, dirnames, filenames)
 
     # Set up provenance tracking
     for i, filename in enumerate(input_files):
@@ -673,6 +671,16 @@ def get_matching(attributes):
     return grouped_products
 
 
+def _allow_skipping(ancestors, variable, config_user):
+    """Allow skipping of datasets."""
+    allow_skipping = all([
+        config_user.get('skip-nonexistent'),
+        not ancestors,
+        variable['dataset'] != variable.get('reference_dataset'),
+    ])
+    return allow_skipping
+
+
 def _get_preprocessor_products(variables, profile, order, ancestor_products,
                                config_user, name):
     """Get preprocessor product definitions for a set of datasets.
@@ -714,7 +722,7 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products,
             try:
                 ancestors = _get_ancestors(variable, config_user)
             except RecipeError as ex:
-                if config_user.get('skip-nonexistent') and not ancestors:
+                if _allow_skipping(ancestors, variable, config_user):
                     logger.info("Skipping: %s", ex.message)
                 else:
                     missing_vars.add(ex.message)

esmvalcore/preprocessor/_io.py

@@ -173,6 +173,8 @@ def _get_concatenation_error(cubes):
 
 def concatenate(cubes):
     """Concatenate all cubes after fixing metadata."""
+    if not cubes:
+        return cubes
     if len(cubes) == 1:
         return cubes[0]
 
@@ -227,7 +229,15 @@ def save(cubes, filename, optimize_access='', compress=False, alias='',
     str
         filename
 
+    Raises
+    ------
+    ValueError
+        cubes is empty.
+
     """
+    if not cubes:
+        raise ValueError(f"Cannot save empty cubes '{cubes}'")
+
     # Rename some arguments
     kwargs['target'] = filename
     kwargs['zlib'] = compress

tests/integration/preprocessor/_io/test_concatenate.py

@@ -1,7 +1,7 @@
 """Integration tests for :func:`esmvalcore.preprocessor._io.concatenate`."""
 
-import warnings
 import unittest
+import warnings
 from unittest.mock import call
 
 import numpy as np
@@ -243,6 +243,12 @@ def test_concatenate(self):
         np.testing.assert_array_equal(
             concatenated.coord('time').points, np.array([1, 2, 3, 4, 5, 6]))
 
+    def test_concatenate_empty_cubes(self):
+        """Test concatenation with empty :class:`iris.cube.CubeList`."""
+        empty_cubes = CubeList([])
+        result = _io.concatenate(empty_cubes)
+        assert result is empty_cubes
+
     def test_concatenate_noop(self):
         """Test concatenation of a single cube."""
         concatenated = _io.concatenate([self.raw_cubes[0]])

tests/integration/preprocessor/_io/test_save.py

@@ -8,7 +8,7 @@
 import netCDF4
 import numpy as np
 from iris.coords import DimCoord
-from iris.cube import Cube
+from iris.cube import Cube, CubeList
 
 from esmvalcore.preprocessor import save
 
@@ -78,6 +78,13 @@ def test_save_zlib(self):
         self.assertEqual(sample_filters['complevel'], 4)
         handler.close()
 
+    def test_fail_empty_cubes(self):
+        """Test save fails if empty cubes is provided."""
+        (_, filename) = self._create_sample_cube()
+        empty_cubes = CubeList([])
+        with self.assertRaises(ValueError):
+            save(empty_cubes, filename)
+
     def test_fail_without_filename(self):
         """Test save fails if filename is not provided."""
         cube, _ = self._create_sample_cube()

tests/unit/test_recipe.py

@@ -1,6 +1,6 @@
 import pytest
 
-from esmvalcore._recipe import Recipe
+from esmvalcore._recipe import Recipe, _allow_skipping
 from esmvalcore._recipe_checks import RecipeError
 
 
@@ -40,3 +40,37 @@ def test_expand_ensemble_nolist(self):
 
         with pytest.raises(RecipeError):
             Recipe._expand_ensemble(datasets)
+
+
+VAR_A = {'dataset': 'A'}
+VAR_A_REF_A = {'dataset': 'A', 'reference_dataset': 'A'}
+VAR_A_REF_B = {'dataset': 'A', 'reference_dataset': 'B'}
+
+
+TEST_ALLOW_SKIPPING = [
+    ([], VAR_A, {}, False),
+    ([], VAR_A, {'skip-nonexistent': False}, False),
+    ([], VAR_A, {'skip-nonexistent': True}, True),
+    ([], VAR_A_REF_A, {}, False),
+    ([], VAR_A_REF_A, {'skip-nonexistent': False}, False),
+    ([], VAR_A_REF_A, {'skip-nonexistent': True}, False),
+    ([], VAR_A_REF_B, {}, False),
+    ([], VAR_A_REF_B, {'skip-nonexistent': False}, False),
+    ([], VAR_A_REF_B, {'skip-nonexistent': True}, True),
+    (['A'], VAR_A, {}, False),
+    (['A'], VAR_A, {'skip-nonexistent': False}, False),
+    (['A'], VAR_A, {'skip-nonexistent': True}, False),
+    (['A'], VAR_A_REF_A, {}, False),
+    (['A'], VAR_A_REF_A, {'skip-nonexistent': False}, False),
+    (['A'], VAR_A_REF_A, {'skip-nonexistent': True}, False),
+    (['A'], VAR_A_REF_B, {}, False),
+    (['A'], VAR_A_REF_B, {'skip-nonexistent': False}, False),
+    (['A'], VAR_A_REF_B, {'skip-nonexistent': True}, False),
+]
+
+
+@pytest.mark.parametrize('ancestors,var,cfg,out', TEST_ALLOW_SKIPPING)
+def test_allow_skipping(ancestors, var, cfg, out):
+    """Test ``_allow_skipping``."""
+    result = _allow_skipping(ancestors, var, cfg)
+    assert result is out