Find files for CMIP6 DCPP startdates

esmvalcore/_data_finder.py

@@ -92,13 +92,21 @@ def select_files(filenames, start_year, end_year):
 def _replace_tags(paths, variable):
     """Replace tags in the config-developer's file with actual values."""
     if isinstance(paths, str):
-        paths = (paths.strip('/'), )
+        paths = set((paths.strip('/'),))
     else:
-        paths = [path.strip('/') for path in paths]
+        paths = set(path.strip('/') for path in paths)
     tlist = set()
-
     for path in paths:
         tlist = tlist.union(re.findall(r'{([^}]*)}', path))
+    if 'sub_experiment' in variable:
+        new_paths = []
+        for path in paths:
+            new_paths.extend((
+                re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path),
+                re.sub(r'({ensemble})', r'{sub_experiment}-\1', path)
+            ))
+            tlist.add('sub_experiment')
+        paths = new_paths
     logger.debug(tlist)
 
     for tag in tlist:
@@ -112,7 +120,6 @@ def _replace_tags(paths, variable):
         else:
             raise KeyError("Dataset key {} must be specified for {}, check "
                            "your recipe entry".format(tag, variable))
-
         paths = _replace_tag(paths, original_tag, replacewith)
     return paths
 
@@ -127,7 +134,7 @@ def _replace_tag(paths, tag, replacewith):
     else:
         text = _apply_caps(str(replacewith), lower, upper)
         result.extend(p.replace('{' + tag + '}', text) for p in paths)
-    return result
+    return list(set(result))
 
 
 def _get_caps_options(tag):
@@ -229,6 +236,17 @@ def _get_filenames_glob(variable, drs):
     return filenames_glob
 
 
+def _update_output_file(variable, files):
+    intervals = [get_start_end_year(name) for name in files]
+    variable.update({'start_year': min(intervals)[0]})
+    variable.update({'end_year': max(intervals)[1]})
+    filename = variable['filename'].replace(
+        '.nc', '_{start_year}-{end_year}.nc'.format(**variable)
+    )
+    variable['filename'] = filename
+    return variable
+
+
 def _find_input_files(variable, rootpath, drs):
     short_name = variable['short_name']
     variable['short_name'] = variable['original_short_name']
@@ -248,6 +266,8 @@ def get_input_filelist(variable, rootpath, drs):
     (files, dirnames, filenames) = _find_input_files(variable, rootpath, drs)
     # do time gating only for non-fx variables
     if variable['frequency'] != 'fx':
+        if 'sub_experiment' in variable:
+            variable = _update_output_file(variable, files)
         files = select_files(files, variable['start_year'],
                              variable['end_year'])
     return (files, dirnames, filenames)
@@ -268,7 +288,7 @@ def get_output_file(variable, preproc_dir):
         variable['variable_group'],
         _replace_tags(cfg['output_file'], variable)[0],
     )
-    if variable['frequency'] != 'fx':
+    if variable['frequency'] != 'fx' and 'sub_experiment' not in variable:
         outfile += '_{start_year}-{end_year}'.format(**variable)
     outfile += '.nc'
     return outfile

esmvalcore/_recipe.py

@@ -997,37 +997,38 @@ def _initialize_datasets(raw_datasets):
         return datasets
 
     @staticmethod
-    def _expand_ensemble(variables):
-        """Expand ensemble members to multiple datasets.
+    def _expand_tag(variables, input_tag):
+        """
+        Expand tags such as ensemble members or stardates to multiple datasets.
 
         Expansion only supports ensembles defined as strings, not lists.
         """
         expanded = []
         regex = re.compile(r'\(\d+:\d+\)')
 
-        def expand_ensemble(variable):
-            ens = variable.get('ensemble', "")
-            match = regex.search(ens)
+        def expand_tag(variable, input_tag):
+            tag = variable.get(input_tag, "")
+            match = regex.search(tag)
             if match:
                 start, end = match.group(0)[1:-1].split(':')
                 for i in range(int(start), int(end) + 1):
                     expand = deepcopy(variable)
-                    expand['ensemble'] = regex.sub(str(i), ens, 1)
-                    expand_ensemble(expand)
+                    expand[input_tag] = regex.sub(str(i), tag, 1)
+                    expand_tag(expand, input_tag)
             else:
                 expanded.append(variable)
 
         for variable in variables:
-            ensemble = variable.get('ensemble', "")
-            if isinstance(ensemble, (list, tuple)):
-                for elem in ensemble:
+            tag = variable.get(input_tag, "")
+            if isinstance(tag, (list, tuple)):
+                for elem in tag:
                     if regex.search(elem):
                         raise RecipeError(
-                            f"In variable {variable}: ensemble expansion "
-                            "cannot be combined with ensemble lists")
+                            f"In variable {variable}: {input_tag} expansion "
+                            f"cannot be combined with {input_tag} lists")
                 expanded.append(variable)
             else:
-                expand_ensemble(variable)
+                expand_tag(variable, input_tag)
 
         return expanded
 
@@ -1074,8 +1075,19 @@ def _initialize_variables(self, raw_variable, raw_datasets):
                 activity = get_activity(variable)
                 if activity:
                     variable['activity'] = activity
-            check.variable(variable, required_keys)
-        variables = self._expand_ensemble(variables)
+            if 'sub_experiment' in variable:
+                subexperiment_keys = deepcopy(required_keys)
+                subexperiment_keys.update({'sub_experiment'})
+                try:
+                    subexperiment_keys.remove('start_year')
+                    subexperiment_keys.remove('end_year')
+                except KeyError:
+                    continue
+                check.variable(variable, subexperiment_keys)
+            else:
+                check.variable(variable, required_keys)
+        variables = self._expand_tag(variables, 'ensemble')
+        variables = self._expand_tag(variables, 'sub_experiment')
         return variables
 
     def _initialize_preprocessor_output(self, diagnostic_name, raw_variables,

esmvalcore/_recipe_checks.py

@@ -127,7 +127,7 @@ def data_availability(input_files, var, dirnames, filenames):
             f"Missing data for {var['alias']}: {var['short_name']}")
 
     # check time avail only for non-fx variables
-    if var['frequency'] == 'fx':
+    if var['frequency'] == 'fx' or 'sub_experiment' in var:
         return
 
     required_years = set(range(var['start_year'], var['end_year'] + 1))

tests/unit/data_finder/test_replace_tags.py

@@ -1,22 +1,65 @@
 """Tests for _replace_tags in _data_finder.py."""
-
 from esmvalcore._data_finder import _replace_tags
 
 VARIABLE = {
+    'project': 'CMIP6',
+    'dataset': 'ACCURATE-MODEL',
+    'activity': 'act',
+    'exp': 'experiment',
+    'institute': 'HMA',
+    'ensemble': 'r1i1p1f1',
+    'mip': 'Amon',
     'short_name': 'tas',
+    'grid': 'gr',
 }
 
 
-def test_replace_tags_str():
-    assert _replace_tags('folder/subfolder/{short_name}',
-                         VARIABLE) == ['folder/subfolder/tas']
+def test_replace_tags():
+    """Tests for get_start_end_year function."""
+    path = _replace_tags(
+        '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/'
+        '{grid}/{latestversion}', VARIABLE)
+    input_file = _replace_tags(
+        '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc', VARIABLE)
+    output_file = _replace_tags(
+        '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}', VARIABLE)
+    assert path == [
+        'act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/'
+        '{latestversion}'
+    ]
+    assert input_file == ['tas_Amon_ACCURATE-MODEL_experiment_r1i1p1f1_gr*.nc']
+    assert output_file == ['CMIP6_ACCURATE-MODEL_Amon_experiment_r1i1p1f1_tas']
 
 
 def test_replace_tags_list_of_str():
-    assert _replace_tags(('folder/subfolder/{short_name}',
-                          'folder2/{short_name}', 'subfolder/{short_name}'),
-                         VARIABLE) == [
-                             'folder/subfolder/tas',
-                             'folder2/tas',
-                             'subfolder/tas',
-                         ]
+    assert sorted(
+        _replace_tags(('folder/subfolder/{short_name}', 'folder2/{short_name}',
+                       'subfolder/{short_name}'), VARIABLE)) == sorted([
+                           'folder2/tas',
+                           'folder/subfolder/tas',
+                           'subfolder/tas',
+                       ])
+
+
+def test_replace_tags_with_subexperiment():
+    """Tests for get_start_end_year function."""
+    variable = {'sub_experiment': '199411', **VARIABLE}
+    path = _replace_tags(
+        '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/'
+        '{grid}/{latestversion}', variable)
+    input_file = _replace_tags(
+        '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc', variable)
+    output_file = _replace_tags(
+        '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}', variable)
+    assert sorted(path) == sorted([
+        'act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/'
+        '{latestversion}',
+        'act/HMA/ACCURATE-MODEL/experiment/199411-r1i1p1f1/Amon/tas/gr/'
+        '{latestversion}'
+    ])
+    assert input_file == [
+        'tas_Amon_ACCURATE-MODEL_experiment_199411-r1i1p1f1_gr*.nc'
+    ]
+    assert output_file == [
+        'CMIP6_ACCURATE-MODEL_Amon_experiment_199411-r1i1p1f1_tas'
+    ]

tests/unit/test_recipe.py

@@ -14,7 +14,7 @@ def test_expand_ensemble(self):
             },
         ]
 
-        expanded = Recipe._expand_ensemble(datasets)
+        expanded = Recipe._expand_tag(datasets, 'ensemble')
 
         ensembles = [
             'r1i2p3',
@@ -29,6 +29,31 @@ def test_expand_ensemble(self):
         for i, ensemble in enumerate(ensembles):
             assert expanded[i] == {'dataset': 'XYZ', 'ensemble': ensemble}
 
+    def test_expand_subexperiment(self):
+
+        datasets = [
+            {
+                'dataset': 'XYZ',
+                'sub_experiment': 's(1998:2005)',
+            },
+        ]
+
+        expanded = Recipe._expand_tag(datasets, 'sub_experiment')
+
+        subexperiments = [
+            's1998',
+            's1999',
+            's2000',
+            's2001',
+            's2002',
+            's2003',
+            's2004',
+            's2005',
+        ]
+        for i, subexperiment in enumerate(subexperiments):
+            assert expanded[i] == {'dataset': 'XYZ',
+                                   'sub_experiment': subexperiment}
+
     def test_expand_ensemble_nolist(self):
 
         datasets = [
@@ -39,4 +64,4 @@ def test_expand_ensemble_nolist(self):
         ]
 
         with pytest.raises(RecipeError):
-            Recipe._expand_ensemble(datasets)
+            Recipe._expand_tag(datasets, 'ensemble')