From cb70031a36181de70e3efd35ac509fbe45309e60 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 8 Sep 2022 16:09:00 +0200 Subject: [PATCH] Added tests for input/output filenames for ICON and EMAC CMORizer --- .../_config/extra_facets/ipslcm-mappings.yml | 6 +- esmvalcore/config-developer.yml | 2 +- tests/integration/data_finder.yml | 217 +++++++++++++++++- tests/integration/test_data_finder.py | 18 ++ 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml index eb99044a6a..cacedeb370 100644 --- a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml +++ b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml @@ -111,7 +111,7 @@ IPSL-CM6: # -> general variables precip: {ipsl_varname: precip, <<: *atmvars} slp: {ipsl_varname: slp, <<: *atmvars} - + # -> Turbulent fluxes taux: {ipsl_varname: taux, <<: *atmvars} tauy: {ipsl_varname: tauy, <<: *atmvars} @@ -136,8 +136,8 @@ IPSL-CM6: SWdnSFC: {ipsl_varname: SWdnSFC, <<: *atmvars} LWdnSFcclr: {ipsl_varname: LWdnSFcclr, <<: *atmvars} SWdnSFcclr: {ipsl_varname: SWdnSFcclr, <<: *atmvars} - - + + # ================================================= Lmon: # =============================================== diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 7d2ca06cac..ff282bd4f0 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -133,7 +133,7 @@ EMAC: default: '{exp}/{channel}' input_file: default: '{exp}*{channel}{postproc_flag}.nc' - output_file: '{dataset}_{exp}_{channel}_{mip}_{short_name}' + output_file: '{project}_{dataset}_{exp}_{channel}_{mip}_{short_name}' cmor_type: 'CMIP6' ignore_warnings: load: diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index c375585da3..25ee1ea267 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -26,6 +26,77 @@ get_output_file: preproc_dir: /test output_file: /test/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical-rcp85_r1i1p1_ta_1960-1980.nc + # EMAC + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '1960/1980' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/EMAC_EMAC_amip_Amon_Amon_tas_1960-1980.nc + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: piControl + channel: CH + postproc_flag: -p-mm + timerange: '199001/199002' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/EMAC_EMAC_piControl_CH_Amon_tas_199001-199002.nc + + # ICON + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '1960/1980' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_atm_2d_ml_Amon_tas_1960-1980.nc + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + var_type: custom_var_type + timerange: '20000101/20000102' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_custom_var_type_Amon_tas_20000101-20000102.nc + get_input_filelist: - drs: default @@ -460,9 +531,9 @@ get_input_filelist: file_patterns: - simulation_*_t2m.nc - simulation_*_histmth.nc - found_files: + found_files: - thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Output/MO/simulation_18500101_18591231_1M_histmth.nc - + - drs: default variable: <<: *ipsl_variable @@ -475,9 +546,9 @@ get_input_filelist: file_patterns: - simulation_*_t2m.nc - simulation_*_histmth.nc - found_files: + found_files: - thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Analyse/TS_MO/simulation_18500101_20141231_1M_t2m.nc - + # Test fx files - drs: default @@ -779,3 +850,141 @@ get_input_filelist: file_patterns: - OBS6_ERA-Interim_reanaly_42_Omon_deptho[_.]*nc found_files: [] + + # EMAC + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '200002/200003' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/Amon/amip___________200001_Amon.nc + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + - amip/Amon/amip___________200001_Amon-p-mm.nc + - amip/Amon/amip___________200002_Amon-p-mm.nc + - amip/Amon/amip___________200003_Amon-p-mm.nc + - amip/rad/amip___________200001_rad.nc + - amip/rad/amip___________200002_rad.nc + - amip/rad/amip___________200003_rad.nc + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + - amip/rad/amip___________200003_rad-p-mm.nc + dirs: + - amip/Amon + file_patterns: + - amip*Amon.nc + found_files: + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + channel: rad + postproc_flag: -p-mm + timerange: '200001/200002' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/Amon/amip___________200001_Amon.nc + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + - amip/Amon/amip___________200001_Amon-p-mm.nc + - amip/Amon/amip___________200002_Amon-p-mm.nc + - amip/Amon/amip___________200003_Amon-p-mm.nc + - amip/rad/amip___________200001_rad.nc + - amip/rad/amip___________200002_rad.nc + - amip/rad/amip___________200003_rad.nc + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + - amip/rad/amip___________200003_rad-p-mm.nc + dirs: + - amip/rad + file_patterns: + - amip*rad-p-mm.nc + found_files: + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + + # ICON + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '200002/200003' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/amip_atm_2d_ml_20000101T000000Z.nc + - amip/amip_atm_2d_ml_20000201T000000Z.nc + - amip/amip_atm_2d_ml_20000301T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000401T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000501T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000601T000000Z.nc + dirs: + - amip + - amip/outdata + file_patterns: + - amip_atm_2d_ml*.nc + found_files: + - amip/amip_atm_2d_ml_20000201T000000Z.nc + - amip/amip_atm_2d_ml_20000301T000000Z.nc + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + var_type: var + timerange: '200003/200005' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/amip_var_20000101T000000Z.nc + - amip/amip_var_20000201T000000Z.nc + - amip/amip_var_20000301T000000Z.nc + - amip/outdata/amip_var_20000401T000000Z.nc + - amip/outdata/amip_var_20000501T000000Z.nc + - amip/outdata/amip_var_20000601T000000Z.nc + dirs: + - amip + - amip/outdata + file_patterns: + - amip_var*.nc + found_files: + - amip/amip_var_20000301T000000Z.nc + - amip/outdata/amip_var_20000401T000000Z.nc + - amip/outdata/amip_var_20000501T000000Z.nc diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 07028b2610..966e5a268d 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -25,6 +25,20 @@ CONFIG = yaml.safe_load(file) +def _augment_with_extra_facets(variable): + """Augment variable dict with extra facets.""" + extra_facets = esmvalcore._config.get_extra_facets( + variable['project'], + variable['dataset'], + variable['mip'], + variable['short_name'], + (), + ) + for (key, val) in extra_facets.items(): + if key not in variable: + variable[key] = val + + def print_path(path): """Print path.""" txt = path @@ -68,6 +82,7 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize('cfg', CONFIG['get_output_file']) def test_get_output_file(cfg): """Test getting output name for preprocessed files.""" + _augment_with_extra_facets(cfg['variable']) output_file = get_output_file(cfg['variable'], cfg['preproc_dir']) assert output_file == cfg['output_file'] @@ -88,6 +103,9 @@ def test_get_input_filelist(root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) + # Augment variable dict with extra facets + _augment_with_extra_facets(cfg['variable']) + # Find files rootpath = {cfg['variable']['project']: [root]} drs = {cfg['variable']['project']: cfg['drs']}