From 100d27a55e866e5e90aad1624b9ab405a5a926ee Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 22 Jan 2026 14:43:09 +0100 Subject: [PATCH 1/6] Update the CMORizer command: remove deprecated argument and add type hints --- esmvaltool/cmorizers/data/cmorizer.py | 316 +++++++++++++------------- esmvaltool/cmorizers/data/typing.py | 12 + 2 files changed, 175 insertions(+), 153 deletions(-) create mode 100644 esmvaltool/cmorizers/data/typing.py diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index c349aaa807..72e058790e 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -5,14 +5,16 @@ reanalysis. """ +from __future__ import annotations + import datetime import importlib import logging import os import shutil import subprocess -import warnings from pathlib import Path +from typing import TYPE_CHECKING, Literal import esmvalcore import yaml @@ -21,11 +23,17 @@ from esmvalcore.config._dask import get_distributed_client from esmvalcore.config._logging import configure_logging -from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.utilities import read_cmor_config +if TYPE_CHECKING: + from esmvalcore.config import Session + + from esmvaltool.cmorizers.data.typing import DatasetInfo + logger = logging.getLogger(__name__) -datasets_file = os.path.join(os.path.dirname(__file__), "datasets.yml") + + +DATASETS_FILE = Path(__file__).parent / "datasets.yml" class _Formatter: @@ -38,25 +46,29 @@ class _Formatter: Datasets information """ - def __init__(self, info): - self.datasets = [] + def __init__(self, info: dict[str, dict[str, DatasetInfo]]) -> None: + self.datasets: list[str] = [] self.datasets_info = info - self.config = "" + self.config: Session | None = None - def start(self, command, datasets, config_file, config_dir, options): + def start( + self, + command: str, + datasets: str | list[str], + config_dir: Path | None, + options: dict, + ) -> None: """Read configuration and set up formatter for data processing. Parameters ---------- - command: str + command: Name of the command to execute. - datasets: str + datasets: List of datasets to process, comma separated. - config_file: str - Config file to use. Option will be removed in v2.14.0. - config_dir: str + config_dir: Config directory to use. - options: dict() + options: Extra options to overwrite configuration. """ @@ -65,9 +77,7 @@ def start(self, command, datasets, config_file, config_dir, options): else: self.datasets = datasets - if config_file is not None: # remove in v2.14.0 - CFG.load_from_file(config_file) - elif config_dir is not None: + if config_dir is not None: config_dir = ( Path(os.path.expandvars(config_dir)).expanduser().absolute() ) @@ -129,20 +139,32 @@ def log_level(self): return self.config["log_level"] @staticmethod - def _dataset_to_module(dataset): + def _dataset_to_module(dataset: str) -> str: return dataset.lower().replace("-", "_") - def download(self, start_date, end_date, overwrite): + def download( + self, + start_date: datetime.datetime | None, + end_date: datetime.datetime | None, + *, + overwrite: bool, + ) -> bool: """Download all datasets. Parameters ---------- - start_date: datetime + start_date: First date to download - end_date: datetime + end_date: Last date to download - overwrite: boolean + overwrite: If True, download again existing files + + Returns + ------- + : + :obj:`True` if all datasets were downloaded, :obj:`False` otherwise. + """ if not self.datasets: logger.error("Missing datasets to download") @@ -160,18 +182,25 @@ def download(self, start_date, end_date, overwrite): return False return True - def download_dataset(self, dataset, start_date, end_date, overwrite): + def download_dataset( + self, + dataset: str, + start_date: datetime.datetime | None, + end_date: datetime.datetime | None, + *, + overwrite: bool, + ) -> None: """Download a single dataset. Parameters ---------- - dataset: str + dataset: Dataset name - start_date: datetime + start_date: First date to download - end_date: datetime + end_date: Last date to download - overwrite: boolean + overwrite: If True, download again existing files """ if not self.has_downloader(dataset): @@ -200,16 +229,22 @@ def download_dataset(self, dataset, start_date, end_date, overwrite): ) logger.info("%s downloaded", dataset) - def format(self, start, end, install): + def format( + self, + start: datetime.datetime | None, + end: datetime.datetime | None, + *, + install: bool, + ) -> None: """Format all available datasets. Parameters ---------- - start: datetime + start: Start of the period to format - end: datetime + end: End of the period to format - install: bool + install: If True, automatically moves the data to the final location if there is no """ @@ -228,16 +263,17 @@ def format(self, start, end, install): with get_distributed_client(): # loop through tier/datasets to be cmorized for dataset in datasets: - if not self.format_dataset(dataset, start, end, install): + if not self.format_dataset( + dataset, start, end, install=install + ): failed_datasets.append(dataset) if failed_datasets: - raise RuntimeError( - f"Format failed for datasets {' '.join(failed_datasets)}", - ) + msg = f"Format failed for datasets {' '.join(failed_datasets)}" + raise RuntimeError(msg) @staticmethod - def has_downloader(dataset): + def has_downloader(dataset: str) -> bool: """Check if a given datasets has an automatic downloader. Parameters @@ -255,11 +291,11 @@ def has_downloader(dataset): f".{dataset.lower().replace('-', '_')}", package="esmvaltool.cmorizers.data.downloaders.datasets", ) - return True except ImportError: return False + return True - def _assemble_datasets(self): + def _assemble_datasets(self) -> list[str]: """Get my datasets as dictionary keyed on Tier.""" # check for desired datasets only (if any) # if not, walk all over rawobs dir @@ -272,27 +308,34 @@ def _assemble_datasets(self): for tier in tiers if os.path.exists(os.path.join(self.rawobs, tier)) ] - datasets = [] if self.datasets: return self.datasets + datasets = [] for tier in tiers: for dataset in os.listdir(os.path.join(self.rawobs, tier)): datasets.append(dataset) return datasets - def format_dataset(self, dataset, start, end, install): + def format_dataset( + self, + dataset: str, + start: datetime.datetime | None, + end: datetime.datetime | None, + *, + install: bool, + ) -> bool: """Format a single dataset. Parameters ---------- - dataset: str + dataset: Dataset name - start: datetime + start: Start of the period to format - end: datetime + end: End of the period to format - install: bool + install: If True, automatically moves the data to the final location if there is no data there. """ @@ -371,7 +414,7 @@ def format_dataset(self, dataset, start, end, install): shutil.move(out_data_dir, target_dir) return True - def _get_dataset_tier(self, dataset): + def _get_dataset_tier(self, dataset: str) -> str | None: for tier in [2, 3]: if os.path.isdir( os.path.join(self.rawobs, f"Tier{tier}", dataset), @@ -414,7 +457,15 @@ def _write_ncl_settings( write_ncl_settings(settings, settings_filename) return settings_filename - def _run_ncl_script(self, in_dir, out_dir, dataset, script, start, end): + def _run_ncl_script( + self, + in_dir: str, + out_dir: str, + dataset: str, + script: str, + start: datetime.datetime | None, + end: datetime.datetime | None, + ) -> bool: """Run the NCL cmorization mechanism.""" logger.info( "CMORizing dataset %s using NCL script %s", @@ -461,7 +512,14 @@ def _run_ncl_script(self, in_dir, out_dir, dataset, script, start, end): return False return True - def _run_pyt_script(self, in_dir, out_dir, dataset, start, end): + def _run_pyt_script( + self, + in_dir: str, + out_dir: str, + dataset: str, + start: datetime.datetime | None, + end: datetime.datetime | None, + ) -> Literal[True]: """Run the Python cmorization mechanism.""" module_name = ( "esmvaltool.cmorizers.data.formatters.datasets." @@ -482,15 +540,14 @@ def _run_pyt_script(self, in_dir, out_dir, dataset, start, end): class DataCommand: """Download and format data to use with ESMValTool.""" - def __init__(self): - with open(datasets_file, encoding="utf8") as data: - self._info = yaml.safe_load(data) + def __init__(self) -> None: + self._info = yaml.safe_load(DATASETS_FILE.read_text(encoding="utf8")) self.formatter = _Formatter(self._info) - def _has_downloader(self, dataset): + def _has_downloader(self, dataset: str) -> Literal["Yes", "No"]: return "Yes" if self.formatter.has_downloader(dataset) else "No" - def list(self): + def list(self) -> None: """List all supported datasets.""" print() print(f"| {'Dataset name':30} | Tier | Auto-download | Last access |") @@ -507,12 +564,12 @@ def list(self): ) print("-" * 71) - def info(self, dataset): + def info(self, dataset: str) -> None: """Show detailed info about a specific dataset. Parameters ---------- - dataset : str + dataset: dataset to show """ dataset_info = self._info["datasets"][dataset] @@ -526,180 +583,132 @@ def info(self, dataset): def download( self, - datasets, - config_file=None, - start=None, - end=None, - overwrite=False, - config_dir=None, + datasets: str | list[str], + start: str | None = None, + end: str | None = None, + overwrite: bool = False, + config_dir: Path | None = None, **kwargs, - ): + ) -> None: """Download datasets. Parameters ---------- datasets: list(str) List of datasets to format - config_file: str, optional - Path to ESMValTool's config user file, by default None. - - .. deprecated:: 2.12.0 - This option has been deprecated in ESMValTool version 2.12.0 - and is scheduled for removal in version 2.14.0. Please use the - option `config_dir` instead. - start: str, optional + start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - end: str, optional + end: End of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - overwrite: bool, optional + overwrite: If true, download already present data again - config_dir: str, optional + config_dir: Path to additional ESMValTool configuration directory. See - :ref:`esmvalcore:config_yaml_files` for details. + https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#yaml-files + for details. """ - if config_file is not None: - msg = ( - "The option `config_file` has been deprecated in ESMValTool " - "version 2.12.0 and is scheduled for removal in version " - "2.14.0. Please use the option ``config_dir`` instead." - ) - warnings.warn(msg, ESMValToolDeprecationWarning, stacklevel=2) - - start = self._parse_date(start) - end = self._parse_date(end) + start_date = self._parse_date(start) + end_date = self._parse_date(end) self.formatter.start( "download", datasets, - config_file, config_dir, kwargs, ) - self.formatter.download(start, end, overwrite) + self.formatter.download(start_date, end_date, overwrite=overwrite) def format( self, - datasets, - config_file=None, - start=None, - end=None, - install=False, - config_dir=None, + datasets: str | list[str], + start: str | None = None, + end: str | None = None, + install: bool = False, + config_dir: Path | None = None, **kwargs, - ): + ) -> None: """Format datasets. Parameters ---------- - datasets : list(str) + datasets: List of datasets to format - config_file : str, optional - Path to ESMValTool's config user file, by default None - - .. deprecated:: 2.12.0 - This option has been deprecated in ESMValTool version 2.12.0 - and is scheduled for removal in version 2.14.0. Please use the - option `config_dir` instead. - start : str, optional + start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - end : str, optional + end: End of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - install : bool, optional + install: If true, move processed data to the folder, by default False - config_dir: str, optional + config_dir: Path to additional ESMValTool configuration directory. See - :ref:`esmvalcore:config_yaml_files` for details. + https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#yaml-files + for details. """ - if config_file is not None: - msg = ( - "The option `config_file` has been deprecated in ESMValTool " - "version 2.12.0 and is scheduled for removal in version " - "2.14.0. Please use the option ``config_dir`` instead." - ) - warnings.warn(msg, ESMValToolDeprecationWarning, stacklevel=2) - - start = self._parse_date(start) - end = self._parse_date(end) + start_date = self._parse_date(start) + end_date = self._parse_date(end) self.formatter.start( "formatting", datasets, - config_file, config_dir, kwargs, ) - self.formatter.format(start, end, install) + self.formatter.format(start_date, end_date, install=install) def prepare( self, - datasets, - config_file=None, - start=None, - end=None, - overwrite=False, - install=False, - config_dir=None, + datasets: str | list[str], + start: str | None = None, + end: str | None = None, + overwrite: bool = False, + install: bool = False, + config_dir: Path | None = None, **kwargs, - ): + ) -> None: """Download and format a set of datasets. Parameters ---------- - datasets : list(str) + datasets: List of datasets to format - config_file : str, optional - Path to ESMValTool's config user file, by default None - - .. deprecated:: 2.12.0 - This option has been deprecated in ESMValTool version 2.12.0 - and is scheduled for removal in version 2.14.0. Please use the - option `config_dir` instead. - start : str, optional + start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - end : str, optional + end: End of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - install : bool, optional + install: If true, move processed data to the folder, by default False - overwrite : bool, optional + overwrite: If true, download already present data again - config_dir: str, optional + config_dir: Path to additional ESMValTool configuration directory. See - :ref:`esmvalcore:config_yaml_files` for details. + https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#yaml-files + for details. """ - if config_file is not None: - msg = ( - "The option `config_file` has been deprecated in ESMValTool " - "version 2.12.0 and is scheduled for removal in version " - "2.14.0. Please use the option ``config_dir`` instead." - ) - warnings.warn(msg, ESMValToolDeprecationWarning, stacklevel=2) - - start = self._parse_date(start) - end = self._parse_date(end) + start_date = self._parse_date(start) + end_date = self._parse_date(end) self.formatter.start( "preparation", datasets, - config_file, config_dir, kwargs, ) - if self.formatter.download(start, end, overwrite): - self.formatter.format(start, end, install) + if self.formatter.download(start_date, end_date, overwrite=overwrite): + self.formatter.format(start_date, end_date, install=install) else: logger.warning("Download failed, skipping format step") @staticmethod - def _parse_date(date): + def _parse_date(date: str | None) -> datetime.datetime | None: if date is None: return None date_string = str(date) @@ -710,9 +719,10 @@ def _parse_date(date): } format_string = date_formats.get(len(date_string), None) if format_string is None: - raise ValueError( + msg = ( f"Unsupported date format for {date}. " 'Supported formats for "start" and "end" are: ' '"None", "YYYY", "YYYYMM", "YYYYMMDD"', ) + raise ValueError(msg) return datetime.datetime.strptime(date_string, format_string) diff --git a/esmvaltool/cmorizers/data/typing.py b/esmvaltool/cmorizers/data/typing.py new file mode 100644 index 0000000000..aedf65c3dc --- /dev/null +++ b/esmvaltool/cmorizers/data/typing.py @@ -0,0 +1,12 @@ +"""Type definitions for CMORizers.""" + +from typing import TypedDict + + +class DatasetInfo(TypedDict): + """Dataset description.""" + + tier: int + source: str + last_access: str + info: str From ca8b2588eb498046044bc323a23fbba65e819ee3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 22 Jan 2026 22:01:44 +0100 Subject: [PATCH 2/6] Add a command line option to specify original data dir --- .../source/community/code_documentation.rst | 5 + doc/sphinx/source/community/dataset.rst | 15 +- doc/sphinx/source/develop/dataset.rst | 38 +++-- doc/sphinx/source/input.rst | 17 ++- esmvaltool/cmorizers/data/cmorizer.py | 144 +++++++++--------- esmvaltool/cmorizers/data/datasets.yml | 45 +++--- esmvaltool/cmorizers/data/datasets_schema.yml | 1 - .../download_scripts/download_era_interim.py | 44 ++---- esmvaltool/cmorizers/data/downloaders/cds.py | 49 +++--- .../data/downloaders/datasets/aeronet.py | 8 +- .../data/downloaders/datasets/aphro_ma.py | 8 +- .../data/downloaders/datasets/aura_tes.py | 8 +- .../downloaders/datasets/berkeleyearth.py | 8 +- .../downloaders/datasets/calipso_goccp.py | 8 +- .../downloaders/datasets/calipso_icecloud.py | 8 +- .../data/downloaders/datasets/cams.py | 8 +- .../datasets/cds_satellite_albedo.py | 8 +- .../datasets/cds_satellite_lai_fapar.py | 8 +- .../datasets/cds_satellite_soil_moisture.py | 20 +-- .../data/downloaders/datasets/cds_uerra.py | 8 +- .../data/downloaders/datasets/cds_xch4.py | 8 +- .../data/downloaders/datasets/cmap.py | 8 +- .../data/downloaders/datasets/cowtanway.py | 8 +- .../data/downloaders/datasets/cru.py | 8 +- .../data/downloaders/datasets/ct2019.py | 8 +- .../downloaders/datasets/duveiller2018.py | 8 +- .../data/downloaders/datasets/e_obs.py | 8 +- .../downloaders/datasets/eppley_vgpm_modis.py | 8 +- .../downloaders/datasets/esacci_aerosol.py | 8 +- .../downloaders/datasets/esacci_biomass.py | 8 +- .../data/downloaders/datasets/esacci_cloud.py | 16 +- .../data/downloaders/datasets/esacci_fire.py | 8 +- .../downloaders/datasets/esacci_landcover.py | 8 +- .../data/downloaders/datasets/esacci_oc.py | 8 +- .../data/downloaders/datasets/esacci_ozone.py | 10 +- .../datasets/esacci_sea_surface_salinity.py | 8 +- .../downloaders/datasets/esacci_seaice.py | 8 +- .../datasets/esacci_soilmoisture.py | 8 +- .../data/downloaders/datasets/esacci_sst.py | 8 +- .../data/downloaders/datasets/ghcn.py | 8 +- .../data/downloaders/datasets/ghcn_cams.py | 8 +- .../data/downloaders/datasets/gistemp.py | 8 +- .../data/downloaders/datasets/glodap.py | 8 +- .../data/downloaders/datasets/glwd.py | 8 +- .../data/downloaders/datasets/gpcc.py | 12 +- .../data/downloaders/datasets/gpcp_sg.py | 8 +- .../data/downloaders/datasets/hadcrut3.py | 8 +- .../data/downloaders/datasets/hadcrut4.py | 8 +- .../data/downloaders/datasets/hadcrut5.py | 8 +- .../data/downloaders/datasets/hadex3.py | 8 +- .../data/downloaders/datasets/hadisst.py | 8 +- .../data/downloaders/datasets/haloe.py | 8 +- .../data/downloaders/datasets/iap.py | 8 +- .../data/downloaders/datasets/isccp_fh.py | 8 +- .../data/downloaders/datasets/jra_25.py | 8 +- .../data/downloaders/datasets/jra_55.py | 8 +- .../data/downloaders/datasets/kadow2020.py | 8 +- .../downloaders/datasets/landflux_eval.py | 8 +- .../downloaders/datasets/landschuetzer2016.py | 8 +- .../downloaders/datasets/landschuetzer2020.py | 8 +- .../data/downloaders/datasets/merra.py | 8 +- .../data/downloaders/datasets/merra2.py | 8 +- .../downloaders/datasets/mobo_dic2004_2019.py | 8 +- .../downloaders/datasets/mobo_dic_mpim.py | 8 +- .../data/downloaders/datasets/ncep_doe_r2.py | 8 +- .../data/downloaders/datasets/ncep_ncar_r1.py | 8 +- .../data/downloaders/datasets/ndp.py | 8 +- .../datasets/noaa_cires_20cr_v2.py | 8 +- .../datasets/noaa_cires_20cr_v3.py | 8 +- .../downloaders/datasets/noaa_ersstv3b.py | 8 +- .../data/downloaders/datasets/noaa_ersstv5.py | 8 +- .../datasets/noaa_gml_surface_flask_ch4.py | 8 +- .../datasets/noaa_gml_surface_flask_co2.py | 8 +- .../datasets/noaa_gml_surface_flask_n2o.py | 8 +- .../data/downloaders/datasets/noaa_mbl_ch4.py | 8 +- .../downloaders/datasets/noaaglobaltemp.py | 8 +- .../downloaders/datasets/nsidc_0116_nh.py | 8 +- .../downloaders/datasets/nsidc_0116_sh.py | 8 +- .../downloaders/datasets/nsidc_g02202_sh.py | 8 +- .../downloaders/datasets/oceansoda_ethz.py | 8 +- .../data/downloaders/datasets/osi_450_nh.py | 8 +- .../data/downloaders/datasets/osi_450_sh.py | 8 +- .../data/downloaders/datasets/patmos_x.py | 8 +- .../data/downloaders/datasets/persiann_cdr.py | 8 +- .../data/downloaders/datasets/phc.py | 8 +- .../data/downloaders/datasets/regen.py | 8 +- .../downloaders/datasets/scripps_co2_kum.py | 8 +- .../data/downloaders/datasets/tcom_ch4.py | 8 +- .../data/downloaders/datasets/tcom_n2o.py | 8 +- .../data/downloaders/datasets/wad2m.py | 8 +- .../data/downloaders/datasets/woa.py | 8 +- .../data/downloaders/datasets/yang2020.py | 8 +- .../cmorizers/data/downloaders/downloader.py | 50 +++--- esmvaltool/cmorizers/data/downloaders/ftp.py | 69 ++++++--- esmvaltool/cmorizers/data/downloaders/wget.py | 22 ++- .../datasets/cds_satellite_lai_fapar.py | 1 - .../data/formatters/datasets/cds_uerra.py | 2 +- .../data/formatters/datasets/ct2019.py | 3 +- .../formatters/datasets/eppley_vgpm_modis.py | 2 +- .../formatters/datasets/esacci_biomass.py | 2 +- .../formatters/datasets/esacci_landcover.py | 2 +- .../data/formatters/datasets/esacci_lst.py | 2 +- .../data/formatters/datasets/esacci_oc.py | 2 +- .../data/formatters/datasets/esacci_ozone.py | 2 +- .../datasets/esacci_soilmoisture.py | 2 +- .../data/formatters/datasets/grace.py | 3 +- .../data/formatters/datasets/mls_aura.py | 3 +- esmvaltool/cmorizers/data/typing.py | 3 +- tests/integration/test_cmorizer.py | 58 +++---- .../cmorizers/test_cmorization_interface.py | 2 +- tests/unit/cmorizers/test_datasets_info.py | 10 +- 111 files changed, 662 insertions(+), 626 deletions(-) diff --git a/doc/sphinx/source/community/code_documentation.rst b/doc/sphinx/source/community/code_documentation.rst index 82bd72dbfb..4e9f077760 100644 --- a/doc/sphinx/source/community/code_documentation.rst +++ b/doc/sphinx/source/community/code_documentation.rst @@ -260,6 +260,11 @@ However, a pull request where the CircleCI/pre-commit tests are failing cannot b NCL ~~~ +.. warning:: + + Writing new NCL code is not recommended because the + `NCL interpreter `__ is no longer maintained. + Because there is no standard best practices document for NCL, we use `PEP8 `__ for NCL code as well, with some minor adjustments to accommodate for differences in the diff --git a/doc/sphinx/source/community/dataset.rst b/doc/sphinx/source/community/dataset.rst index 7a24e7c923..165531fff7 100644 --- a/doc/sphinx/source/community/dataset.rst +++ b/doc/sphinx/source/community/dataset.rst @@ -43,14 +43,15 @@ and run the recipe, to make sure the CMOR checks pass without warnings or errors To test a pull request for a new CMORizer script: #. Download the data following the instructions included in the script and - place it in the ``RAWOBS`` ``rootpath`` specified in your - :ref:`configuration ` + place it in the directory specified by the ``--original-data-dir`` flag + followed by a subdirectory ``Tier1``, ``Tier2`` or ``Tier3``, followed by + a subdirectory with the same name as the dataset. #. If available, use the downloading script by running - ``esmvaltool data download --config_file `` -#. Run the cmorization by running ``esmvaltool data format `` -#. Copy the resulting data to the ``OBS`` (for CMIP5 compliant data) or ``OBS6`` - (for CMIP6 compliant data) ``rootpath`` specified in your - :ref:`configuration ` + ``esmvaltool data download --original-data-dir `` +#. Run the cmorization by running ``esmvaltool data format --original-data-dir `` +#. Copy the resulting data to the directory specified in your + :ref:`configuration ` for the ``OBS`` + (for CMIP5 compliant data) or ``OBS6`` (for CMIP6 compliant data) project. #. Run ``recipes/examples/recipe_check_obs.yml`` with the new dataset to check that the data can be used diff --git a/doc/sphinx/source/develop/dataset.rst b/doc/sphinx/source/develop/dataset.rst index 032860af61..8bcbd2f0fb 100644 --- a/doc/sphinx/source/develop/dataset.rst +++ b/doc/sphinx/source/develop/dataset.rst @@ -60,17 +60,10 @@ guidelines: It is recommended to use an existing custom table as a template, to edit the content and save it as ``CMOR_.dat``. -2. Edit your configuration file -=============================== - -Make sure that beside the paths to the model simulations and observations, also -the path to raw observational data to be cmorized (``RAWOBS``) is present in -your configuration file. - -3. Store your dataset in the right place +2. Store your dataset in the right place ======================================== -The folder ``RAWOBS`` needs the subdirectories ``Tier1``, ``Tier2`` and +The folder specified by ``--original-data-dir`` needs the subdirectories ``Tier1``, ``Tier2`` and ``Tier3``. The different tiers describe the different levels of restrictions for downloading (e.g. providing contact information, licence agreements) and using the observations. The unformatted (raw) observations @@ -89,12 +82,12 @@ The dataset entry should contain: Note that these fields should be identical to the content of the header of the cmorizing script (see Section `4. Create a cmorizer for the dataset`_). -3.1 Downloader script (optional) +2.1 Downloader script (optional) -------------------------------- A Python script can be written to download raw observations from source and store the data in the appropriate tier subdirectory of the -folder ``RAWOBS`` automatically. +folder specified by the ``--original-data-dir`` flag automatically. There are many downloading scripts available in `/esmvaltool/cmorizers/data/downloaders/datasets/ `_ @@ -117,7 +110,7 @@ The downloading script for the given dataset can be run with: .. code-block:: console - esmvaltool data download --config_dir + esmvaltool data download --original-data-dir The options ``--start`` and ``--end`` can be added to the command above to restrict the download of raw data to a time range. They will be ignored if a specific dataset @@ -125,7 +118,7 @@ does not support it (i.e. because it is provided as a single file). Valid format ``YYYY``, ``YYYYMM`` and ``YYYYMMDD``. By default, already downloaded data are not overwritten unless the option ``--overwrite=True`` is used. -4. Create a cmorizer for the dataset +3. Create a cmorizer for the dataset ==================================== There are many cmorizing scripts available in @@ -145,7 +138,7 @@ the strict CMOR standard. In the following two subsections two cmorizing scripts, one written in Python and one written in NCL, are explained in more detail. -4.1 Cmorizer script written in python +3.1 Cmorizer script written in python ------------------------------------- Find here an example of a cmorizing script, written for the ``MTE`` dataset @@ -166,7 +159,7 @@ augmented with the custom variables described above, are available to your scrip The first part of this configuration file defines the filename of the raw observations file. The second part defines the common global attributes for the cmorizer output, e.g. information that is needed to piece together the -final observations file name in the correct structure (see Section `6. Naming convention of the observational data files`_). +final observations file name in the correct structure (see Section `5. Naming convention of the observational data files`_). Another global attribute is ``reference`` which includes a ``doi`` related to the dataset. Please see the section `adding references `_ @@ -218,9 +211,14 @@ saves a single variable from the raw data. .. _utilities.py: https://github.com/ESMValGroup/ESMValTool/blob/main/esmvaltool/cmorizers/data/utilities.py -4.2 Cmorizer script written in NCL +3.2 Cmorizer script written in NCL ---------------------------------- +.. warning:: + + Writing new NCL code is not recommended because the + `NCL interpreter `__ is no longer maintained. + Find here an example of a cmorizing script, written for the ``ESACCI XCH4`` dataset that is available on the Copernicus Climate Data Store: `cds_xch4.ncl @@ -253,7 +251,7 @@ This module contains different kinds of small fixes to the data attributes, coordinates, and metadata which are necessary for the data field to be CMOR-compliant. -5. Run the cmorizing script +4. Run the cmorizing script =========================== The cmorizing script for the given dataset can be run with: @@ -293,7 +291,7 @@ the cmorizing scripts can be run in a single command with: Note that options from the ```esmvaltool data download`` and ``esmvaltool data format`` commands can be passed to the above command. -6. Naming convention of the observational data files +5. Naming convention of the observational data files ==================================================== For the ESMValTool to be able to read the observations from the NetCDF file, @@ -343,8 +341,8 @@ The different parts of the name are explained in more detail here: specifications for the obs4MIPs data file naming convention in the ``config-developer.yml`` file). -7. Test the cmorized dataset -====================================== +6. Test the cmorized dataset +============================ To verify that the cmorized data file is indeed correctly formatted, you can run a dedicated test recipe, that does not include any diagnostic, but only diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index 9df22ccc2a..f53898973b 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -171,7 +171,10 @@ Datasets for which auto-download is supported can be downloaded with: .. code-block:: bash - esmvaltool data download --config_file [CONFIG_FILE] [DATASET_LIST] + esmvaltool data download --original-data-dir ~/RAWOBS [DATASET_LIST] + +where the ``--original-data-dir`` option specifies a directory where the orignal +data will be stored, ``~/RAWOBS`` in this example but it can be chosen freely. Note that all Tier3 and some Tier2 datasets for which auto-download is supported will require an authentication. In such cases enter your credentials in your @@ -204,11 +207,12 @@ To CMORize one or more datasets, run: .. code-block:: bash - esmvaltool data format --config_file [CONFIG_FILE] [DATASET_LIST] + esmvaltool data format --original-data-dir ~/RAWOBS [DATASET_LIST] + +where the ``--original-data-dir`` option specifies a directory where the orignal +data is stored, ``~/RAWOBS`` in this example but it can be chosen freely. -The ``rootpath`` to the raw data to be CMORized must be specified in the -:ref:`configuration ` as ``RAWOBS``. -Within this path, the data are expected to be organized in subdirectories +Within the directory pointed to by ``--original-data-dir``, the data are expected to be organized in subdirectories corresponding to the data tier: Tier2 for freely-available datasets (other than obs4MIPs and ana4mips) and Tier3 for restricted datasets (i.e., dataset which requires a registration to be retrieved or provided upon request to the @@ -219,8 +223,7 @@ The CMORization follows the `CMIP5 CMOR tables respectively. The resulting output is saved in the output_dir, again following the Tier structure. -The output file names follow the definition given in :ref:`config-developer -file ` for the ``OBS`` project: +The output file names follow the convention: .. code-block:: diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index 72e058790e..444e97738c 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -46,7 +46,10 @@ class _Formatter: Datasets information """ - def __init__(self, info: dict[str, dict[str, DatasetInfo]]) -> None: + def __init__( + self, + info: dict[Literal["datasets"], dict[str, DatasetInfo]], + ) -> None: self.datasets: list[str] = [] self.datasets_info = info self.config: Session | None = None @@ -55,6 +58,7 @@ def start( self, command: str, datasets: str | list[str], + original_data_dir: Path | None, config_dir: Path | None, options: dict, ) -> None: @@ -66,12 +70,17 @@ def start( Name of the command to execute. datasets: List of datasets to process, comma separated. + original_data_dir: + Directory containing the original data. config_dir: Config directory to use. options: Extra options to overwrite configuration. """ + self.original_data_dir = ( + Path.cwd() if original_data_dir is None else original_data_dir + ) if isinstance(datasets, str): self.datasets = datasets.split(",") else: @@ -82,16 +91,16 @@ def start( Path(os.path.expandvars(config_dir)).expanduser().absolute() ) if not config_dir.is_dir(): - raise NotADirectoryError( + msg = ( f"Invalid --config_dir given: {config_dir} is not an " f"existing directory", ) + raise NotADirectoryError(msg) CFG.update_from_dirs([config_dir]) CFG.nested_update(options) self.config = CFG.start_session(f"data_{command}") - if not os.path.isdir(self.run_dir): - os.makedirs(self.run_dir) + self.run_dir.mkdir(parents=True, exist_ok=True) # configure logging log_files = configure_logging( @@ -101,7 +110,7 @@ def start( logger.info("Writing program log files to:\n%s", "\n".join(log_files)) # run - timestamp1 = datetime.datetime.utcnow() + timestamp1 = datetime.datetime.now(datetime.UTC) timestamp_format = "%Y-%m-%d %H:%M:%S" logger.info( @@ -110,19 +119,18 @@ def start( ) logger.info(70 * "-") - logger.info("input_dir = %s", self.rawobs) + logger.info("input_dir = %s", self.original_data_dir) # check if the inputdir actually exists - if not os.path.isdir(self.rawobs): - logger.error("Directory %s does not exist", self.rawobs) - raise ValueError + if not self.original_data_dir.is_dir(): + msg = ( + f"The path '{self.original_data_dir}' is not a directory. " + "Please specify the correct path to the input data using the " + "--original-data-dir flag." + ) + raise NotADirectoryError(msg) logger.info("output_dir = %s", self.output_dir) logger.info(70 * "-") - @property - def rawobs(self): - """Raw obs folder path.""" - return self.config["rootpath"]["RAWOBS"][0] - @property def output_dir(self): """Output folder path.""" @@ -173,7 +181,12 @@ def download( failed_datasets = [] for dataset in self.datasets: try: - self.download_dataset(dataset, start_date, end_date, overwrite) + self.download_dataset( + dataset, + start_date, + end_date, + overwrite=overwrite, + ) except ValueError: logger.exception("Failed to download %s", dataset) failed_datasets.append(dataset) @@ -204,9 +217,8 @@ def download_dataset( If True, download again existing files """ if not self.has_downloader(dataset): - raise ValueError( - f"Dataset {dataset} does not have an automatic downloader", - ) + msg = f"Dataset {dataset} does not have an automatic downloader" + raise ValueError(msg) dataset_module = self._dataset_to_module(dataset) logger.info("Downloading %s", dataset) logger.debug("Download module: %s", dataset_module) @@ -220,12 +232,12 @@ def download_dataset( raise downloader.download_dataset( - self.config, - dataset, - self.datasets_info["datasets"][dataset], - start_date, - end_date, - overwrite, + original_data_dir=self.original_data_dir, + dataset=dataset, + dataset_info=self.datasets_info["datasets"][dataset], + start_date=start_date, + end_date=end_date, + overwrite=overwrite, ) logger.info("%s downloaded", dataset) @@ -255,7 +267,7 @@ def format( logger.warning( "Check input: could not find required %s in %s", self.datasets, - self.rawobs, + self.original_data_dir, ) logger.info("Processing datasets %s", datasets) @@ -296,26 +308,18 @@ def has_downloader(dataset: str) -> bool: return True def _assemble_datasets(self) -> list[str]: - """Get my datasets as dictionary keyed on Tier.""" + """Get the datasets to CMORize.""" # check for desired datasets only (if any) - # if not, walk all over rawobs dir - # assume a RAWOBS/TierX/DATASET input structure - - # get all available tiers in source dir - tiers = [f"Tier{i}" for i in [2, 3]] - tiers = [ - tier - for tier in tiers - if os.path.exists(os.path.join(self.rawobs, tier)) - ] if self.datasets: return self.datasets - datasets = [] - for tier in tiers: - for dataset in os.listdir(os.path.join(self.rawobs, tier)): - datasets.append(dataset) - - return datasets + # if not, look in configuration file and at the available data. + return [ + dataset + for dataset, info in self.datasets_info["datasets"].items() + if ( + self.original_data_dir / f"Tier{info['tier']}" / dataset + ).is_dir() + ] def format_dataset( self, @@ -346,19 +350,16 @@ def format_dataset( self._dataset_to_module(dataset), ) tier = self._get_dataset_tier(dataset) - if tier is None: - logger.error( - "Data for %s not found. Perhaps you are not" - " storing it in a RAWOBS/TierX/%s" - " (X=2 or 3) directory structure?", - dataset, - dataset, - ) - return False # in-data dir; build out-dir tree - in_data_dir = os.path.join(self.rawobs, tier, dataset) + in_data_dir = os.path.join(self.original_data_dir, tier, dataset) logger.info("Input data from: %s", in_data_dir) + if not os.path.isdir(in_data_dir): + msg = ( + f"Data for dataset '{dataset}' not found. " + f"Path to original data '{in_data_dir}' is not a directory'" + ) + raise NotADirectoryError(msg) out_data_dir = os.path.join(self.output_dir, tier, dataset) logger.info("Output will be written to: %s", out_data_dir) if not os.path.isdir(out_data_dir): @@ -414,13 +415,8 @@ def format_dataset( shutil.move(out_data_dir, target_dir) return True - def _get_dataset_tier(self, dataset: str) -> str | None: - for tier in [2, 3]: - if os.path.isdir( - os.path.join(self.rawobs, f"Tier{tier}", dataset), - ): - return f"Tier{tier}" - return None + def _get_dataset_tier(self, dataset: str) -> str: + return f"Tier{self.datasets_info['datasets'][dataset]['tier']}" def _write_ncl_settings( self, @@ -584,6 +580,7 @@ def info(self, dataset: str) -> None: def download( self, datasets: str | list[str], + original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, overwrite: bool = False, @@ -596,6 +593,8 @@ def download( ---------- datasets: list(str) List of datasets to format + original_data_dir: + Directory where original data will be stored. start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -615,15 +614,17 @@ def download( self.formatter.start( "download", - datasets, - config_dir, - kwargs, + datasets=datasets, + original_data_dir=original_data_dir, + config_dir=config_dir, + options=kwargs, ) self.formatter.download(start_date, end_date, overwrite=overwrite) def format( self, datasets: str | list[str], + original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, install: bool = False, @@ -636,6 +637,8 @@ def format( ---------- datasets: List of datasets to format + original_data_dir: + Directory where original data is stored. start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -655,15 +658,17 @@ def format( self.formatter.start( "formatting", - datasets, - config_dir, - kwargs, + datasets=datasets, + original_data_dir=original_data_dir, + config_dir=config_dir, + options=kwargs, ) self.formatter.format(start_date, end_date, install=install) def prepare( self, datasets: str | list[str], + original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, overwrite: bool = False, @@ -677,6 +682,8 @@ def prepare( ---------- datasets: List of datasets to format + original_data_dir: + Directory where original data is stored or will be stored after download. start: Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -698,9 +705,10 @@ def prepare( self.formatter.start( "preparation", - datasets, - config_dir, - kwargs, + datasets=datasets, + original_data_dir=original_data_dir, + config_dir=config_dir, + options=kwargs, ) if self.formatter.download(start_date, end_date, overwrite=overwrite): self.formatter.format(start_date, end_date, install=install) diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 16d5960693..a1cbc5820c 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -151,7 +151,7 @@ datasets: - Tick all available months - Tick Nominal day 20 - Click 'submit form' - - According to ESMValTool practice, put them in the right rawobsdir folder + - According to ESMValTool practice, put them in the Tier3/CDS-SATELLITE-LAI-FAPAR folder - Request might need to be split into chunks to not exceed download limit - Fails setting standard name for variable FAPAR @@ -316,8 +316,7 @@ datasets: source: https://www.esrl.noaa.gov/gmd/ccgg/carbontracker/index.php last_access: 2020-03-23 info: | - Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your configuration) where the raw + Create a new empty directory ``Tier2/CT2019`` where the original data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). @@ -346,7 +345,7 @@ datasets: last_access: 2019-05-15 info: | Download and unpack all the *.tar files under a single directory - (no subdirectories with years) in ${RAWOBS}/Tier2/Eppley-VGPM-MODIS + (no subdirectories with years) in Tier2/Eppley-VGPM-MODIS ERA-Interim-Land: tier: 3 @@ -463,13 +462,13 @@ datasets: source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/biomass/data/agb/maps/v6.0/netcdf last_access: 2025-07-16 info: | - Download and processing instructions: - Use the following CLI to download all the files: - esmvaltool data download ESACCI-BIOMASS - The underlying downloader is located here: - /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py - and it will download all the file regridded to 10 km available on CEDA (2007, 2010, 2015-2021) - under a single directory as follow: ${RAWOBS}/Tier2/ESACCI-BIOMASS + Download and processing instructions: + Use the following CLI to download all the files: + esmvaltool data download ESACCI-BIOMASS + The underlying downloader is located here: + /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py + and it will download all the file regridded to 10 km available on CEDA (2007, 2010, 2015-2021) + under a single directory as follow: Tier2/ESACCI-BIOMASS ESACCI-CLOUD: tier: 2 @@ -505,13 +504,13 @@ datasets: source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/land_cover/data/pft/v2.0.8/ last_access: 2024-07-11 info: | - Download and processing instructions: - Use the following CLI to download all the files: - esmvaltool data download ESACCI-LANDCOVER - The underlying downloader is located here: - /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py - and it will download all the files currently available on CEDA (1992-2020) - under a single directory as follow: ${RAWOBS}/Tier2/ESACCI-LANDCOVER + Download and processing instructions: + Use the following CLI to download all the files: + esmvaltool data download ESACCI-LANDCOVER + The underlying downloader is located here: + /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py + and it will download all the files currently available on CEDA (1992-2020) + under a single directory as follow: Tier2/ESACCI-LANDCOVER ESACCI-LST: tier: 2 @@ -522,7 +521,7 @@ datasets: https://accounts.jasmin.ac.uk/services/group_workspaces/esacci_lst/ Download and processing instructions: Put all files under a single directory (no subdirectories with years) - in ${RAWOBS}/Tier2/ESACCI-LST + in Tier2/ESACCI-LST BOTH DAY and NIGHT files are needed for each month Caveats @@ -537,7 +536,7 @@ datasets: OceanColour webpage https://esa-oceancolour-cci.org/ Username and password are provided on this website Put all files under a single directory (no subdirectories with years) - in ${RAWOBS}/Tier2/ESACCI-OC + in Tier2/ESACCI-OC ESACCI-OZONE: tier: 2 @@ -765,7 +764,7 @@ datasets: - GRCTellus.JPL.200204_202004.GLO.RL06M.MSCNv02CRI.nc - LAND_MASK.CRI.nc - Download the grace months table which holds important information - on data coverage. Save it in the RAWOBSDIR. + on data coverage. Save it in the Tier3/GRACE directory. https://podaac-tools.jpl.nasa.gov/drive/files/allData/tellus/L3/docs/GRACE_GRACE-FO_Months_RL06.csv - Manually inspect and check the months table @@ -998,8 +997,7 @@ datasets: info: | Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the - $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the configuration. Apply this procedure to both + Tier3/MLS-AURA directory. Apply this procedure to both links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). @@ -1069,7 +1067,6 @@ datasets: surface pr_wtr.eatm.mon.mean.nc - NCEP-NCAR-R1: tier: 2 source: https://psl.noaa.gov/data/gridded/data.ncep.reanalysis.html diff --git a/esmvaltool/cmorizers/data/datasets_schema.yml b/esmvaltool/cmorizers/data/datasets_schema.yml index 5d43f00724..7eeef726ee 100644 --- a/esmvaltool/cmorizers/data/datasets_schema.yml +++ b/esmvaltool/cmorizers/data/datasets_schema.yml @@ -11,6 +11,5 @@ datasets: map(include('dataset'), required=True) dataset: tier: int(min=2, max=3, required=False) source: str() - latest: int(required=False, min=20150000, max=20210000) info: str() last_access: day() diff --git a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py index 621e4818a6..69f34aaa0e 100644 --- a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py +++ b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py @@ -12,34 +12,26 @@ 4. Copy/paste the text in https://api.ecmwf.int/v1/key/ into a blank text file and save it as $HOME/.ecmwfapirc -5. Copy the default configuration file with - -```bash -esmvaltool config get_config_user --path=config-user.yml -``` - -and set the ``rootpath`` for the RAWOBS project. - -6. Check the description of the variables at +5. Check the description of the variables at https://apps.ecmwf.int/codes/grib/param-db -7. Check the invariant variables at +6. Check the invariant variables at https://apps.ecmwf.int/datasets/data/interim-full-invariant ```bash -python download_era_interim.py --config_file config-user.yml --start_year 2000 ---end_year 2000 +python download_era_interim.py --original-data-dir /path/to/save/data +--start_year 2000 --end_year 2000 ``` -This will download and save the data in the RAWOBS directory, +This will download and save the data in the /path/to/save/data directory, under Tier3/ERA-Interim. """ import argparse import os +from pathlib import Path -import yaml from ecmwfapi import ECMWFDataServer DAY_TIMESTEPS = { @@ -249,13 +241,14 @@ def _get_invariant_data(params, server, era_interim_dir): ) -def cli(): +def cli() -> None: """Download ERA-Interim variables from ECMWF data server.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--config_file", - "-c", - default=os.path.join(os.path.dirname(__file__), "config-user.yml"), + "--original-data-dir", + "-o", + type=Path, + default=Path.cwd(), help="Config file", ) parser.add_argument( @@ -267,20 +260,9 @@ def cli(): parser.add_argument("--end_year", type=int, default=2019, help="End year") args = parser.parse_args() - # get and read config file - config_file_name = os.path.abspath( - os.path.expandvars(os.path.expanduser(args.config_file)), - ) - - with open(config_file_name) as config_file: - config = yaml.safe_load(config_file) - - rawobs_dir = os.path.abspath( - os.path.expandvars(os.path.expanduser(config["rootpath"]["RAWOBS"])), - ) - era_interim_dir = f"{rawobs_dir}/Tier3/ERA-Interim" + era_interim_dir = f"{args.original_data_dir}/Tier3/ERA-Interim" os.makedirs(era_interim_dir, exist_ok=True) - era_interim_land_dir = f"{rawobs_dir}/Tier3/ERA-Interim-Land" + era_interim_land_dir = f"{args.original_data_dir}/Tier3/ERA-Interim-Land" os.makedirs(era_interim_land_dir, exist_ok=True) years = range(args.start_year, args.end_year + 1) diff --git a/esmvaltool/cmorizers/data/downloaders/cds.py b/esmvaltool/cmorizers/data/downloaders/cds.py index a0e8032b64..59f28d227b 100644 --- a/esmvaltool/cmorizers/data/downloaders/cds.py +++ b/esmvaltool/cmorizers/data/downloaders/cds.py @@ -1,13 +1,20 @@ """Downloader for the Climate Data Store.""" +from __future__ import annotations + import logging import os from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING import cdsapi from .downloader import BaseDownloader +if TYPE_CHECKING: + from esmvaltool.cmorizers.data.typing import DatasetInfo + logger = logging.getLogger(__name__) @@ -16,35 +23,41 @@ class CDSDownloader(BaseDownloader): Parameters ---------- - product_name : str + product_name: Name of the product in the CDS - config : dict - ESMValTool's user configuration - request_dictionary : dict + original_data_dir: + Directory where original data will be stored. + request_dictionary: Common CDS request parameters - dataset : str + dataset: Name of the dataset - dataset_info : dict + dataset_info: Dataset information from the datasets.yml file - overwrite : bool + overwrite: Overwrite already downloaded files - extra_name : str, optional + extra_name: Some products have a subfix appended to their name for certain variables. This parameter is to specify it, by default '' """ def __init__( self, - product_name, - config, - request_dictionary, - dataset, - dataset_info, - overwrite, - extra_name="", - cds_url="https://cds.climate.copernicus.eu/api", - ): - super().__init__(config, dataset, dataset_info, overwrite) + original_data_dir: Path, + product_name: str, + request_dictionary: dict, + dataset: str, + dataset_info: DatasetInfo, + *, + overwrite: bool, + extra_name: str = "", + cds_url: str = "https://cds.climate.copernicus.eu/api", + ) -> None: + super().__init__( + original_data_dir=original_data_dir, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) try: self._client = cdsapi.Client(url=cds_url) except Exception as ex: diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py index d6409bd61c..b21edb884f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py index e93677cb16..f8c18ea122 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py index b9c031bf6c..4c0b180c14 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py @@ -4,7 +4,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -15,8 +15,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -29,7 +29,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py index 164c97889a..0e446801f8 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py @@ -4,7 +4,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -15,8 +15,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -29,7 +29,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py index ea9b062bd6..5f2fdec622 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="ftp.climserv.ipsl.polytechnique.fr", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py index d38ec2dc9b..9af5cd5e19 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cams.py b/esmvaltool/cmorizers/data/downloaders/datasets/cams.py index 1a0720e3cc..92ce1ac9ae 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cams.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cams.py @@ -7,7 +7,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -18,8 +18,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -45,7 +45,7 @@ def download_dataset( "time_aggregation": "monthly_mean", "version": "v23r1", }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py index cf7960035b..fdec099cbd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -52,7 +52,7 @@ def download_dataset( ], "nominal_day": "20", }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py index 5fcb33dea5..92a87cf1e5 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -53,7 +53,7 @@ def download_dataset( "nominal_day": "20", "format": "tgz", }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py index 46099bebf3..637a6ff56f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -51,7 +51,7 @@ def download_dataset( "time_aggregation": "month_average", "day": ["01"], }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, @@ -62,7 +62,7 @@ def download_dataset( for sensor in ["combined_passive_and_active", "passive", "active"]: monthly_downloaders[sensor] = get_downloader( - config, + original_data_dir, dataset, dataset_info, overwrite, @@ -70,7 +70,7 @@ def download_dataset( "month", ) daily_downloaders[sensor] = get_downloader( - config, + original_data_dir, dataset, dataset_info, overwrite, @@ -108,7 +108,7 @@ def download_dataset( def get_downloader( - config, + original_data_dir, dataset, dataset_info, overwrite, @@ -119,8 +119,8 @@ def get_downloader( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -147,7 +147,7 @@ def get_downloader( "version": "v201912.0.0", "time_aggregation": f"{frequency}_average", }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py index c9eac75603..5c30770eb6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -52,7 +52,7 @@ def download_dataset( ], "time": ["00:00", "06:00", "12:00", "18:00"], }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py index d1119eb29b..7748b5f087 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py @@ -5,7 +5,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -16,8 +16,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -38,7 +38,7 @@ def download_dataset( "sensor_and_algorithm": "merged_obs4mips", "version": "4.1", }, - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cmap.py b/esmvaltool/cmorizers/data/downloaders/datasets/cmap.py index cf31a710ca..fe2255de4d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cmap.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cmap.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="ftp2.psl.noaa.gov", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py index 6f58d05e5d..80f7901ff6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py index 4a1c3518a1..4238350958 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py index c0d998cb5a..c455bd7b48 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py index d7516634fa..6bee7a4ad6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py index e399b032c1..96438bb28c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py index 36b71e1f65..516fd0d228 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py @@ -12,7 +12,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -23,8 +23,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py index 9025fd5925..21ac1f4280 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py index ef9f04b32c..888ee638c7 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_biomass.py @@ -4,7 +4,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -15,8 +15,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -30,7 +30,7 @@ def download_dataset( """ # Initialize the downloader downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py index 9a4bdb9abb..1e90444bdb 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py @@ -12,7 +12,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -23,8 +23,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -45,20 +45,20 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, ) # check if daily data needs to be downloaded - cmor_config = read_cmor_config(dataset) - daily_data = cmor_config["daily_data"] + cmor_original_data_dir = read_cmor_config(dataset) + daily_data = cmor_original_data_dir["daily_data"] if not daily_data: logger.info( 'If daily data needs to be downloaded change "daily_data" in the ' - 'cmor_config file to "True" ' - "(esmvaltool/cmorizers/data/cmor_config/ESACCI-CLOUD.yml)", + 'cmor_original_data_dir file to "True" ' + "(esmvaltool/cmorizers/data/cmor_original_data_dir/ESACCI-CLOUD.yml)", ) # Base paths for L3U (daily data) and L3C (monthly data) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py index fdacbce64b..6d6ebfd872 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py index d61d73282e..8d641e3842 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py @@ -6,7 +6,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -17,8 +17,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -38,7 +38,7 @@ def download_dataset( # Initialize the downloader downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py index 07400d4bfc..b04c332b79 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -40,7 +40,7 @@ def download_dataset( loop_date = start_date downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py index c29cd3245d..82f528023c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py @@ -4,7 +4,6 @@ import logging import shutil import zipfile -from pathlib import Path import cdsapi @@ -12,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -25,7 +24,7 @@ def download_dataset( https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1. - The file named .cdspirc containing the key associated to the ECMWF account needs to be saved in user's ${HOME} directory. - - All the files will be saved in ${RAWOBS}/Tier2/ESACCI-OZONE. + - All the files will be saved in Tier2/ESACCI-OZONE. """ cds_url = "https://cds.climate.copernicus.eu/api" @@ -52,8 +51,9 @@ def download_dataset( } client = cdsapi.Client(cds_url) - raw_obs_dir = Path(config["rootpath"]["RAWOBS"][0]) - output_folder = raw_obs_dir / f"Tier{dataset_info['tier']}" / dataset + output_folder = ( + original_data_dir / f"Tier{dataset_info['tier']}" / dataset + ) output_folder.mkdir(parents=True, exist_ok=True) for var_name, request in requests.items(): diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py index 9ac67b9ce8..638f1e564c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py @@ -11,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -22,8 +22,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -42,7 +42,7 @@ def download_dataset( loop_date = start_date downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_seaice.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_seaice.py index 5b4cfafcbb..b66d8cb3cd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_seaice.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_seaice.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( end_date = datetime(2020, 12, 31) downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py index 3afe6fa862..d403a60e67 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -40,7 +40,7 @@ def download_dataset( loop_date = start_date downloader = CCIDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py index ee74a439c5..6f315f4a1d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sst.py @@ -11,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -22,8 +22,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -43,7 +43,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py index c599fd8f1d..32bce61b8d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py index 4205280bf0..86d45666d3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py index 88b9f7f97e..3c99e94269 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py index 8ccd4dd16a..421e195dd1 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/glwd.py b/esmvaltool/cmorizers/data/downloaders/datasets/glwd.py index b1d4decd4c..66e7d55005 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/glwd.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/glwd.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py index 8490cdcf27..1f25b9a906 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py @@ -12,7 +12,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -23,8 +23,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,18 +37,18 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, ) - cmor_config = read_cmor_config(dataset) + cmor_original_data_dir = read_cmor_config(dataset) raw_path = ( "https://opendata.dwd.de/climate_environment/GPCC/" "full_data_2018/full_data_monthly_{version}.nc.gz" ) - for version in cmor_config["attributes"]["version"].values(): + for version in cmor_original_data_dir["attributes"]["version"].values(): downloader.download_file( raw_path.format(version=version), wget_options=[], diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py index d56ffc6812..076f013d1c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py index 8e3a600294..3594e301dd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py index ae0c61c3cf..13f8d5c2d3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py index 7e20692031..f37c1c9ca0 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py index 8d39fc97b8..46b863f5ed 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py index a9c886f7a7..5e7007e89c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py index fdd4dcc4cc..9c61beefec 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/iap.py b/esmvaltool/cmorizers/data/downloaders/datasets/iap.py index 468d4c7230..a3dc334a67 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/iap.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/iap.py @@ -14,7 +14,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -25,8 +25,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -46,7 +46,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py index 79b38f8fb9..8ecb1aac30 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -40,7 +40,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py index 15afe6c113..a7df087716 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py @@ -12,7 +12,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -23,8 +23,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py index 900b031979..af6b7a7627 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py @@ -12,7 +12,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -23,8 +23,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py index 3bfbe827df..eb0389e72c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py index 79c72a3bf3..4f41cab690 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py index 28c4f4a510..4a1c59c8d9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py index 67cedea469..0c12c80d5f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py index 64155125df..fb831e9559 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py index 7a9e0d8ec3..a10467c640 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py index 7a45c9c4e8..d11f8c2218 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py index 544ea394c1..da135e09d8 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py index 4d321c1e8a..6d3092eed4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py index 46be6616b8..a8b2248c7f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( if end_date is None: end_date = datetime(2021, 1, 1) downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="ftp.cdc.noaa.gov", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py index 01123e0ea6..f07a5bfe16 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py index 6de5f5ee45..f737dbf282 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="ftp.cdc.noaa.gov", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py index 02451e0ea1..90b3101d89 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="ftp.cdc.noaa.gov", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py index c93f18a62d..8d31a07203 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py @@ -11,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -22,8 +22,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -43,7 +43,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py index ed961f591d..7c1bf29dbf 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py @@ -11,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -22,8 +22,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -42,7 +42,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_ch4.py index cd4c4857ab..65bde135d4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_ch4.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_co2.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_co2.py index 99930d7e35..9c9cbc739d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_co2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_co2.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_n2o.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_n2o.py index 2ba36543f8..f899f502dd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_n2o.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_gml_surface_flask_n2o.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py index 979df55221..f42c21e3f6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py index 9b4bab5de5..d72beb8b92 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py index 322ce2954a..44a5b40b83 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py index 2f9be1c9a4..04e799abdf 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -39,7 +39,7 @@ def download_dataset( loop_date = start_date downloader = NASADownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py index f32d0eac2e..4c5260b6b4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py @@ -11,7 +11,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -22,8 +22,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -43,7 +43,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py index 95a09be5c0..8270f2dd6b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py index 79b076f68c..536592450b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( if end_date is None: end_date = datetime(2020, 12, 1) downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="osisaf.met.no", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py index a2f1e09adc..daec4f17d1 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -37,7 +37,7 @@ def download_dataset( if end_date is None: end_date = datetime(2020, 12, 1) downloader = FTPDownloader( - config=config, + original_data_dir=original_data_dir, server="osisaf.met.no", dataset=dataset, dataset_info=dataset_info, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py index 71feb28cbd..9c74d34276 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -40,7 +40,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py index 0ce1c5a443..6b4587ffeb 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -46,7 +46,7 @@ def download_dataset( print(base_path.format(year=loop_date.year)) print(base_path) downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py index d207f22d5f..609693b568 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -34,7 +34,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py index b4cec171ce..dcac15860b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py @@ -9,7 +9,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -20,8 +20,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -40,7 +40,7 @@ def download_dataset( loop_date = start_date downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py index 32aba5f93f..2779c0328f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py index 8d96b527ab..5d7697d2af 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py index 0aa9923016..2da7a54e90 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/wad2m.py b/esmvaltool/cmorizers/data/downloaders/datasets/wad2m.py index 7d0bdb2946..4169f70687 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/wad2m.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/wad2m.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py index 8089773293..5ff75ff390 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py @@ -10,7 +10,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -21,8 +21,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -35,7 +35,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/yang2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/yang2020.py index 24a4a209dc..daa4e4ee4b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/yang2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/yang2020.py @@ -8,7 +8,7 @@ def download_dataset( - config, + original_data_dir, dataset, dataset_info, start_date, @@ -19,8 +19,8 @@ def download_dataset( Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir : Path + Directory where original data will be stored. dataset : str Name of the dataset dataset_info : dict @@ -33,7 +33,7 @@ def download_dataset( Overwrite already downloaded files """ downloader = WGetDownloader( - config=config, + original_data_dir=original_data_dir, dataset=dataset, dataset_info=dataset_info, overwrite=overwrite, diff --git a/esmvaltool/cmorizers/data/downloaders/downloader.py b/esmvaltool/cmorizers/data/downloaders/downloader.py index 9d0366f274..a84c4a622b 100644 --- a/esmvaltool/cmorizers/data/downloaders/downloader.py +++ b/esmvaltool/cmorizers/data/downloaders/downloader.py @@ -1,6 +1,12 @@ """Downloader base class.""" -import os +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from esmvaltool.cmorizers.data.typing import DatasetInfo class BaseDownloader: @@ -8,45 +14,37 @@ class BaseDownloader: Parameters ---------- - config : dict - ESMValTool's user configuration - dataset : str + original_data_dir: + Directory where original data will be stored. + dataset: Dataset to download - dataset_info : dict + dataset_info: Dataset information from the datasets.yml file - overwrite : bool + overwrite: Overwrite already downloaded files """ - def __init__(self, config, dataset, dataset_info, overwrite): - self._config = config + def __init__( + self, + original_data_dir: Path, + dataset: str, + dataset_info: DatasetInfo, + *, + overwrite: bool, + ) -> None: + self.original_data_dir = original_data_dir self.tier = dataset_info["tier"] self.dataset = dataset self.dataset_info = dataset_info self.overwrite = overwrite @property - def local_folder(self): + def local_folder(self) -> str: """Folder to store the downloader date. Returns ------- - str + : Path to the download folder """ - return os.path.join( - self.rawobs_folder, - f"Tier{self.tier}", - self.dataset, - ) - - @property - def rawobs_folder(self): - """RAWOBS base path. - - Returns - ------- - str - Path to the RAWOBS folder - """ - return self._config["rootpath"]["RAWOBS"][0] + return str(self.original_data_dir / f"Tier{self.tier}" / self.dataset) diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index e15860a662..d57c22d725 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -1,9 +1,13 @@ """Downloader for FTP repositories.""" +from __future__ import annotations + import ftplib import logging import os import re +from pathlib import Path +from typing import TYPE_CHECKING from progressbar import ( ETA, @@ -16,6 +20,10 @@ from .downloader import BaseDownloader +if TYPE_CHECKING: + from esmvaltool.cmorizers.data.typing import DatasetInfo + + logger = logging.getLogger(__name__) @@ -24,29 +32,39 @@ class FTPDownloader(BaseDownloader): Parameters ---------- - config : dict - ESMValTool's user configuration - server : str + original_data_dir: + Directory where original data will be stored. + server: FTP server URL - dataset : str + dataset: Dataset to download - dataset_info : dict + dataset_info: Dataset information from the datasets.yml file - overwrite : bool + overwrite: Overwrite already downloaded files + user: + Username + passwd: + Password """ def __init__( self, - config, - server, - dataset, - dataset_info, - overwrite, - user=None, - passwd=None, + original_data_dir: Path, + server: str, + dataset: str, + dataset_info: DatasetInfo, + *, + overwrite: bool, + user: str | None = None, + passwd: str | None = None, ): - super().__init__(config, dataset, dataset_info, overwrite) + super().__init__( + original_data_dir=original_data_dir, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) self._client = None self.server = server self.user = user @@ -190,8 +208,8 @@ class CCIDownloader(FTPDownloader): Parameters ---------- - config : dict - ESMValTool's user configuration + original_data_dir: + Directory where original data will be stored. dataset : str Dataset to download dataset_info : dict @@ -200,13 +218,20 @@ class CCIDownloader(FTPDownloader): Overwrite already downloaded files """ - def __init__(self, config, dataset, dataset_info, overwrite): + def __init__( + self, + original_data_dir: Path, + dataset: str, + dataset_info: DatasetInfo, + *, + overwrite: bool, + ) -> None: super().__init__( - config, - "anon-ftp.ceda.ac.uk", - dataset, - dataset_info, - overwrite, + original_data_dir=original_data_dir, + server="anon-ftp.ceda.ac.uk", + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, ) self.ftp_name = self.dataset_name[7:] diff --git a/esmvaltool/cmorizers/data/downloaders/wget.py b/esmvaltool/cmorizers/data/downloaders/wget.py index 6b79987e3b..fa13f18d02 100644 --- a/esmvaltool/cmorizers/data/downloaders/wget.py +++ b/esmvaltool/cmorizers/data/downloaders/wget.py @@ -1,11 +1,17 @@ """wget based downloader.""" +from __future__ import annotations + import logging import subprocess from pathlib import Path +from typing import TYPE_CHECKING from .downloader import BaseDownloader +if TYPE_CHECKING: + from esmvaltool.cmorizers.data.typing import DatasetInfo + logger = logging.getLogger(__name__) @@ -109,8 +115,20 @@ def login(self, server_path, wget_options): class NASADownloader(WGetDownloader): """Downloader for the NASA repository.""" - def __init__(self, config, dataset, dataset_info, overwrite): - super().__init__(config, dataset, dataset_info, overwrite) + def __init__( + self, + original_data_dir: Path, + dataset: str, + dataset_info: DatasetInfo, + *, + overwrite: bool, + ) -> None: + super().__init__( + original_data_dir=original_data_dir, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) self._wget_common_options = [ "--load-cookies=~/.urs_cookies", diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cds_satellite_lai_fapar.py b/esmvaltool/cmorizers/data/formatters/datasets/cds_satellite_lai_fapar.py index ed56063bcd..3e81d06125 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/cds_satellite_lai_fapar.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/cds_satellite_lai_fapar.py @@ -19,7 +19,6 @@ - Tick all available months - Tick Nominal day 20 - Click 'submit form' - - According to ESMValTool practice, put them in the right rawobsdir folder Notes ----- diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cds_uerra.py b/esmvaltool/cmorizers/data/formatters/datasets/cds_uerra.py index 850775a35d..52c3a1e72e 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/cds_uerra.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/cds_uerra.py @@ -18,7 +18,7 @@ - Tick all available days - Tick all available timesteps - Click 'submit form' - - According to ESMValTool practice, put them in the right rawobsdir folder + - According to ESMValTool practice, put them in the right Tier3/CDS-UERRA folder Notes ----- diff --git a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py index eef3f371e1..8bdb87816e 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py @@ -10,8 +10,7 @@ 20200323 Download and processing instructions - Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given in the configuration) where the raw + Create a new empty directory ``Tier2/CT2019`` where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). diff --git a/esmvaltool/cmorizers/data/formatters/datasets/eppley_vgpm_modis.py b/esmvaltool/cmorizers/data/formatters/datasets/eppley_vgpm_modis.py index f313fa5ed7..ddf5826fbb 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/eppley_vgpm_modis.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/eppley_vgpm_modis.py @@ -10,7 +10,7 @@ Download and processing instructions Download and unpack all the *.tar files under a single directory - (no subdirectories with years) in ${RAWOBS}/Tier2/Eppley-VGPM-MODIS + (no subdirectories with years) in Tier2/Eppley-VGPM-MODIS Modification history 20190515-lovato_tomas: written. diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_biomass.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_biomass.py index 0de5518d33..11366e9a3a 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_biomass.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_biomass.py @@ -12,7 +12,7 @@ Download and processing instructions Download 10 km file: v6.0/netcd/ESACCI-BIOMASS-L4-AGB-MERGED-10000m-fv6.0.nc - Put file in ${RAWOBS}/Tier2/ESACCI-BIOMASS + Put file in Tier2/ESACCI-BIOMASS Or use automatic download script: esmvaltool data download ESACCI-BIOMASS """ diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_landcover.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_landcover.py index 84a976457d..7cdac95cb4 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_landcover.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_landcover.py @@ -13,7 +13,7 @@ Download the data from: pft/v2.0.8/ Put all files under a single directory (no subdirectories with years). - in ${RAWOBS}/Tier2/ESACCI-LANDCOVER + in Tier2/ESACCI-LANDCOVER """ diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index b3f2c913e2..b600ff5210 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -11,7 +11,7 @@ Download and processing instructions Put all files under a single directory (no subdirectories with years) - in ${RAWOBS}/Tier2/ESACCI-LST + in Tier2/ESACCI-LST BOTH DAY and NIGHT files are needed for each month Currently set to work with only the MODIS AQUA L3 monthly data diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_oc.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_oc.py index a76162368e..ac32e6fd2d 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_oc.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_oc.py @@ -14,7 +14,7 @@ In case of issues with data download, check also the information provided at OceanColour webpage https://esa-oceancolour-cci.org/ Put all files under a single directory (no subdirectories with years) - in ${RAWOBS}/Tier2/ESACCI-OC + in Tier2/ESACCI-OC Modification history 20190227-lovato_tomas: written. diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_ozone.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_ozone.py index c077be562c..fc965b6ab3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_ozone.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_ozone.py @@ -36,7 +36,7 @@ Version = "v0008" Put all files under a single directory (no subdirectories with years). - in ${RAWOBS}/Tier2/ESACCI-OZONE + in Tier2/ESACCI-OZONE """ diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py index 581d93581a..312b173975 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py @@ -14,7 +14,7 @@ daily_files/COMBINED/v08.1/ ancillary/v08.1/ Put all files under a single directory (no subdirectories with years). - in ${RAWOBS}/Tier2/ESACCI-SOILMOISTURE + in Tier2/ESACCI-SOILMOISTURE """ diff --git a/esmvaltool/cmorizers/data/formatters/datasets/grace.py b/esmvaltool/cmorizers/data/formatters/datasets/grace.py index 465063b5ee..bb0e8cf08b 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/grace.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/grace.py @@ -16,7 +16,7 @@ - GRCTellus.JPL.200204_202108.GLO.RL06M.MSCNv02CRI.nc - LAND_MASK.CRI.nc - Download the grace months table which holds important information - on data coverage. Save it in the RAWOBSDIR. + on data coverage. Save it in the Tier3/GRACE directory. https://podaac-tools.jpl.nasa.gov/drive/files/allData/gracefo/docs/GRACE_GRACE-FO_Months_RL06.csv - Manually inspect and check the months table @@ -165,7 +165,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" cfg["work_dir"] = cfg_user.work_dir # Pass on some parameters to cfg file - cfg["rawobsdir"] = cfg_user["rootpath"]["RAWOBS"][0] cfg["in_dir"] = in_dir # If it doesn't exist, create it if not os.path.isdir(cfg["work_dir"]): diff --git a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py index 4dcfba81a9..68e5f7d93f 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py @@ -13,8 +13,7 @@ Download and processing instructions Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the - $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the configuration. Apply this procedure to + Tier3/MLS-AURA directory. Apply this procedure to both links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). diff --git a/esmvaltool/cmorizers/data/typing.py b/esmvaltool/cmorizers/data/typing.py index aedf65c3dc..320ca9325e 100644 --- a/esmvaltool/cmorizers/data/typing.py +++ b/esmvaltool/cmorizers/data/typing.py @@ -1,5 +1,6 @@ """Type definitions for CMORizers.""" +import datetime from typing import TypedDict @@ -8,5 +9,5 @@ class DatasetInfo(TypedDict): tier: int source: str - last_access: str + last_access: datetime.datetime info: str diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index 82b9205baf..6b28556dea 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -3,6 +3,7 @@ import contextlib import os import sys +from pathlib import Path import iris import iris.coord_systems @@ -34,9 +35,6 @@ def write_config_file(dirname): config_file = dirname / "config-user.yml" cfg = { "output_dir": str(dirname / "output_dir"), - "rootpath": { - "RAWOBS": str(dirname / "raw_stuff"), - }, "log_level": "debug", } config_file.write_text(yaml.safe_dump(cfg, encoding=None)) @@ -83,7 +81,7 @@ def _create_sample_cube(time_step): return cube -def put_dummy_data(data_path): +def put_dummy_data(data_path: Path) -> None: """Create a small dummy netCDF file to be cmorized.""" data_info = [ # dir_name, file_name_prefix, var_name @@ -106,16 +104,6 @@ def put_dummy_data(data_path): iris.save(gen_cube, file_path) -def check_log_file(log_file, no_data=False): - """Check the cmorization log file.""" - with open(log_file) as log: - if no_data: - msg = "Data for WOA not found" - else: - msg = "Fixing data" - assert any(msg in line for line in log) - - def check_output_exists(output_path): """Check if cmorizer outputted.""" # eg Tier2/WOA/OBS6_WOA_clim_2018_Omon_thetao_200001-200012.nc @@ -154,30 +142,34 @@ def arguments(*args): sys.argv = backup -def test_cmorize_obs_woa_no_data(tmp_path): +def test_cmorize_obs_woa_no_data(tmp_path: Path) -> None: """Test for example run of cmorize_obs command.""" write_config_file(tmp_path) - os.makedirs(os.path.join(tmp_path, "raw_stuff", "Tier2")) - with keep_cwd(), pytest.raises(RuntimeError): - DataCommand().format("WOA", config_dir=str(tmp_path)) - - log_dir = os.path.join(tmp_path, "output_dir") - log_file = os.path.join( - log_dir, - os.listdir(log_dir)[0], - "run", - "main_log.txt", - ) - check_log_file(log_file, no_data=True) - - -def test_cmorize_obs_woa_data(tmp_path): + os.makedirs(os.path.join(tmp_path, "original_data", "Tier2")) + with ( + keep_cwd(), + pytest.raises( + NotADirectoryError, match="Data for dataset 'WOA' not found" + ), + ): + DataCommand().format( + "WOA", + original_data_dir=tmp_path / "original_data", + config_dir=tmp_path, + ) + + +def test_cmorize_obs_woa_data(tmp_path: Path) -> None: """Test for example run of cmorize_obs command.""" write_config_file(tmp_path) - data_path = os.path.join(tmp_path, "raw_stuff", "Tier2", "WOA") + data_path = os.path.join(tmp_path, "original_data", "Tier2", "WOA") put_dummy_data(data_path) with keep_cwd(): - DataCommand().format("WOA", config_dir=str(tmp_path)) + DataCommand().format( + "WOA", + original_data_dir=tmp_path / "original_data", + config_dir=tmp_path, + ) log_dir = os.path.join(tmp_path, "output_dir") log_file = os.path.join( @@ -186,7 +178,7 @@ def test_cmorize_obs_woa_data(tmp_path): "run", "main_log.txt", ) - check_log_file(log_file, no_data=False) + assert "Fixing data" in Path(log_file).read_text(encoding="utf-8") output_path = os.path.join(log_dir, os.listdir(log_dir)[0], "Tier2", "WOA") check_output_exists(output_path) check_conversion(output_path) diff --git a/tests/unit/cmorizers/test_cmorization_interface.py b/tests/unit/cmorizers/test_cmorization_interface.py index d9d8b96d45..ef11060af4 100644 --- a/tests/unit/cmorizers/test_cmorization_interface.py +++ b/tests/unit/cmorizers/test_cmorization_interface.py @@ -46,7 +46,7 @@ def test_formatters_have_required_interface(): def test_downloaders_have_required_interface(): formatters_folder = os.path.dirname(ddt.__file__) arg_names = ( - "config", + "original_data_dir", "dataset", "dataset_info", "start_date", diff --git a/tests/unit/cmorizers/test_datasets_info.py b/tests/unit/cmorizers/test_datasets_info.py index 21193c81fa..1da4542fb3 100644 --- a/tests/unit/cmorizers/test_datasets_info.py +++ b/tests/unit/cmorizers/test_datasets_info.py @@ -5,16 +5,16 @@ import yaml import esmvaltool -from esmvaltool.cmorizers.data.cmorizer import datasets_file +from esmvaltool.cmorizers.data.cmorizer import DATASETS_FILE -yaml_folder = os.path.abspath(os.path.dirname(datasets_file)) +yaml_folder = os.path.abspath(os.path.dirname(DATASETS_FILE)) recipes_folder = os.path.abspath( os.path.join(os.path.dirname(esmvaltool.__file__), "recipes"), ) def test_only_datasets_are_present(): - recipe = yamale.make_data(datasets_file) + recipe = yamale.make_data(DATASETS_FILE) schema = yamale.make_schema( os.path.join(yaml_folder, "datasets_schema.yml"), ) @@ -22,7 +22,7 @@ def test_only_datasets_are_present(): def test_latest_version_format(): - with open(datasets_file) as file: + with open(DATASETS_FILE) as file: cfg = yaml.safe_load(file) for dataset_info in cfg["datasets"].values(): datetime.datetime.strptime( @@ -32,7 +32,7 @@ def test_latest_version_format(): def test_datasets_are_added_to_test_recipe(): - with open(datasets_file) as file: + with open(DATASETS_FILE) as file: cfg = yaml.safe_load(file) recipe_path = os.path.join(recipes_folder, "examples/recipe_check_obs.yml") From c1c8f71fcba93dd8a857bec5e43ab8d28d75cecb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 22 Jan 2026 22:04:38 +0100 Subject: [PATCH 3/6] Undo unwanted change --- .../cmorizers/data/downloaders/datasets/esacci_cloud.py | 8 ++++---- esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py index 1e90444bdb..7eec8cf348 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py @@ -52,13 +52,13 @@ def download_dataset( ) # check if daily data needs to be downloaded - cmor_original_data_dir = read_cmor_config(dataset) - daily_data = cmor_original_data_dir["daily_data"] + cmor_config = read_cmor_config(dataset) + daily_data = cmor_config["daily_data"] if not daily_data: logger.info( 'If daily data needs to be downloaded change "daily_data" in the ' - 'cmor_original_data_dir file to "True" ' - "(esmvaltool/cmorizers/data/cmor_original_data_dir/ESACCI-CLOUD.yml)", + 'cmor_config file to "True" ' + "(esmvaltool/cmorizers/data/cmor_config/ESACCI-CLOUD.yml)", ) # Base paths for L3U (daily data) and L3C (monthly data) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py index 1f25b9a906..08d58ea08c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py @@ -43,12 +43,12 @@ def download_dataset( overwrite=overwrite, ) - cmor_original_data_dir = read_cmor_config(dataset) + cmor_config = read_cmor_config(dataset) raw_path = ( "https://opendata.dwd.de/climate_environment/GPCC/" "full_data_2018/full_data_monthly_{version}.nc.gz" ) - for version in cmor_original_data_dir["attributes"]["version"].values(): + for version in cmor_config["attributes"]["version"].values(): downloader.download_file( raw_path.format(version=version), wget_options=[], From 4493f1e579d6d3b6aafb3ad7207e68009e98075b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 22 Jan 2026 22:49:00 +0100 Subject: [PATCH 4/6] Fix install flag --- esmvaltool/cmorizers/data/cmorizer.py | 46 +++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index 444e97738c..e423181999 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -396,9 +396,49 @@ def format_dataset( logger.error("Formatting failed for dataset %s", dataset) return False if install: - rootpath = self.config["rootpath"] - target_dir = rootpath.get("OBS", rootpath["default"])[0] - target_dir = os.path.join(target_dir, tier, dataset) + if "rootpath" in self.config: + rootpath = self.config["rootpath"] + target_dir = rootpath.get("OBS", rootpath["default"])[0] + target_dir = os.path.join(target_dir, tier, dataset) + else: + attributes = read_cmor_config(dataset)["attributes"] + for attr in ["dataset", "project"]: + if attr not in attributes: + attributes[attr] = attributes[f"{attr}_id"] + msg = "Unable determine install path, please move files manually." + try: + import esmvalcore.io + import esmvalcore.io.local + except ImportError: + logger.warning(msg) + return True + data_sources = esmvalcore.io.load_data_sources( + self.config, + project=attributes["project"], + ) + for data_source in sorted( + data_sources, + key=lambda ds: ds.priority, + ): + if isinstance( + data_source, + esmvalcore.io.local.LocalDataSource, + ): + try: + target_dir = ( + data_source.rootpath + / data_source.dirname_template.format( + **attributes + ) + ) + except KeyError: + pass + else: + break + else: + logger.warning(msg) + return True + if os.path.isdir(target_dir): logger.info( "Automatic installation of dataset %s skipped: " From 558662d7991b10966c96fc0e440c0189ec683c35 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 10:08:10 +0100 Subject: [PATCH 5/6] Fix links in docs --- doc/sphinx/source/develop/dataset.rst | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/doc/sphinx/source/develop/dataset.rst b/doc/sphinx/source/develop/dataset.rst index 8bcbd2f0fb..a81e9e80c8 100644 --- a/doc/sphinx/source/develop/dataset.rst +++ b/doc/sphinx/source/develop/dataset.rst @@ -10,15 +10,14 @@ The following steps are necessary to prepare an observational data set for the use in ESMValTool. | `1. Check if your variable is CMOR standard`_ -| `2. Edit your configuration file`_ -| `3. Store your dataset in the right place`_ -| `3.1 Downloader script (optional)`_ -| `4. Create a cmorizer for the dataset`_ -| `4.1 Cmorizer script written in python`_ -| `4.2 Cmorizer script written in NCL`_ -| `5. Run the cmorizing script`_ -| `6. Naming convention of the observational data files`_ -| `7. Test the cmorized dataset`_ +| `2. Store your dataset in the right place`_ +| `2.1 Downloader script (optional)`_ +| `3. Create a cmorizer for the dataset`_ +| `3.1 Cmorizer script written in python`_ +| `3.2 Cmorizer script written in NCL`_ +| `4. Run the cmorizing script`_ +| `5. Naming convention of the observational data files`_ +| `6. Test the cmorized dataset`_ .. note:: **CMORization as a fix.** As of early 2020, we've started implementing cmorization as @@ -80,7 +79,7 @@ The dataset entry should contain: - the ``info`` that explain how to download the data. Note that these fields should be identical to the content of the header -of the cmorizing script (see Section `4. Create a cmorizer for the dataset`_). +of the cmorizing script (see Section `3. Create a cmorizer for the dataset`_). 2.1 Downloader script (optional) -------------------------------- @@ -267,10 +266,9 @@ does not support it (i.e. because it is provided as a single file). Valid format .. note:: - The output path given in the configuration file is the path where + The ``output_dir`` path given in the :ref:`configuration file ` is the path where your cmorized dataset will be stored. The ESMValTool will create a folder - with the correct tier information - (see Section `2. Edit your configuration file`_) if that tier folder is not + with the correct tier information if that tier folder is not already available, and then a folder named after the dataset. In this folder the cmorized data set will be stored as a NetCDF file. The cmorized dataset will be automatically moved to the correct tier From b1500254d308b74fef7b63a85eb2ddc635e8027d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 11:28:07 +0100 Subject: [PATCH 6/6] Fix style issues --- esmvaltool/cmorizers/data/cmorizer.py | 216 ++++++++++-------- esmvaltool/cmorizers/data/downloaders/cds.py | 12 +- .../cmorizers/data/downloaders/downloader.py | 3 +- esmvaltool/cmorizers/data/downloaders/ftp.py | 3 +- 4 files changed, 134 insertions(+), 100 deletions(-) mode change 100755 => 100644 esmvaltool/cmorizers/data/cmorizer.py diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py old mode 100755 new mode 100644 index e423181999..2ff971d963 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -271,14 +271,18 @@ def format( ) logger.info("Processing datasets %s", datasets) - failed_datasets = [] with get_distributed_client(): # loop through tier/datasets to be cmorized - for dataset in datasets: + failed_datasets = [ + dataset + for dataset in datasets if not self.format_dataset( - dataset, start, end, install=install - ): - failed_datasets.append(dataset) + dataset, + start, + end, + install=install, + ) + ] if failed_datasets: msg = f"Format failed for datasets {' '.join(failed_datasets)}" @@ -343,8 +347,8 @@ def format_dataset( If True, automatically moves the data to the final location if there is no data there. """ - reformat_script_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), + reformat_script_root = Path( + Path(__file__).parent.resolve(), "formatters", "datasets", self._dataset_to_module(dataset), @@ -352,25 +356,25 @@ def format_dataset( tier = self._get_dataset_tier(dataset) # in-data dir; build out-dir tree - in_data_dir = os.path.join(self.original_data_dir, tier, dataset) + in_data_dir = self.original_data_dir / tier / dataset logger.info("Input data from: %s", in_data_dir) - if not os.path.isdir(in_data_dir): + if not in_data_dir.is_dir(): msg = ( f"Data for dataset '{dataset}' not found. " f"Path to original data '{in_data_dir}' is not a directory'" ) raise NotADirectoryError(msg) - out_data_dir = os.path.join(self.output_dir, tier, dataset) + out_data_dir = self.output_dir / tier / dataset logger.info("Output will be written to: %s", out_data_dir) - if not os.path.isdir(out_data_dir): - os.makedirs(out_data_dir) + if not out_data_dir.is_dir(): + out_data_dir.mkdir(parents=True) # all operations are done in the working dir now os.chdir(out_data_dir) # figure out what language the script is in logger.info("Reformat script: %s", reformat_script_root) - if os.path.isfile(reformat_script_root + ".ncl"): - reformat_script = reformat_script_root + ".ncl" + if reformat_script_root.with_suffix(".ncl").is_file(): + reformat_script = reformat_script_root.with_suffix(".ncl") success = self._run_ncl_script( in_data_dir, out_data_dir, @@ -379,7 +383,7 @@ def format_dataset( start, end, ) - elif os.path.isfile(reformat_script_root + ".py"): + elif reformat_script_root.with_suffix(".py").is_file(): success = self._run_pyt_script( in_data_dir, out_data_dir, @@ -396,50 +400,15 @@ def format_dataset( logger.error("Formatting failed for dataset %s", dataset) return False if install: - if "rootpath" in self.config: - rootpath = self.config["rootpath"] - target_dir = rootpath.get("OBS", rootpath["default"])[0] - target_dir = os.path.join(target_dir, tier, dataset) - else: - attributes = read_cmor_config(dataset)["attributes"] - for attr in ["dataset", "project"]: - if attr not in attributes: - attributes[attr] = attributes[f"{attr}_id"] - msg = "Unable determine install path, please move files manually." - try: - import esmvalcore.io - import esmvalcore.io.local - except ImportError: - logger.warning(msg) - return True - data_sources = esmvalcore.io.load_data_sources( - self.config, - project=attributes["project"], + target_dir = self._get_install_dir(dataset) + if target_dir is None: + logger.warning( + "Unable determine install path for dataset '%s', please " + "move files from %s to the desired location manually.", + dataset, + out_data_dir, ) - for data_source in sorted( - data_sources, - key=lambda ds: ds.priority, - ): - if isinstance( - data_source, - esmvalcore.io.local.LocalDataSource, - ): - try: - target_dir = ( - data_source.rootpath - / data_source.dirname_template.format( - **attributes - ) - ) - except KeyError: - pass - else: - break - else: - logger.warning(msg) - return True - - if os.path.isdir(target_dir): + elif target_dir.is_dir(): logger.info( "Automatic installation of dataset %s skipped: " "target folder %s already exists", @@ -455,27 +424,90 @@ def format_dataset( shutil.move(out_data_dir, target_dir) return True + def _get_install_dir(self, dataset) -> Path | None: + """Get the installation directory for a dataset. + + Parameters + ---------- + dataset: + Dataset name. + + Returns + ------- + : + Path to the installation directory, or None if it cannot be + determined. + """ + tier = self._get_dataset_tier(dataset) + if "rootpath" in self.config: + # May not be available since ESMValCore v2.14, will be removed in + # ESMValCore v2.16. + rootpath = self.config["rootpath"] + target_dir = rootpath.get("OBS", rootpath["default"])[0] + return Path(target_dir, tier, dataset) + + try: + # Only available since ESMValCore v2.14 + import esmvalcore.io + import esmvalcore.io.local + except ImportError: + return None + + target_dir = None + # Normalize the attributes so they can be used as facets in the + # directory name template of a LocalDataSource. + try: + attributes = read_cmor_config(dataset)["attributes"] + except FileNotFoundError: + # NCL scripts do not use a cmor config file. + return None + for attr in ["dataset", "project"]: + if attr not in attributes: + attributes[attr] = attributes[f"{attr}_id"] + # Load data sources from configuration. + data_sources = esmvalcore.io.load_data_sources( + self.config, + project=attributes["project"], + ) + # Loop over potential target directories and try if the right attributes + # are available to format the directory name template. Use the first + # one that works. + for data_source in sorted( + data_sources, + key=lambda ds: ds.priority, + ): + if isinstance( + data_source, + esmvalcore.io.local.LocalDataSource, + ): + try: + target_dir = ( + data_source.rootpath + / data_source.dirname_template.format( + **attributes, + ) + ) + except KeyError: + pass + else: + break + return target_dir + def _get_dataset_tier(self, dataset: str) -> str: return f"Tier{self.datasets_info['datasets'][dataset]['tier']}" def _write_ncl_settings( self, - project_info, - dataset, - run_dir, - reformat_script, - start_year, - end_year, - ): + project_info: dict[str, dict[str, str]], + dataset: str, + run_dir: Path, + reformat_script: Path, + start: datetime.datetime | None, + end: datetime.datetime | None, + ) -> Path: """Write the information needed by the ncl reformat script.""" - if start_year is None: - start_year = 0 - else: - start_year = start_year.year - if end_year is None: - end_year = 0 - else: - end_year = end_year.year + start_year = 0 if start is None else start.year + end_year = 0 if end is None else end.year settings = { "cmorization_script": reformat_script, "input_dir_path": project_info[dataset]["indir"], @@ -486,19 +518,18 @@ def _write_ncl_settings( "start_year": start_year, "end_year": end_year, } - settings_filename = os.path.join(run_dir, dataset, "settings.ncl") - if not os.path.isdir(os.path.join(run_dir, dataset)): - os.makedirs(os.path.join(run_dir, dataset)) + settings_filename = run_dir / dataset / "settings.ncl" + (run_dir / dataset).mkdir(parents=True, exist_ok=True) # write the settings file write_ncl_settings(settings, settings_filename) return settings_filename def _run_ncl_script( self, - in_dir: str, - out_dir: str, + in_dir: Path, + out_dir: Path, dataset: str, - script: str, + script: Path, start: datetime.datetime | None, end: datetime.datetime | None, ) -> bool: @@ -510,29 +541,27 @@ def _run_ncl_script( ) project = {} project[dataset] = {} - project[dataset]["indir"] = in_dir - project[dataset]["outdir"] = out_dir + project[dataset]["indir"] = str(in_dir) + project[dataset]["outdir"] = str(out_dir) settings_file = self._write_ncl_settings( - project, - dataset, - self.run_dir, - script, - start, - end, + project_info=project, + dataset=dataset, + run_dir=self.run_dir, + reformat_script=script, + start=start, + end=end, ) # put settings in environment env = dict(os.environ) - env["settings"] = settings_file - env["esmvaltool_root"] = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(script))), - ) + env["settings"] = str(settings_file) + env["esmvaltool_root"] = str(script.parents[3]) env["cmor_tables"] = str( Path(esmvalcore.cmor.__file__).parent / "tables", ) logger.info("Using CMOR tables at %s", env["cmor_tables"]) # call NCL - ncl_call = ["ncl", script] + ncl_call = ["ncl", str(script)] logger.info("Executing cmd: %s", " ".join(ncl_call)) with subprocess.Popen( ncl_call, @@ -620,6 +649,7 @@ def info(self, dataset: str) -> None: def download( self, datasets: str | list[str], + *, original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, @@ -664,6 +694,7 @@ def download( def format( self, datasets: str | list[str], + *, original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, @@ -708,6 +739,7 @@ def format( def prepare( self, datasets: str | list[str], + *, original_data_dir: Path | None = None, start: str | None = None, end: str | None = None, diff --git a/esmvaltool/cmorizers/data/downloaders/cds.py b/esmvaltool/cmorizers/data/downloaders/cds.py index 59f28d227b..bdb32606ae 100644 --- a/esmvaltool/cmorizers/data/downloaders/cds.py +++ b/esmvaltool/cmorizers/data/downloaders/cds.py @@ -5,7 +5,6 @@ import logging import os from collections.abc import Iterable -from pathlib import Path from typing import TYPE_CHECKING import cdsapi @@ -13,6 +12,8 @@ from .downloader import BaseDownloader if TYPE_CHECKING: + from pathlib import Path + from esmvaltool.cmorizers.data.typing import DatasetInfo logger = logging.getLogger(__name__) @@ -62,7 +63,7 @@ def __init__( self._client = cdsapi.Client(url=cds_url) except Exception as ex: if str(ex).endswith(".cdsapirc"): - logger.error( + logger.exception( "Could not connect to the CDS due to issues with your " '".cdsapirc" file. More info in ' "https://cds.climate.copernicus.eu/api-how-to.", @@ -105,9 +106,8 @@ def download( request_dict["day"] = f"{day:02d}" date_str = f"{year}{month:02d}" - if day: - if not isinstance(day, Iterable): - date_str += f"{day:02d}" + if day and not isinstance(day, Iterable): + date_str += f"{day:02d}" os.makedirs(self.local_folder, exist_ok=True) if file_pattern is None: @@ -168,5 +168,5 @@ def download_request(self, filename, request=None): filename, ) except Exception: - logger.error("Failed request: %s", request) + logger.exception("Failed request: %s", request) raise diff --git a/esmvaltool/cmorizers/data/downloaders/downloader.py b/esmvaltool/cmorizers/data/downloaders/downloader.py index a84c4a622b..056a8e9929 100644 --- a/esmvaltool/cmorizers/data/downloaders/downloader.py +++ b/esmvaltool/cmorizers/data/downloaders/downloader.py @@ -2,10 +2,11 @@ from __future__ import annotations -from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: + from pathlib import Path + from esmvaltool.cmorizers.data.typing import DatasetInfo diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index d57c22d725..51f4202fa4 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -6,7 +6,6 @@ import logging import os import re -from pathlib import Path from typing import TYPE_CHECKING from progressbar import ( @@ -21,6 +20,8 @@ from .downloader import BaseDownloader if TYPE_CHECKING: + from pathlib import Path + from esmvaltool.cmorizers.data.typing import DatasetInfo