Update ESACCI Cloud CMORizer (daily and monthly data)

doc/sphinx/source/input.rst

@@ -311,7 +311,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ESACCI-AEROSOL               | abs550aer, od550aer, od550aerStderr, od550lt1aer, od870aer, od870aerStderr (aero)                    |   2  | NCL             |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
-| ESACCI-CLOUD                 | clivi, clt, cltStderr, clwvi, lwp, rlut, rlutcs, rsut, rsutcs, rsdt, rlus, rsus, rsuscs (Amon)       |   2  | NCL             |
+| ESACCI-CLOUD                 | clivi, clt, cltStderr, clwvi, lwp, rlut, rlutcs, rsut, rsutcs, rsdt, rlus, rsus, rsuscs (Amon),      |   2  | Python          |
+|                              | clt, clwvi, cod (day)                                                                                |   2  |                 |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ESACCI-FIRE                  | burntArea (Lmon)                                                                                     |   2  | NCL             |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+

esmvaltool/cmorizers/data/cmor_config/ESACCI-CLOUD.yml

@@ -0,0 +1,122 @@
+# CMORIZE ESA CCI CLOUD daily/monthly data
+---
+start_year_monthly: 1982
+end_year_monthly: 2016
+start_year_daily: 2003
+end_year_daily: 2007
+
+daily_data: False
+
+# Common global attributes for Cmorizer output
+attributes:
+  dataset_id: ESACCI-CLOUD
+  version: 'v3.0-AVHRR'
+  tier: 2
+  project_id: OBS6
+  source: 'ESA CCI'
+  modeling_realm: sat
+  reference: 'esacci_cloud'
+  comment: ''
+
+
+# Variables to cmorize
+variables:
+  # daily data
+  clt_day:
+    short_name: clt
+    mip: day
+    raw: [cmask_desc, cmask_asc]
+    raw_units: '1'
+    file: '-ESACCI-L3U_CLOUD-CLD_MASKTYPE-AVHRR_*-fv3.0.nc'
+  clwvi_day:
+    short_name: clwvi
+    mip: CFday
+    raw: [cwp_desc, cwp_asc]
+    raw_units: g/m2
+    file: '-ESACCI-L3U_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  cod_day:
+    short_name: cod
+    mip: day
+    raw: [cot_desc, cot_asc]
+    raw_units: 1
+    file: '-ESACCI-L3U_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+
+  # monthly data
+  clt_mon:
+    short_name: clt
+    mip: Amon
+    raw: cfc
+    raw_units: '1'
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  cltStderr_mon:
+    short_name: cltStderr
+    mip: Amon
+    raw: cfc_unc
+    raw_units: '%'
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  lwp_mon:
+    short_name: lwp
+    mip: Amon
+    raw: lwp_allsky
+    raw_units: g/m2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  clivi_mon:
+    short_name: clivi
+    mip: Amon
+    raw: iwp_allsky
+    raw_units: g/m2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  clwvi_mon:
+    short_name: clwvi
+    mip: Amon
+    raw: iwp_allsky
+    raw_units: g/m2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rlut_mon:
+    short_name: rlut
+    mip: Amon
+    raw: toa_lwup
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rlutcs_mon:
+    short_name: rlutcs
+    mip: Amon
+    raw: toa_lwup_clr
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rsut_mon:
+    short_name: rsut
+    mip: Amon
+    raw: toa_swup
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rsutcs_mon:
+    short_name: rsutcs
+    mip: Amon
+    raw: toa_swup_clr
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rsdt_mon:
+    short_name: rsdt
+    mip: Amon
+    raw: toa_swdn
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rlus_mon:
+    short_name: rlus
+    mip: Amon
+    raw: boa_lwup
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rsus_mon:
+    short_name: rsus
+    mip: Amon
+    raw: boa_swup
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'
+  rsuscs_mon:
+    short_name: rsuscs
+    mip: Amon
+    raw: boa_swup_clr
+    raw_units: W m-2
+    file: '-ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_*-fv3.0.nc'

esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py

@@ -1,10 +1,14 @@
-"""Script to download ESACCI-CLOUD."""
+"""Script to download daily and monthly ESACCI-CLOUD data."""
 
+import logging
 from datetime import datetime
 
 from dateutil import relativedelta
 
-from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
+from esmvaltool.cmorizers.data.utilities import read_cmor_config
+
+logger = logging.getLogger(__name__)
 
 
 def download_dataset(
@@ -27,40 +31,129 @@ def download_dataset(
     overwrite : bool
         Overwrite already downloaded files
     """
+
+    start_date_day = False
     if start_date is None:
         start_date = datetime(1982, 1, 1)
+        start_date_day = datetime(2003, 1, 1)
     if end_date is None:
-        end_date = datetime(2016, 1, 1)
+        end_date = datetime(2016, 12, 31)
     loop_date = start_date
 
-    downloader = CCIDownloader(
+    downloader = WGetDownloader(
         config=config,
         dataset=dataset,
         dataset_info=dataset_info,
         overwrite=overwrite,
     )
-    downloader.connect()
-    end_of_file = "ESACCI-L3C_CLOUD-CLD_PRODUCTS-AVHRR_NOAA-12-fv3.0.nc"
-    filler_data = {
-        1994: [
-            f"AVHRR_NOAA_12/1994/199409-{end_of_file}",
-            f"AVHRR_NOAA_12/1994/199410-{end_of_file}",
-            f"AVHRR_NOAA_12/1994/199411-{end_of_file}",
-            f"AVHRR_NOAA_12/1994/199412-{end_of_file}",
-        ],
-        1995: [
-            f"AVHRR_NOAA_12/1995/199501-{end_of_file}",
-        ],
-    }
+
+    # check if daily data needs to be downloaded
+    cmor_config = read_cmor_config(dataset)
+    daily_data = cmor_config["daily_data"]
+    if not daily_data:
+        logger.info(
+            'If daily data needs to be downloaded change "daily_data" in the '
+            'cmor_config file to "True" '
+            "(esmvaltool/cmorizers/data/cmor_config/ESACCI-CLOUD.yml)"
+        )
+
+    # Base paths for L3U (daily data) and L3C (monthly data)
+    base_path_l3u = (
+        "https://public.satproj.klima.dwd.de/data/ESA_Cloud_CCI/"
+        "CLD_PRODUCTS/v3.0/L3U/"
+    )
+    base_path_l3c = (
+        "https://public.satproj.klima.dwd.de/data/ESA_Cloud_CCI/"
+        "CLD_PRODUCTS/v3.0/L3C/"
+    )
+
+    wget_options = [
+        "-r",
+        "-e robots=off",  # Ignore robots.txt
+        "--no-parent",  # Don't ascend to the parent directory
+        '--reject="index.html"',  # Reject any HTML files
+    ]
 
     while loop_date <= end_date:
         year = loop_date.year
-        downloader.set_cwd("version3/L3C/AVHRR-PM/v3.0")
-        for folder in downloader.list_folders():
-            for year_folder in downloader.list_folders(folder):
-                if int(year_folder) == year:
-                    downloader.download_year(f"{folder}/{year_folder}")
-        downloader.set_cwd("version3/L3C/AVHRR-AM/v3.0")
-        for extra_file in filler_data.get(year, []):
-            downloader.download_file(extra_file)
-        loop_date += relativedelta.relativedelta(years=1)
+        month = loop_date.month
+        date = f"{year}{month:02}"
+
+        if datetime(1982, 1, 1) <= loop_date < datetime(1985, 2, 1):
+            sat_am = ""
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-7/"
+        elif datetime(1985, 2, 1) <= loop_date < datetime(1988, 11, 1):
+            sat_am = ""
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-9/"
+        elif datetime(1988, 11, 1) <= loop_date < datetime(1991, 9, 1):
+            sat_am = ""
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-11/"
+        elif datetime(1991, 9, 1) <= loop_date < datetime(1994, 9, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-12/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-11/"
+        elif datetime(1994, 9, 1) <= loop_date < datetime(1995, 2, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-12/"
+            sat_pm = ""
+        elif datetime(1995, 2, 1) <= loop_date < datetime(1999, 1, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-12/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-14/"
+        elif datetime(1999, 1, 1) <= loop_date < datetime(2001, 4, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-15/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-14/"
+        elif datetime(2001, 4, 1) <= loop_date < datetime(2002, 11, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-15/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-16/"
+        elif datetime(2002, 11, 1) <= loop_date < datetime(2005, 9, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-17/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-16/"
+        elif datetime(2005, 9, 1) <= loop_date < datetime(2007, 7, 1):
+            sat_am = "AVHRR-AM/AVHRR_NOAA-17/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-18/"
+        elif datetime(2007, 7, 1) <= loop_date < datetime(2009, 6, 1):
+            sat_am = "AVHRR-AM/AVHRR_METOPA/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-18/"
+        elif datetime(2009, 6, 1) <= loop_date < datetime(2017, 1, 1):
+            sat_am = "AVHRR-AM/AVHRR_METOPA/"
+            sat_pm = "AVHRR-PM/AVHRR_NOAA-19/"
+        else:
+            msg = f"Data for this date {date} is not available"
+            raise ValueError(msg)
+
+        # Download monthly data from L3C
+        for sat in (sat_am, sat_pm):
+            if sat != "":
+                # monthly data
+                logger.info("Downloading monthly data (L3C) for sat = %s", sat)
+                folder_l3c = base_path_l3c + sat + f"{year}/"
+                wget_options_l3c = wget_options.copy()
+                wget_options_l3c.append(f"--accept={date}*.nc")
+                logger.info(
+                    "Download folder for monthly data (L3C): %s", folder_l3c
+                )
+                downloader.download_file(folder_l3c, wget_options_l3c)
+
+                # daily data
+                if daily_data:
+                    if not start_date_day or (
+                        start_date_day
+                        and datetime(2003, 1, 1)
+                        <= loop_date
+                        <= datetime(2007, 2, 1)
+                    ):
+                        logger.info(
+                            "Downloading daily data (L3U) for sat = %s", sat
+                        )
+                        folder_l3u = base_path_l3u + sat + f"{year}/{month:02}"
+                        wget_options_l3u = wget_options.copy()
+                        wget_options_l3u.append(
+                            f"--accept={date}*CLD_MASKTYPE*.nc,"
+                            f"{date}*CLD_PRODUCTS*.nc"
+                        )
+                        logger.info(
+                            "Download folder for daily data (L3U): %s",
+                            folder_l3u,
+                        )
+                        downloader.download_file(folder_l3u, wget_options_l3u)
+
+        # Increment the loop_date by one month
+        loop_date += relativedelta.relativedelta(months=1)