Add an interface for adding new data sources and add support for intake-esgf as a first example

.circleci/config.yml

@@ -66,7 +66,7 @@ commands:
       - run:
           name: Install git+ssh
           environment:
-            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
           command: apt update && apt install -y git ssh
       - checkout
       - check_changes
@@ -141,6 +141,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             conda activate esmvaltool
+            pip install intake-esgf "globus-sdk<4"  # TODO: remove after merge
             pip install --no-deps .[test] > /logs/install.txt 2>&1
             pip check
       - test_and_report:
@@ -155,7 +156,7 @@ jobs:
           name: Install gpg (required by codecov orb)
           command: apt update && apt install -y gpg
       - codecov/upload:
-          files: 'test-reports/coverage.xml'
+          files: "test-reports/coverage.xml"
           disable_search: true
 
   test_installation_from_source_test_mode:

.git-blame-ignore-revs

@@ -1,2 +1,7 @@
 # Use ruff formatter (#2524)
 436558caacda69d4966a5aff35959ce9188cac37
+# Enable more ruff rules (#2715)
+77c370314bac1e72400392ed98c8f2f75b1b5a98
+02583113c99304e4e99bd010a05856ea429259ea
+0e8e7164ef96c574f77367948f51c7ab822bb694
+e2f8cab2cc5a509452ca720e50f041f46f380ee2

.gitignore

@@ -1,5 +1,6 @@
 # Autogenerated files
 _sidebar.rst.inc
+jupyter_execute/
 
 # Distribution / packaging
 .Python

doc/api/esmvalcore.esgf.rst

@@ -1,18 +1,10 @@
 Find and download files from ESGF
 =================================
 
-This module provides the function :py:func:`esmvalcore.esgf.find_files`
-for searching for files on ESGF using the ESMValTool vocabulary.
-It returns :py:class:`esmvalcore.esgf.ESGFFile` objects, which have a convenient
-:py:meth:`esmvalcore.esgf.ESGFFile.download` method for downloading the files.
-
-See :ref:`config-esgf` for instructions on configuring this module.
-
 esmvalcore.esgf
 ---------------
-.. autofunction:: esmvalcore.esgf.find_files
-.. autofunction:: esmvalcore.esgf.download
-.. autoclass:: esmvalcore.esgf.ESGFFile
+.. automodule:: esmvalcore.esgf
+   :no-inherited-members:
 
 esmvalcore.esgf.facets
 ----------------------

doc/api/esmvalcore.io.intake_esgf.rst

@@ -0,0 +1,5 @@
+esmvalcore.io.intake_esgf
+=========================
+
+.. automodule:: esmvalcore.io.intake_esgf
+    :no-inherited-members:

doc/api/esmvalcore.io.protocol.rst

@@ -0,0 +1,5 @@
+esmvalcore.io.protocol
+======================
+
+.. automodule:: esmvalcore.io.protocol
+    :no-inherited-members:

doc/api/esmvalcore.io.rst

@@ -0,0 +1,18 @@
+Access data from any source
+===========================
+
+ESMValCore supports a modular system for reading data from various data sources.
+In the future, this module may be extended with support for writing output data.
+
+The interface is defined in the :mod:`esmvalcore.io.protocol` module and
+the other modules here provide an implementation for a particular data source.
+
+.. toctree::
+   :maxdepth: 1
+
+   esmvalcore.io.protocol
+   esmvalcore.io.intake_esgf
+
+esmvalcore.io
+-------------
+.. automodule:: esmvalcore.io

doc/api/esmvalcore.rst

@@ -14,6 +14,7 @@ library. This section documents the public API of ESMValCore.
    esmvalcore.dataset
    esmvalcore.esgf
    esmvalcore.exceptions
+   esmvalcore.io
    esmvalcore.iris_helpers
    esmvalcore.local
    esmvalcore.preprocessor

doc/conf.py

@@ -466,6 +466,7 @@
     'dask': ('https://docs.dask.org/en/stable/', None),
     'distributed': ('https://distributed.dask.org/en/stable/', None),
     'iris': ('https://scitools-iris.readthedocs.io/en/stable/', None),
+    'intake_esgf': ('https://intake-esgf.readthedocs.io/en/stable/', None),
     'esmf_regrid': ('https://iris-esmf-regrid.readthedocs.io/en/stable/', None),
     'matplotlib': ('https://matplotlib.org/stable/', None),
     'ncdata': ('https://ncdata.readthedocs.io/en/stable/', None),

doc/configurations

@@ -0,0 +1 @@
+../esmvalcore/config/configurations