Update docker recipe

.circleci/config.yml

@@ -20,29 +20,12 @@ jobs:
     # Run tests
     working_directory: /test
     docker:
-      - image: continuumio/miniconda3
+      - image: esmvalgroup/esmvalcore:development
     steps:
       - checkout
       - run:
           command: |
-            # Create a file to checksum as cache key
-            date --rfc-3339 date > cache_key.txt
-            cat environment.yml >> cache_key.txt
-      - restore_cache:
-          key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-      - run:
-          # Update/Create Conda environment and run tests
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            # conda update -y conda
-            conda env update
-            conda activate esmvaltool
             python setup.py test --addopts '-m "not installation"'
-      - save_cache:
-          key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-          paths:
-            - "/opt/conda/envs/esmvaltool"
-            - ".eggs"
       - store_test_results:
           path: test-reports/
       - store_artifacts:
@@ -137,14 +120,14 @@ jobs:
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            set -x
             mkdir /logs
             # Install
             # conda update -y conda > /logs/conda.txt 2>&1
             conda env update >> /logs/conda.txt 2>&1
-            set +x; conda activate esmvaltool; set -x
-            conda install -yS r-base r-yaml ncl -c conda-forge
+            conda activate esmvaltool
             pip install -e .[develop] > /logs/install.txt 2>&1
+            # install additional requirements for running all tests
+            conda install -yS r-base r-yaml ncl -c conda-forge
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml

doc/quickstart/install.rst

@@ -16,6 +16,71 @@ Once you have installed conda, you can install ESMValCore by running:
     conda install -c esmvalgroup -c conda-forge esmvalcore
 
 
+Using it through Docker
+-----------------------
+
+ESMValCore is also provided thorugh DockerHub in the form of docker containers.
+See https://docs.docker.com/ for more info about docker containers and how to
+run the
+
+You can get the latest release with
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvalcore:stable
+
+If yoy want to use the current master branch, use
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvalcore:latest
+
+To run a container using those images, use:
+
+.. code-block:: bash
+
+   docker run esmvalgroup/esmvalcore:stable esmvaltool -v
+
+.. warning::
+
+   The container does not see the data available in the host by default.
+   You can make host data available with `-v /path:/path/in/container`
+
+Using it through Singularity
+----------------------------
+
+Docker is usually forbidden in clusters due to security reasons. However, 
+there is a more secure alternative to run containers that is usually available 
+on them: Singularity.
+
+Singularity can use docker containers directly from DockerHub with the
+following command
+
+.. code-block:: bash
+
+   singularity run docker://esmvalgroup/esmvalcore:${DESIRED_TAG} -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE}
+
+.. warning::
+
+   As with docker, The container does not see the data available in the host by default.
+   You can make host data available with `-B /path:/path/in/container`
+
+Some clusters may not allow to connect to external services, in those cases
+you can create a singularity image locally that can later be uploaded to the cluster.
+
+.. code-block:: bash
+
+   singularity build esmvalcore_${TAG}.sif docker://esmvalgroup/esmvalcore:${TAG}
+
+To run the container using the newly created image file use:
+
+.. code-block:: bash
+
+   singularity run esmvalcore_${TAG}.sif -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE}
+
+
+
+
 Development installation
 ------------------------
 

docker/Dockerfile

@@ -1,16 +1,19 @@
+# To build this container, go to ESMValCore root folder and execute :
+# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile
 FROM continuumio/miniconda3
 
 # update the conda packages
 RUN conda update -y conda pip
 
-# install development tools
-RUN apt-get update -y && apt-get install -y \
-    build-essential \
-    curl \
-    unzip
 
 # install environment packages
-RUN conda install -c conda-forge -c esmvalgroup -c birdhouse esmvaltool
+COPY . /home/root/source
+WORKDIR /home/root/source
+RUN ls
+RUN conda env update --name base --file environment.yml
+RUN pip install .
+RUN conda clean --all -y
+RUN rm -r /home/root/source
 
 # run tests
 RUN esmvaltool -h

docker/Dockerfile.dev

@@ -0,0 +1,20 @@
+# To build this container, go to ESMValCore root folder and execute :
+# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev
+FROM continuumio/miniconda3
+
+# update the conda packages
+RUN conda update -y conda pip
+
+# Copy source
+COPY . /home/root/source
+WORKDIR /home/root/source
+
+# Install
+RUN apt-get update
+RUN conda env update --name base --file environment.yml
+RUN pip install -e .[test]
+RUN pip uninstall esmvalcore -y
+
+# Clean up
+RUN rm -r /home/root/source
+RUN conda clean --all -y

environment.yml

@@ -4,10 +4,10 @@ channels:
   - conda-forge
 
 dependencies:
+  - compilers
   - esmpy
   - iris>=2.2.1
   - graphviz
-  - gcc_linux-64
   - libunwind  # Needed for Python3.7+
   - python>=3.6
   - python-stratify