Merge pull request #62 from C2SM/v3.1-rc

Processing Chain v3.1

.github/workflows/docs-latest.yml

@@ -8,18 +8,20 @@ on:
       - main
 
 jobs:
-  deploy-docs:
+  deploy-docs-latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
-      - uses: peaceiris/actions-gh-pages@v3
+
+      - name: Deploy on GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build

.github/workflows/docs-pr-preview.yml

@@ -1,7 +1,9 @@
-name: Deploy PR previews
+name: Build and Deploy Documentation to PR Previews
 
 on:
   pull_request:
+    paths:
+      - 'docs/**'
     types:
       - opened
       - reopened
@@ -16,10 +18,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
 

.github/workflows/docs-tag.yml

@@ -6,23 +6,26 @@ on:
       - '*'
 
 jobs:
-  deploy-docs:
+  deploy-docs-tag:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Get release
         id: get_release
         uses: bruceadams/[email protected]
         env: 
           GITHUB_TOKEN: ${{ github.token }}
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
-      - uses: peaceiris/actions-gh-pages@v3
+
+      - name: Deploy on GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build

.gitignore

@@ -1,10 +1,12 @@
 docs/build/
+docs/_build/
 *__pycache__/
 *.pyc
 fieldextra.diagnostic
 input_processing-chain.tgz
 input/
 output/
 work/
-src/*/
+ext/*/
 *.code-workspace
+.vscode/

README.md

@@ -1,4 +1,4 @@
-# Processing Chain for COSMO and ICON Simulations
+# <img src="https://polybox.ethz.ch/index.php/s/yc3zMmoXKyI2rJm/download" width="64" valign="middle" alt="Processing Chain"/> Processing Chain
 
 The Processing Chain is a python script that prepares necessary input
 data, submits compute-jobs to the queue on Piz Daint and does
@@ -9,71 +9,19 @@ e.g., by creating your own case or adding new jobs.
 
 ## Environment Setup
 
-The following steps allow you to create and use your own virtual
-environment to run the Processing Chain. We recommend to use a conda
-environment for the usage of the provided scripts. Please follow the
-instruction for the installation. The following steps only need to be 
-performed once.
-
-### 1\. Install Miniconda
-
-Install as user specific Miniconda, e.g. on your `$HOME` directory,
-which is the default location.
-
-> **Note**: Only conda itself should be installed in your `$HOME`.
-> All environments should be stored in your `$PROJECT` directory,
-> otherwise you risk filling up your `$HOME` directory. See below for instructions.
-
-To install the latest Miniconda, type:
-
-    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-    bash Miniconda3-latest-Linux-x86_64.sh
-
-Further deails on Miniconda can be found on the [Miniconda documentation page](https://docs.conda.io/en/latest/miniconda.html).
-
-### 2\. Create the Conda Environment
-
-Create a conda environment `proc-chain` with and install requirements:
-
-    conda env create --prefix $PROJECT/envs/proc-chain -f env/environment.yml
-
-To be able to activate your conda environment by simply using `conda activate proc-chain` instead of the full path, add the following to your `.bashrc`:
-
-    export CONDA_ENVS_PATH=$PROJECT/envs
-
-Activate the environment (use "source activate" in case "conda activate"
-does not work):
-
-    conda activate proc-chain
-
-If you already have the environment but want to update it:
-
-    conda env update --file env/environment.yml --prune
-
-### 3\. Store user-specific data
-
-To register your email address and standard project account, store them into
-these files within your home directory:
-
-    echo <your_account_id> > ~/.acct
-    echo <your_email_address> > ~/.forward
-
-These settings are optional. The Processing Chain will first check the content
-of those files. If desired, the corresponding variables can be overridden by setting
-the `compute_account` and `user_mail` variables in the `config.yaml` file.
+To setup your conda environment for the Processing Chain, please refer
+to the part in the [official documentation](https://c2sm.github.io/processing-chain/latest/environment.html).
 
 ## Run the Chain
 
-Once everything has been set up correctly according to the above steps,
-you just need to execute the following command to activate your
-environment (if not done already):
+To activate your conda environment, type:
 
     conda activate proc-chain
 
 To test if your environment has been successfully set, use the command
 line help to display the available arguments for the main script:
 
-    python run_chain.py -h
+    ./run_chain.py -h
 
 To run the test cases with their standard jobs, please ensure
 that you clone the Processing Chain to `$SCRATCH`, as input and
@@ -89,9 +37,8 @@ For these pre-defined test cases, you can use the Jenkins script
 
     ./jenkins/scripts/jenkins.sh
 
-This script calls other scripts that are located in `jenkins/scripts/`. 
-They will
-- activate the conda environment (if not done already)
+This script calls other scripts that are located in `jenkins/scripts/` and will: 
+- activate the conda environment
 - setup spack-c2sm
 - download input data to `input/`
 - build `int2lm`, `cosmo-ghg`, `icon` and `icon-art`
@@ -104,16 +51,14 @@ They will
 
 To run the test cases manually, type:
 
-```bash
     # replace <casename> with one of the above tests
-    python run_chain.py <casename>
-```
+    ./run_chain.py <casename>
 
 ## Documentation
 
 For more information about the file structure, configuration options,
-namelist templates etc., please read the official
-[documentation](https://c2sm.github.io/processing-chain/).
+namelist templates etc., please read the [official
+documentation](https://c2sm.github.io/processing-chain/latest/).
 
 ## Contributing
 

cases/cosmo-ghg-spinup-test/config.yaml

@@ -1,10 +1,11 @@
 # Configuration file for the 'cosmo-ghg-spinup-test' case with COSMO-GHG
 
-model: cosmo-ghg
+workflow: cosmo-ghg-spinup
 constraint: gpu
+run_on: gpu
+compute_queue: normal
 ntasks_per_node: 12
 restart_step: PT6H
-variant: spinup
 spinup: 3
 startdate: 2015-01-01T00:00:00Z
 enddate: 2015-01-01T18:00:00Z
@@ -47,7 +48,7 @@ online_vprm:
 int2lm:
     extpar_dir: ./input/cosmo-ghg/extpar
     extpar_filename: test_domain.nc
-    binary_file: ./src/int2lm/test/testsuite/int2lm
+    binary_file: ./ext/int2lm/test/testsuite/int2lm
     namelist_filename: int2lm_INPUT.cfg
     runjob_filename: int2lm_runjob.cfg
     compute_queue: normal
@@ -69,7 +70,7 @@ post_int2lm:
         - CO2_A2
 
 cosmo:
-    binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu
+    binary_file: ./ext/cosmo-ghg/cosmo/ACC/cosmo_gpu
     namelist_prefix: cosmo_INPUT_
     runjob_filename: cosmo_runjob.cfg
     compute_queue: normal

cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg

@@ -1,5 +1,5 @@
 #!/bin/bash -l
-#SBATCH --job-name="cosmo_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}"
+#SBATCH --job-name=cosmo
 #SBATCH --account={cfg.compute_account}
 #SBATCH --time={walltime}
 #SBATCH --nodes={np_tot}
@@ -34,7 +34,7 @@ echo "============== StartTime: `date +%s` s"
 echo "============== StartTime: `date`"
 echo "====================================================="
 
-srun -u ./{execname} >> {logfile} 2>&1
+srun -u ./{cfg.cosmo_execname} >> {logfile} 2>&1
 pid=$?
 
 echo "====================================================="

cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg

@@ -1,5 +1,5 @@
 #!/bin/bash -l
-#SBATCH --job-name=int2lm_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}
+#SBATCH --job-name=int2lm
 #SBATCH --account={cfg.compute_account}
 #SBATCH --time={walltime}
 #SBATCH --nodes={nodes}

cases/cosmo-ghg-test/config.yaml

@@ -1,7 +1,9 @@
 # Configuration file for the 'cosmo-ghg-test' case with COSMO-GHG
 
-model: cosmo-ghg
+workflow: cosmo-ghg
 constraint: gpu
+run_on: gpu
+compute_queue: normal
 ntasks_per_node: 12
 restart_step: PT6H
 startdate: 2015-01-01T00:00:00Z
@@ -45,7 +47,7 @@ online_vprm:
 int2lm:
     extpar_dir: ./input/cosmo-ghg/extpar
     extpar_filename: test_domain.nc
-    binary_file: ./src/int2lm/test/testsuite/int2lm
+    binary_file: ./ext/int2lm/test/testsuite/int2lm
     namelist_filename: int2lm_INPUT.cfg
     runjob_filename: int2lm_runjob.cfg
     compute_queue: normal
@@ -67,7 +69,7 @@ post_int2lm:
         - CO2_A2
 
 cosmo:
-    binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu
+    binary_file: ./ext/cosmo-ghg/cosmo/ACC/cosmo_gpu
     namelist_prefix: cosmo_INPUT_
     runjob_filename: cosmo_runjob.cfg
     compute_queue: normal

cases/cosmo-ghg-test/cosmo_runjob.cfg

@@ -1,5 +1,5 @@
 #!/bin/bash -l
-#SBATCH --job-name="cosmo_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}"
+#SBATCH --job-name=cosmo
 #SBATCH --account={cfg.compute_account}
 #SBATCH --time={walltime}
 #SBATCH --nodes={np_tot}
@@ -34,7 +34,7 @@ echo "============== StartTime: `date +%s` s"
 echo "============== StartTime: `date`"
 echo "====================================================="
 
-srun -u ./{execname} >> {logfile} 2>&1
+srun -u ./{cfg.cosmo_execname} >> {logfile} 2>&1
 pid=$?
 
 echo "====================================================="

cases/cosmo-ghg-test/int2lm_runjob.cfg

@@ -1,5 +1,5 @@
 #!/bin/bash -l
-#SBATCH --job-name=int2lm_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}
+#SBATCH --job-name=int2lm
 #SBATCH --account={cfg.compute_account}
 #SBATCH --time={walltime}
 #SBATCH --nodes={nodes}