Add automated CI testing through cron on Orion

ci/driver.sh

@@ -0,0 +1,100 @@
+#!/bin/bash --login
+
+my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+
+# ==============================================================================
+usage() {
+  set +x
+  echo
+  echo "Usage: $0 -t <target> -h"
+  echo
+  echo "  -t  target/machine script is running on    DEFAULT: $(hostname)"
+  echo "  -h  display this message and quit"
+  echo
+  exit 1
+}
+
+# ==============================================================================
+# First, set up runtime environment
+
+export TARGET="$(hostname)"
+
+while getopts "t:h" opt; do
+  case $opt in
+    t)
+      TARGET=$OPTARG
+      ;;
+    h|\?|:)
+      usage
+      ;;
+  esac
+done
+
+case ${TARGET} in
+  hera | orion)
+    echo "Running Automated Testing on $TARGET"
+    source $MODULESHOME/init/sh
+    source $my_dir/${TARGET}.sh
+    module purge
+    module use $GDAS_MODULE_USE
+    module load GDAS/$TARGET
+    module list
+    ;;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1
+    ;;
+esac
+
+
+# ==============================================================================
+# pull on the repo and get list of open PRs
+cd $GDAS_CI_ROOT/repo
+CI_LABEL="${GDAS_CI_HOST}-RT"
+gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list
+open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list)
+
+# ==============================================================================
+# clone, checkout, build, test, etc.
+repo_url="https://github.com/NOAA-EMC/GDASApp.git"
+# loop through all open PRs
+for pr in $open_pr_list; do
+  gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running
+  echo "Processing Pull Request #${pr}"
+  mkdir -p $GDAS_CI_ROOT/PR/$pr
+  cd $GDAS_CI_ROOT/PR/$pr
+
+  # clone copy of repo
+  git clone $repo_url
+  cd GDASApp
+
+  # checkout pull request
+  git pull
+  gh pr checkout $pr
+
+  # get commit hash
+  commit=$(git log --pretty=format:'%h' -n 1)
+  if [ -f "$GDAS_CI_ROOT/PR/$pr/commit" ]; then
+    oldcommit=$(cat $GDAS_CI_ROOT/PR/$pr/commit)
+    if [ $oldcommit == $commit ]; then
+      # do no more for this PR, as the commit has already been tested
+      continue
+    fi
+  fi
+  echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit
+
+  # run build and testing command
+  $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit}
+  ci_status=$?
+  gh pr comment $pr --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit}
+  if [ $ci_status -eq 0 ]; then
+    gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed
+  else
+    gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed
+  fi
+done
+
+# ==============================================================================
+# scrub working directory for older files
+find $GDAS_CI_ROOT/PR/* -mtime +3 -exec rm -rf {} \;
+

ci/orion.sh

@@ -0,0 +1,10 @@
+GDAS_CI_ROOT=/work2/noaa/stmp/cmartin/CI/GDASApp
+GDAS_CI_HOST='orion'
+export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles
+export SLURM_ACCOUNT=da-cpu
+export SALLOC_ACCOUNT=$SLURM_ACCOUNT
+export SBATCH_ACCOUNT=$SLURM_ACCOUNT
+export SLURM_QOS=debug
+export SLURM_EXCLUSIVE=user
+export OMP_NUM_THREADS=1
+ulimit -s unlimited

ci/run_ci.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#set -eu
+
+# ==============================================================================
+usage() {
+  set +x
+  echo
+  echo "Usage: $0 -d <directory> -o <output> -h"
+  echo
+  echo "  -d  Run build and ctest for clone in <directory>"
+  echo "  -o  Path to output message detailing results of CI tests"
+  echo "  -h  display this message and quit"
+  echo
+  exit 1
+}
+
+# ==============================================================================
+while getopts "d:o:h" opt; do
+  case $opt in
+    d)
+      repodir=$OPTARG
+      ;;
+    o)
+      outfile=$OPTARG
+      ;;
+    h|\?|:)
+      usage
+      ;;
+  esac
+done
+
+# ==============================================================================
+# start output file
+echo "Automated Pull Request Testing Results:" > $outfile
+echo "Machine: ${TARGET}" >> $outfile
+echo '```' >> $outfile
+echo "Start: $(date) on $(hostname)" >> $outfile
+echo "---------------------------------------------------" >> $outfile
+# ==============================================================================
+# run build script
+cd $repodir
+module purge
+./build.sh -t $TARGET &>> log.build
+build_status=$?
+if [ $build_status -eq 0 ]; then
+  echo "Build:                                 *SUCCESS*" >> $outfile
+  echo "Build: Completed at $(date)" >> $outfile
+else
+  echo "Build:                                  *FAILED*" >> $outfile
+  echo "Build: Failed at $(date)" >> $outfile
+  echo "Build: see output at $repodir/log.build" >> $outfile
+  echo '```' >> $outfile
+  exit $build_status
+fi
+# ==============================================================================
+# run ctests
+cd $repodir/build
+module use $GDAS_MODULE_USE
+module load GDAS/$TARGET
+echo "---------------------------------------------------" >> $outfile
+ctest --output-on-failure &>> log.ctest
+ctest_status=$?
+npassed=$(cat log.ctest | grep "tests passed")
+if [ $ctest_status -eq 0 ]; then
+  echo "Tests:                                 *SUCCESS*" >> $outfile
+  echo "Tests: Completed at $(date)" >> $outfile
+  echo "Tests: $npassed" >> $outfile
+else
+  echo "Tests:                                  *Failed*" >> $outfile
+  echo "Tests: Failed at $(date)" >> $outfile
+  echo "Tests: $npassed" >> $outfile
+  echo "Tests: see output at $repodir/build/log.ctest" >> $outfile
+fi
+echo '```' >> $outfile
+exit $ctest_status

modulefiles/GDAS/orion.lua

@@ -1,4 +1,4 @@
--- NOAA HPC Orion Modulefile for UFS-DA
+-- NOAA HPC Orion Modulefile for GDASApp 
 help([[
 ]])
 
@@ -11,19 +11,50 @@ setenv('JEDI_OPT', jedi_opt)
 local jedi_core = pathJoin(jedi_opt, 'modulefiles/core')
 prepend_path("MODULEPATH", jedi_core)
 
-load('jedi/intel-impi')
+prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')
 
+load("cmake/3.18.1")
+load("git/2.28.0")
+load("git-lfs/2.13.2")
+
+load("jedi-intel/2020.2")
+load("mkl/2020.2")
+load("szip/2.1.1")
+load("zlib/1.2.11")
+load("udunits/2.2.28")
+load("gsl_lite/0.37.0")
+load("jedi-impi/2020.2")
+
+load("hdf5/1.12.0")
+load("pnetcdf/1.12.1")
+load("netcdf/4.7.4")
+
+load("boost-headers/1.68.0")
+load("eigen/3.3.7")
+load("bufr/noaa-emc-11.5.0")
+load("pybind11/2.7.0")
+load("nccmp/1.8.7.0")
+load("pio/2.5.1-debug")
+
+load("ecbuild/ecmwf-3.6.1")
+load("eckit/ecmwf-1.16.0")
+load("fckit/ecmwf-0.9.2")
+load("atlas/ecmwf-0.24.1")
+
+load("hpc")
+load("miniconda3")
+load("gdasapp")
+
+setenv("CC","mpiicc")
+setenv("FC","mpiifort")
+setenv("CXX","mpiicpc")
 local mpiexec = '/opt/slurm/bin/srun'
 local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
--- add R2D2 and SOLO to PYTHONPATH
-prepend_path("PYTHONPATH", "/work2/noaa/da/cmartin/UFSDA/python/local/lib/python3.9/site-packages")
--- add R2D2 to path
-prepend_path("PATH", "/work2/noaa/da/cmartin/UFSDA/python/local/bin")
 
 whatis("Name: ".. pkgName)
 whatis("Version: " .. pkgVersion)
-whatis("Category: UFS-DA")
-whatis("Description: Load JEDI-Stack for UFS-DA")
+whatis("Category: GDASApp")
+whatis("Description: Load all libraries needed for GDASApp")