From bd2d63600aac55e0ed75bf2dca296b19622ddff2 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 21 Mar 2022 15:27:42 -0500 Subject: [PATCH 01/22] Use new python env on orion --- modulefiles/GDAS/orion.lua | 47 +++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index 34e329334..c68a1e093 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -1,4 +1,4 @@ --- NOAA HPC Orion Modulefile for UFS-DA +-- NOAA HPC Orion Modulefile for GDASApp help([[ ]]) @@ -11,19 +11,50 @@ setenv('JEDI_OPT', jedi_opt) local jedi_core = pathJoin(jedi_opt, 'modulefiles/core') prepend_path("MODULEPATH", jedi_core) -load('jedi/intel-impi') +prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') +load("cmake/3.18.1") +load("git/2.28.0") +load("git-lfs/2.13.2") + +load("jedi-intel/2020.2") +load("mkl/2020.2") +load("szip/2.1.1") +load("zlib/1.2.11") +load("udunits/2.2.28") +load("gsl_lite/0.37.0") +load("jedi-impi/2020.2") + +load("hdf5/1.12.0") +load("pnetcdf/1.12.1") +load("netcdf/4.7.4") + +load("boost-headers/1.68.0") +load("eigen/3.3.7") +load("bufr/noaa-emc-11.5.0") +load("pybind11/2.7.0") +load("nccmp/1.8.7.0") +load("pio/2.5.1-debug") + +load("ecbuild/ecmwf-3.6.1") +load("eckit/ecmwf-1.16.0") +load("fckit/ecmwf-0.9.2") +load("atlas/ecmwf-0.24.1") + +load("hpc") +load("miniconda3") +load("gdasapp") + +setenv("CC","mpiicc") +setenv("FC","mpiifort") +setenv("CXX","mpiicpc") local mpiexec = '/opt/slurm/bin/srun' local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) --- add R2D2 and SOLO to PYTHONPATH -prepend_path("PYTHONPATH", "/work2/noaa/da/cmartin/UFSDA/python/local/lib/python3.9/site-packages") --- add R2D2 to path -prepend_path("PATH", "/work2/noaa/da/cmartin/UFSDA/python/local/bin") whatis("Name: ".. pkgName) whatis("Version: " .. pkgVersion) -whatis("Category: UFS-DA") -whatis("Description: Load JEDI-Stack for UFS-DA") +whatis("Category: GDASApp") +whatis("Description: Load all libraries needed for GDASApp") From b9d6e10bc72f1250e1098c2747517c449a9acf1c Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 10:42:27 -0500 Subject: [PATCH 02/22] Start of CI cron scripts --- ush/HPC-CI/driver.sh | 88 ++++++++++++++++++++++++++++++++++++++++++++ ush/HPC-CI/orion.sh | 10 +++++ ush/HPC-CI/run_ci.sh | 59 +++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100755 ush/HPC-CI/driver.sh create mode 100644 ush/HPC-CI/orion.sh create mode 100755 ush/HPC-CI/run_ci.sh diff --git a/ush/HPC-CI/driver.sh b/ush/HPC-CI/driver.sh new file mode 100755 index 000000000..646f27046 --- /dev/null +++ b/ush/HPC-CI/driver.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" + +# ============================================================================== +usage() { + set +x + echo + echo "Usage: $0 -t -h" + echo + echo " -t target/machine script is running on DEFAULT: $(hostname)" + echo " -h display this message and quit" + echo + exit 1 +} + +# ============================================================================== +# First, set up runtime environment + +export TARGET="$(hostname)" + +while getopts "t:h" opt; do + case $opt in + t) + TARGET=$OPTARG + ;; + h|\?|:) + usage + ;; + esac +done + +case ${TARGET} in + hera | orion) + echo "Running Automated Testing on $TARGET" + source $MODULESHOME/init/sh + source $my_dir/${TARGET}.sh + module purge + module use $GDAS_MODULE_USE + module load GDAS/$TARGET + module list + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac + + +# ============================================================================== +# pull on the repo and get list of open PRs +cd $GDAS_CI_ROOT/repo +CI_LABEL="${GDAS_CI_HOST}-RT" +gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list +open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) + +# ============================================================================== +# clone, checkout, build, test, etc. +repo_url="https://github.com/NOAA-EMC/GDASApp.git" +# loop through all open PRs +for pr in $open_pr_list; do + echo "Processing Pull Request #${pr}" + mkdir -p $GDAS_CI_ROOT/PR/$pr + cd $GDAS_CI_ROOT/PR/$pr + + # clone copy of repo + git clone $repo_url + cd GDASApp + + # checkout pull request + git pull + gh pr checkout $pr + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + if [ -f "$GDAS_CI_ROOT/PR/$pr/commit" ]; then + oldcommit=$(cat $GDAS_CI_ROOT/PR/$pr/commit) + if [ $oldcommit == $commit ]; then + # do no more for this PR, as the commit has already been tested + continue + fi + fi + echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit + + # run build and testing command + $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} +done + diff --git a/ush/HPC-CI/orion.sh b/ush/HPC-CI/orion.sh new file mode 100644 index 000000000..b07ff5550 --- /dev/null +++ b/ush/HPC-CI/orion.sh @@ -0,0 +1,10 @@ +GDAS_CI_ROOT=/work2/noaa/stmp/cmartin/CI/GDASApp +GDAS_CI_HOST='orion' +GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug +export SLURM_EXCLUSIVE=user +export OMP_NUM_THREADS=1 +ulimit -s unlimited diff --git a/ush/HPC-CI/run_ci.sh b/ush/HPC-CI/run_ci.sh new file mode 100755 index 000000000..40281aae5 --- /dev/null +++ b/ush/HPC-CI/run_ci.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#set -eu + +# ============================================================================== +usage() { + set +x + echo + echo "Usage: $0 -d -o -h" + echo + echo " -d Run build and ctest for clone in " + echo " -o Path to output message detailing results of CI tests" + echo " -h display this message and quit" + echo + exit 1 +} + +# ============================================================================== +while getopts "d:o:h" opt; do + case $opt in + d) + repodir=$OPTARG + ;; + o) + outfile=$OPTARG + ;; + h|\?|:) + usage + ;; + esac +done + +# ============================================================================== +# start output file +echo "Automated ${TARGET} Pull Request Testing Results:" > $outfile +echo '```' >> $outfile +echo "Start: $(date) on $(hostname)" >> $outfile +# ============================================================================== +# run build script +cd $repodir +module purge +./build.sh -t $TARGET +build_status=$? +if [ $build_status -eq 0 ]; then + echo "Build: *SUCCESS*" >> $outfile + echo "Build: Completed at $(date)" >> $outfile +else + echo "Build: *FAILED*" >> $outfile + echo "Build: Failed at $(date)" >> $outfile + echo '```' >> $outfile + exit 1 +fi +# ============================================================================== +# run ctests +cd $repodir/build +echo "---------------------------------" >> $outfile +ctest --output-on-failure &>> $outfile +ctest_status=$? +echo '```' >> $outfile +exit $ctest_status From 14a2d1e1c6c88241d1956632039d27e6e87131e2 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 10:56:01 -0500 Subject: [PATCH 03/22] Add writing of comment message --- ush/HPC-CI/driver.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ush/HPC-CI/driver.sh b/ush/HPC-CI/driver.sh index 646f27046..e3085aa37 100755 --- a/ush/HPC-CI/driver.sh +++ b/ush/HPC-CI/driver.sh @@ -84,5 +84,10 @@ for pr in $open_pr_list; do # run build and testing command $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} + ci_status=$? + git pr comment $pr --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --remove-label $CI_LABEL + fi done From 52a0c02511d1a2077dc11dc7d8c56e9c5c20482e Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 11:06:07 -0500 Subject: [PATCH 04/22] Make it a login shell for modules --- ush/HPC-CI/driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/HPC-CI/driver.sh b/ush/HPC-CI/driver.sh index e3085aa37..dd1b84b27 100755 --- a/ush/HPC-CI/driver.sh +++ b/ush/HPC-CI/driver.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash --login my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" From 5d9a9efca7a0ea0c8e7d6969dcb66ca8a7106ca4 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 13:35:39 -0500 Subject: [PATCH 05/22] Fix typo --- ush/HPC-CI/driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/HPC-CI/driver.sh b/ush/HPC-CI/driver.sh index dd1b84b27..1c14b72f4 100755 --- a/ush/HPC-CI/driver.sh +++ b/ush/HPC-CI/driver.sh @@ -85,7 +85,7 @@ for pr in $open_pr_list; do # run build and testing command $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} ci_status=$? - git pr comment $pr --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + gh pr comment $pr --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} if [ $ci_status -eq 0 ]; then gh pr edit $pr --remove-label $CI_LABEL fi From d54c8c687dda1622bd0698a2254b4c0808e1e153 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:11:35 -0500 Subject: [PATCH 06/22] Commit to save --- ush/HPC-CI/orion.sh | 2 +- ush/HPC-CI/run_ci.sh | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ush/HPC-CI/orion.sh b/ush/HPC-CI/orion.sh index b07ff5550..1084046ce 100644 --- a/ush/HPC-CI/orion.sh +++ b/ush/HPC-CI/orion.sh @@ -1,6 +1,6 @@ GDAS_CI_ROOT=/work2/noaa/stmp/cmartin/CI/GDASApp GDAS_CI_HOST='orion' -GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles +export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles export SLURM_ACCOUNT=da-cpu export SALLOC_ACCOUNT=$SLURM_ACCOUNT export SBATCH_ACCOUNT=$SLURM_ACCOUNT diff --git a/ush/HPC-CI/run_ci.sh b/ush/HPC-CI/run_ci.sh index 40281aae5..5f13683d4 100755 --- a/ush/HPC-CI/run_ci.sh +++ b/ush/HPC-CI/run_ci.sh @@ -31,14 +31,15 @@ done # ============================================================================== # start output file -echo "Automated ${TARGET} Pull Request Testing Results:" > $outfile +echo "Automated Pull Request Testing Results:" > $outfile +echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile # ============================================================================== # run build script cd $repodir module purge -./build.sh -t $TARGET +./build.sh -t $TARGET &>> log.build build_status=$? if [ $build_status -eq 0 ]; then echo "Build: *SUCCESS*" >> $outfile @@ -46,14 +47,18 @@ if [ $build_status -eq 0 ]; then else echo "Build: *FAILED*" >> $outfile echo "Build: Failed at $(date)" >> $outfile + echo "Build: see output at $repodir/log.build" >> $outfile echo '```' >> $outfile exit 1 fi # ============================================================================== # run ctests cd $repodir/build +module use $GDAS_MODULE_USE +module load GDAS/$TARGET echo "---------------------------------" >> $outfile ctest --output-on-failure &>> $outfile +echo "Completed at $(date)" >> $outfile ctest_status=$? echo '```' >> $outfile exit $ctest_status From 31350264bc19079ecca0b2906139b9fb73b1f168 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:13:38 -0500 Subject: [PATCH 07/22] move files to ci/ --- {ush/HPC-CI => ci}/driver.sh | 0 {ush/HPC-CI => ci}/orion.sh | 0 {ush/HPC-CI => ci}/run_ci.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {ush/HPC-CI => ci}/driver.sh (100%) rename {ush/HPC-CI => ci}/orion.sh (100%) rename {ush/HPC-CI => ci}/run_ci.sh (100%) diff --git a/ush/HPC-CI/driver.sh b/ci/driver.sh similarity index 100% rename from ush/HPC-CI/driver.sh rename to ci/driver.sh diff --git a/ush/HPC-CI/orion.sh b/ci/orion.sh similarity index 100% rename from ush/HPC-CI/orion.sh rename to ci/orion.sh diff --git a/ush/HPC-CI/run_ci.sh b/ci/run_ci.sh similarity index 100% rename from ush/HPC-CI/run_ci.sh rename to ci/run_ci.sh From 3189c35945bdc62e2679bd996fc1f9ad8451d393 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:14:28 -0500 Subject: [PATCH 08/22] Comment out repos for testing/debugging --- CMakeLists.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc3f2264..a34cb9663 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,23 +62,23 @@ if(BUILD_GDASBUNDLE) # Core JEDI repositories ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) + #ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/noaa-emc/ufo.git" BRANCH feature/ufsda UPDATE ) + #ecbuild_bundle( PROJECT ufo GIT "https://github.com/noaa-emc/ufo.git" BRANCH feature/ufsda UPDATE ) # FMS and FV3 dynamical core - ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE ) - ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE ) + #ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE ) + #ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE ) # fv3-jedi and associated repositories - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop UPDATE ) + #ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop UPDATE ) + #ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) + #ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop UPDATE ) # SOCA associated repositories - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop ) - ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop ) + #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop ) + #ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE ) + #ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop ) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF) From 4bfc5c7f8b9e0184535c56e3f1c938a800dbf330 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:23:01 -0500 Subject: [PATCH 09/22] Remove CRTM --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a34cb9663..f2a29da1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,7 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) # External (required) observation operators - ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) + #ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) # Core JEDI repositories ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop UPDATE ) From 9f11f281700e569961d27e821974943e5da232be Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:40:41 -0500 Subject: [PATCH 10/22] Add scrubber and change of labels --- ci/driver.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/driver.sh b/ci/driver.sh index 1c14b72f4..cbd326960 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -59,6 +59,7 @@ open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) repo_url="https://github.com/NOAA-EMC/GDASApp.git" # loop through all open PRs for pr in $open_pr_list; do + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running echo "Processing Pull Request #${pr}" mkdir -p $GDAS_CI_ROOT/PR/$pr cd $GDAS_CI_ROOT/PR/$pr @@ -87,7 +88,13 @@ for pr in $open_pr_list; do ci_status=$? gh pr comment $pr --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} if [ $ci_status -eq 0 ]; then - gh pr edit $pr --remove-label $CI_LABEL + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed fi done +# ============================================================================== +# scrub working directory for older files +find $GDAS_CI_ROOT/PR/* -mtime +3 -exec rm -rf {} \; + From 39dab947ad3c6d01892045708cc6954ee3662863 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 14:52:19 -0500 Subject: [PATCH 11/22] Changes to the CI script --- ci/run_ci.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 5f13683d4..84b0434c9 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -42,23 +42,33 @@ module purge ./build.sh -t $TARGET &>> log.build build_status=$? if [ $build_status -eq 0 ]; then - echo "Build: *SUCCESS*" >> $outfile + echo "Build: *SUCCESS*" >> $outfile echo "Build: Completed at $(date)" >> $outfile else - echo "Build: *FAILED*" >> $outfile + echo "Build: *FAILED*" >> $outfile echo "Build: Failed at $(date)" >> $outfile echo "Build: see output at $repodir/log.build" >> $outfile echo '```' >> $outfile - exit 1 + exit $build_status fi # ============================================================================== # run ctests cd $repodir/build module use $GDAS_MODULE_USE module load GDAS/$TARGET -echo "---------------------------------" >> $outfile -ctest --output-on-failure &>> $outfile -echo "Completed at $(date)" >> $outfile +echo "---------------------------------------------------" >> $outfile +ctest --output-on-failure &>> log.ctest ctest_status=$? +npassed=$(cat log.ctest | grep "tests passed") +if [ $ctest_status -eq 0 ]; then + echo "Tests: *SUCCESS*" >> $outfile + echo "Tests: Completed at $(date)" >> $outfile + echo "Tests: $npassed" >> $outfile +else + echo "Tests: *Failed*" >> $outfile + echo "Tests: Failed at $(date)" >> $outfile + echo "Tests: $npassed" >> $outfile + echo "Tests: see output at $repodir/build/log.ctest" >> $outfile +fi echo '```' >> $outfile exit $ctest_status From 86d99c26254aac1f08641dd65fec51ae85c39324 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 15:10:59 -0500 Subject: [PATCH 12/22] Add back all repos --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2a29da1d..fdc3f2264 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,27 +58,27 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) # External (required) observation operators - #ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) + ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) # Core JEDI repositories ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop UPDATE ) - #ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop UPDATE ) - #ecbuild_bundle( PROJECT ufo GIT "https://github.com/noaa-emc/ufo.git" BRANCH feature/ufsda UPDATE ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/noaa-emc/ufo.git" BRANCH feature/ufsda UPDATE ) # FMS and FV3 dynamical core - #ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE ) - #ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE ) + ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE ) + ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE ) # fv3-jedi and associated repositories - #ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop UPDATE ) - #ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) - #ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop UPDATE ) # SOCA associated repositories - #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop ) - #ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE ) - #ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop ) + ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop ) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF) From 78484ef0397fb4aaf6b51c874848c8bb627f7f12 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Mar 2022 15:24:52 -0500 Subject: [PATCH 13/22] Update comment format --- ci/run_ci.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 84b0434c9..39669486f 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -35,6 +35,7 @@ echo "Automated Pull Request Testing Results:" > $outfile echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile +echo "---------------------------------------------------" >> $outfile # ============================================================================== # run build script cd $repodir @@ -42,10 +43,10 @@ module purge ./build.sh -t $TARGET &>> log.build build_status=$? if [ $build_status -eq 0 ]; then - echo "Build: *SUCCESS*" >> $outfile + echo "Build: *SUCCESS*" >> $outfile echo "Build: Completed at $(date)" >> $outfile else - echo "Build: *FAILED*" >> $outfile + echo "Build: *FAILED*" >> $outfile echo "Build: Failed at $(date)" >> $outfile echo "Build: see output at $repodir/log.build" >> $outfile echo '```' >> $outfile @@ -61,13 +62,13 @@ ctest --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then - echo "Tests: *SUCCESS*" >> $outfile + echo "Tests: *SUCCESS*" >> $outfile echo "Tests: Completed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile + echo "Tests: $npassed" >> $outfile else - echo "Tests: *Failed*" >> $outfile + echo "Tests: *Failed*" >> $outfile echo "Tests: Failed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile + echo "Tests: $npassed" >> $outfile echo "Tests: see output at $repodir/build/log.ctest" >> $outfile fi echo '```' >> $outfile From d78686c80278e47de775a68f59f57cb74e526aef Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 15:22:38 +0000 Subject: [PATCH 14/22] Add back unit tests on push --- .github/workflows/unittests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 5622a4de6..2a409cbd5 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -1,5 +1,5 @@ name: Run Unit Tests -on: [pull_request] +on: [push, pull_request] jobs: ctests: From a25b6a3844d95dcdac3373843879ea81cff8dbd9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 15:31:41 +0000 Subject: [PATCH 15/22] Changes to hera modules, and add hera ci config --- ci/hera.sh | 8 ++++++ modulefiles/GDAS/hera.lua | 51 +++++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 ci/hera.sh diff --git a/ci/hera.sh b/ci/hera.sh new file mode 100644 index 000000000..60b1c9a1f --- /dev/null +++ b/ci/hera.sh @@ -0,0 +1,8 @@ +GDAS_CI_ROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp +GDAS_CI_HOST='hera' +export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug +ulimit -s unlimited diff --git a/modulefiles/GDAS/hera.lua b/modulefiles/GDAS/hera.lua index 0ae09e4ad..3a688b6a9 100644 --- a/modulefiles/GDAS/hera.lua +++ b/modulefiles/GDAS/hera.lua @@ -1,17 +1,58 @@ --- NOAA RDHPCS Hera Modulefile for UFS-DA help([[ +Load environment for running JEDI applications with Intel compilers and MPI. ]]) local pkgName = myModuleName() local pkgVersion = myModuleVersion() local pkgNameVer = myModuleFullName() +conflict(pkgName) + local jedi_opt = '/scratch1/NCEPDEV/jcsda/jedipara/opt/modules' setenv('JEDI_OPT', jedi_opt) local jedi_core = pathJoin(jedi_opt, 'modulefiles/core') prepend_path("MODULEPATH", jedi_core) -load('jedi/intel-impi/2020.2') +prepend_path("MODULEPATH", '/scratch1/NCEPDEV/da/python/opt/modulefiles/stack') + +load("cmake/3.20.1") +load("git-lfs/2.11.0") + +load("jedi-intel/2020.2") +load("szip/2.1.1") +load("zlib/1.2.11") +load("udunits/2.2.28") +load("gsl_lite/0.37.0") +load("jedi-impi/2020.2") + +load("hdf5/1.12.0") +load("pnetcdf/1.12.1") +load("netcdf/4.7.4") + +load("boost-headers/1.68.0") +load("eigen/3.3.7") +load("bufr/noaa-emc-11.5.0") + +load("nccmp/1.8.7.0") +load("pio/2.5.1-debug") + +load("ecbuild/ecmwf-3.6.1") +load("eckit/ecmwf-1.16.0") +load("fckit/ecmwf-0.9.2") +load("atlas/ecmwf-0.24.1") +load("nco/4.9.1") + +load("pybind11/2.7.0") +load("json/3.9.1") +load("json-schema-validator/2.1.0") + +load("hpc") +load("miniconda3") +load("gdasapp") + +setenv("CC","mpiicc") +setenv("FC","mpiifort") +setenv("CXX","mpiicpc") local mpiexec = '/apps/slurm/default/bin/srun' local mpinproc = '-n' @@ -19,6 +60,6 @@ setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) whatis("Name: ".. pkgName) -whatis("Version: " .. pkgVersion) -whatis("Category: UFS-DA") -whatis("Description: Load JEDI-Stack for UFS-DA") +whatis("Version: ".. pkgVersion) +whatis("Category: GDASApp") +whatis("Description: Load all libraries needed for GDASApp") From 2ff4158d653c29f557fb312c61d88015b8543db6 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 15:41:35 +0000 Subject: [PATCH 16/22] Address reviewer comments round 1 --- ci/run_ci.sh | 2 +- modulefiles/GDAS/orion.lua | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 39669486f..c3bdbd7f1 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -58,7 +58,7 @@ cd $repodir/build module use $GDAS_MODULE_USE module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile -ctest --output-on-failure &>> log.ctest +ctest -R gdasapp --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index c68a1e093..56e9ca73f 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -41,9 +41,9 @@ load("eckit/ecmwf-1.16.0") load("fckit/ecmwf-0.9.2") load("atlas/ecmwf-0.24.1") -load("hpc") -load("miniconda3") -load("gdasapp") +load("hpc/1.2.0") +load("miniconda3/4.6.14") +load("gdasapp/1.0.0") setenv("CC","mpiicc") setenv("FC","mpiifort") From e1de501ac25644c0dd0c68ad43f28315eb3ed9e4 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 16:03:41 +0000 Subject: [PATCH 17/22] Change test names --- test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b8ff8c906..f006b413b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,14 +28,14 @@ install(FILES ${test_data} ##### unit tests # test for python coding norms -add_test(NAME GDASApp_check_python_norms +add_test(NAME test_gdasapp_check_python_norms COMMAND pycodestyle -v --config ./.pycodestyle . WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) # test for ush/check_yaml_keys.py -add_test(NAME GDASApp_check_yaml_keys +add_test(NAME test_gdasapp_check_yaml_keys COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/check_yaml_keys.py ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_ref.yaml ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_test.yaml WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) # test for ush/jediinc2fv3.py -add_test(NAME GDASApp_jedi_increment_to_fv3 +add_test(NAME test_gdasapp_jedi_increment_to_fv3 COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/jediinc2fv3.py ${PROJECT_BINARY_DIR}/test/testdata/atminc_compress.nc4 ${PROJECT_BINARY_DIR}/test/testdata/fv_increment.nc WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) From 276e6d585d1831696dcb3da9b55bd07816505713 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 16:04:52 +0000 Subject: [PATCH 18/22] Remove ulimit --- ci/hera.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/hera.sh b/ci/hera.sh index 60b1c9a1f..854c86097 100644 --- a/ci/hera.sh +++ b/ci/hera.sh @@ -5,4 +5,3 @@ export SLURM_ACCOUNT=da-cpu export SALLOC_ACCOUNT=$SLURM_ACCOUNT export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SLURM_QOS=debug -ulimit -s unlimited From baf9baf9d86bb0ef40e382e1e544524711d1c7ee Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 20:17:42 +0000 Subject: [PATCH 19/22] Reviewer comments --- modulefiles/GDAS/hera.lua | 8 ++++---- modulefiles/GDAS/orion.lua | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modulefiles/GDAS/hera.lua b/modulefiles/GDAS/hera.lua index 3a688b6a9..d4e0dc4c3 100644 --- a/modulefiles/GDAS/hera.lua +++ b/modulefiles/GDAS/hera.lua @@ -1,5 +1,5 @@ help([[ -Load environment for running JEDI applications with Intel compilers and MPI. +Load environment for running the GDAS application with Intel compilers and MPI. ]]) local pkgName = myModuleName() @@ -46,9 +46,9 @@ load("pybind11/2.7.0") load("json/3.9.1") load("json-schema-validator/2.1.0") -load("hpc") -load("miniconda3") -load("gdasapp") +load("hpc/1.2.0") +load("miniconda3/4.6.14") +load("gdasapp/1.0.0") setenv("CC","mpiicc") setenv("FC","mpiifort") diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index 56e9ca73f..f797f0c2e 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -13,7 +13,7 @@ prepend_path("MODULEPATH", jedi_core) prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') -load("cmake/3.18.1") +load("cmake/3.22.1") load("git/2.28.0") load("git-lfs/2.13.2") From 99b9c582a6e2ef787f9491aad411fbba8f6640e7 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 20:35:52 +0000 Subject: [PATCH 20/22] have pycodestyle only check certain dirs --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f006b413b..4a52bf72e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,7 @@ install(FILES ${test_data} ##### unit tests # test for python coding norms add_test(NAME test_gdasapp_check_python_norms - COMMAND pycodestyle -v --config ./.pycodestyle . + COMMAND pycodestyle -v --config ./.pycodestyle ./ush ./scripts ./jobs WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) # test for ush/check_yaml_keys.py add_test(NAME test_gdasapp_check_yaml_keys From db7ee8b70d869435b87f0cca61727c68a044c8ba Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 20:37:11 +0000 Subject: [PATCH 21/22] Remove dirs that donot exist yet --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4a52bf72e..b22794d7e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,7 @@ install(FILES ${test_data} ##### unit tests # test for python coding norms add_test(NAME test_gdasapp_check_python_norms - COMMAND pycodestyle -v --config ./.pycodestyle ./ush ./scripts ./jobs + COMMAND pycodestyle -v --config ./.pycodestyle ./ush WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) # test for ush/check_yaml_keys.py add_test(NAME test_gdasapp_check_yaml_keys From ae9633d1e5a7b1f755f338724b1a119856234e7b Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 23 Mar 2022 20:38:57 +0000 Subject: [PATCH 22/22] make extract verbose --- test/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b22794d7e..1ccb4bc8d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,7 +15,8 @@ file(DOWNLOAD EXPECTED_HASH SHA256=${SHA} ) # Extract downloaded tarball. -file(ARCHIVE_EXTRACT INPUT ${TAR}) +file(ARCHIVE_EXTRACT INPUT ${TAR} + VERBOSE) # list of test files to install list(APPEND test_data