Release/public v2 UFS-COMMUNITY SRW repo - current

.cicd/Jenkinsfile

@@ -135,6 +135,12 @@ pipeline {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
                             sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
                         }
+
+                        post {
+                            always {
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                            }
+                        }
                     }
                 }
             }

.cicd/scripts/srw_test.sh

@@ -17,5 +17,120 @@ else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
 fi
 
+we2e_experiment_base_dir="${workspace}/experiments"
+we2e_test_dir="${workspace}/regional_workflow/tests/WE2E"
+
+we2e_test_file="${we2e_test_dir}/experiments.txt"
+
+# The default set of end-to-end tests to run.
+# TODO: Create a list of additional tests that can be run when a parameter
+# is set to true.
+declare -a we2e_default_tests
+we2e_default_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16')
+
+# Parses the test log for the status of a specific test.
+function workflow_status() {
+    local test="$1"
+
+    local test_dir="${we2e_experiment_base_dir}/${test}"
+    local log_file="${test_dir}/log.launch_FV3LAM_wflow"
+
+    if [[ -f "${log_file}" ]]; then
+        local status
+        status="$(awk 'BEGIN {FS=":";} $1 ~ "^[[:space:]]+Workflow status" {print $2}' "${log_file}" |\
+            tail -1 |\
+            sed --regexp-extended --expression 's/^[[:space:]]*(.*)[[:space:]]*$/\1/')"
+        if [[ "${status}" == 'IN PROGRESS' || "${status}" == 'SUCCESS' || "${status}" == 'FAILURE' ]]; then
+            echo "${status}"
+        else
+            echo 'UNKNOWN'
+        fi
+    else
+        echo 'NOT FOUND'
+    fi
+}
+
+# Gets the status of all tests. Prints the number of tests that are running.
+# Returns a non-zero code when all tests reach a final state.
+function check_progress() {
+    local in_progress=false
+    local remaining=0
+
+    for test in "${we2e_default_tests[@]}"; do
+        local status
+        status="$(workflow_status "${test}")"
+        if [[ "${status}" == 'IN PROGRESS' ]]; then
+            in_progress=true
+            (( remaining++ ))
+        fi
+    done
+
+    if "${in_progress}"; then
+        echo "Tests remaining: ${remaining}"
+    else
+        return 1
+    fi
+}
+
+# Prints the status of all tests.
+function get_results() {
+    for test in "${we2e_default_tests[@]}"; do
+        local status
+        status="$(workflow_status "${test}")"
+        echo "${test} ${status}"
+    done
+}
+
 # Verify that there is a non-zero sized weather model executable.
 [[ -s "${workspace}/bin/ufs_model" ]] || [[ -s "${workspace}/bin/NEMS.exe" ]]
+
+# Set test related environment variables and load required modules.
+source "${workspace}/etc/lmod-setup.sh" "${SRW_PLATFORM}"
+module use "${workspace}/modulefiles"
+module load "build_${SRW_PLATFORM}_${SRW_COMPILER}"
+module load "wflow_${SRW_PLATFORM}"
+
+if [[ "${SRW_PLATFORM}" == 'cheyenne' ]]; then
+    export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}"
+else
+    conda activate regional_workflow
+fi
+
+# Create the experiments/tests base directory.
+mkdir "${we2e_experiment_base_dir}"
+
+# Generate the experiments/tests file.
+for test in "${we2e_default_tests[@]}"; do
+    echo "${test}" >> "${we2e_test_file}"
+done
+
+# Run the end-to-end tests.
+"${we2e_test_dir}/run_WE2E_tests.sh" \
+    tests_file="${we2e_test_file}" \
+    machine="${SRW_PLATFORM}" \
+    account="${SRW_PROJECT}" \
+    expt_basedir="${we2e_experiment_base_dir}" \
+    compiler="${SRW_COMPILER}"
+
+# Allow the tests to start before checking for status.
+# TODO: Create a parameter that sets the initial start delay.
+sleep 180
+
+# Wait for all tests to complete.
+while check_progress; do
+    # TODO: Create a paremeter that sets the poll frequency.
+    sleep 60
+done
+
+# Get test results and write to a file.
+results="$(get_results |\
+    tee "${workspace}/test_results-${SRW_PLATFORM}-${SRW_COMPILER}.txt")"
+
+# Check that the number of tests equals the number of successes, otherwise
+# exit with a non-zero code that equals the difference.
+successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)"
+if [[ "${#we2e_default_tests[@]}" -ne "${successes}" ]]; then
+    exit "$(( "${#we2e_default_tests[@]}" - "${successes}" ))"
+fi

devbuild.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash 
 
 # usage instructions
 usage () {
@@ -153,7 +153,7 @@ set -eu
 if [ -z "${COMPILER}" ] ; then
   case ${PLATFORM} in
     jet|hera|gaea) COMPILER=intel ;;
-    orion) COMPILER=intel ;;
+    orion|noaacloud) COMPILER=intel ;;
     wcoss_dell_p3) COMPILER=intel ;;
     cheyenne) COMPILER=intel ;;
     macos,singularity) COMPILER=gnu ;;

etc/lmod-setup.csh

@@ -19,6 +19,12 @@ else if ( "$L_MACHINE" == singularity ) then
 
    module purge
 
+else if ( "$L_MACHINE" == noaacloud ) then
+   set ENV="/usr/share/lmod/lmod/init/csh"
+   source $ENV
+
+   module purge
+
 else if ( "$L_MACHINE" == gaea ) then
    set ENV="/lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/lmod/lmod/init/csh"
    source $ENV

etc/lmod-setup.sh

@@ -13,6 +13,11 @@ if [ "$L_MACHINE" = macos ]; then
 
    module purge
 
+elif [ "$L_MACHINE" = noaacloud ]; then
+   export BASH_ENV="/usr/share/lmod/lmod/init/bash"
+   source $BASH_ENV
+
+   module purge
 elif [ "$L_MACHINE" = singularity ]; then
    export BASH_ENV="/usr/share/lmod/lmod/init/bash"
    source $BASH_ENV

modulefiles/build_macos_gnu

@@ -1,14 +1,15 @@
 #%Module
 
 proc ModulesHelp { } {
-   puts stderr "This module loads libraries for building SRW on"
-   puts stderr "macOS Big Sur (Darwin20) using GNU 11.2 + openmpi/4.2.1"
+   puts stderr "Option 1: M1/arm64 platform, OS BigSur,Monterey (Darwin 20,21) "
+   puts stderr "Option 2: Intel/x86_64 platform, OS BigSur (Darwin 19) "
+   puts stderr "
 }
 
-module-whatis "Loads libraries needed for building SRW on Hera"
+module-whatis "Loads libraries needed for building SRW on MacOS"
 
 if { [module-info mode load] } {
-   system "ulimit -S -s unlimited;"
+   puts "ulimit -S -s unlimited;"
 }
 
 # This path should point to your HPCstack installation directory
@@ -51,11 +52,11 @@ module load yafyaml/v0.5.1
 module load mapl/2.12.2-esmf-8_2_0
 module load gfsio/1.4.1
 module load landsfcutil/2.4.1
-module load nemsio/2.5.4
+module load nemsio/2.5.2
 module load nemsiogfs/2.5.3
 module load sfcio/1.4.1
 module load sigio/2.3.2
-module load w3emc/2.9.2
+module load w3emc/2.7.3
 module load wgrib2/2.0.8
 module list
 
@@ -64,7 +65,7 @@ setenv CC "/opt/homebrew/bin/gcc"
 setenv FC "/opt/homebrew/bin/gfortran"
 setenv CXX "/opt/homebrew/bin/g++"
 
-# Option 2 compiler paths:
+# Option 2 compiler paths (uncomment):
 #setenv CC "/usr/local/bin/gcc"
 #setenv FC "/usr/local/bin/gfortran"
 #setenv CXX "/usr/local/bin/g++"
@@ -79,5 +80,7 @@ setenv CMAKE_Fortran_COMPILER $env(MPI_FC)
 setenv CMAKE_Platform macos.gnu
 
 setenv CMAKE_Fortran_COMPILER_ID "GNU"
-setenv LDFLAGS "-L$env(MPI_ROOT)/lib"
 setenv FFLAGS "-DNO_QUAD_PRECISION -fallow-argument-mismatch "
+# export the environment variable LDFLAGS from the command line
+# after loading the current module:
+# export LDFLAGS="-L$MPI_ROOT/lib"

modulefiles/wflow_macos

@@ -5,7 +5,7 @@ proc ModulesHelp { } {
    put stderr "macOS"
 }
 
-module-whatis "Loads libraries needed for running SRW on macOS"
+module-whatis "This module activates python environment for running SRW on macOS"
 
 setenv CMAKE_Platform macos
 
@@ -14,7 +14,7 @@ setenv CMAKE_Platform macos
 # 
 setenv VENV "/Users/username/venv/regional_workflow"
 if { [module-info mode load] } {
-   system "source $env(VENV)/bin/activate;"
+   puts "source $env(VENV)/bin/activate;"
 }
 
 # Uncomment if Rocoto workflow manager is used