diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 095eecb200..076ab532e8 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -135,6 +135,12 @@ pipeline { echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}" sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"' } + + post { + always { + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } + } } } } diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 43b9935888..b33169d935 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -17,5 +17,120 @@ else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi +we2e_experiment_base_dir="${workspace}/experiments" +we2e_test_dir="${workspace}/regional_workflow/tests/WE2E" + +we2e_test_file="${we2e_test_dir}/experiments.txt" + +# The default set of end-to-end tests to run. +# TODO: Create a list of additional tests that can be run when a parameter +# is set to true. +declare -a we2e_default_tests +we2e_default_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' + 'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16' + 'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16') + +# Parses the test log for the status of a specific test. +function workflow_status() { + local test="$1" + + local test_dir="${we2e_experiment_base_dir}/${test}" + local log_file="${test_dir}/log.launch_FV3LAM_wflow" + + if [[ -f "${log_file}" ]]; then + local status + status="$(awk 'BEGIN {FS=":";} $1 ~ "^[[:space:]]+Workflow status" {print $2}' "${log_file}" |\ + tail -1 |\ + sed --regexp-extended --expression 's/^[[:space:]]*(.*)[[:space:]]*$/\1/')" + if [[ "${status}" == 'IN PROGRESS' || "${status}" == 'SUCCESS' || "${status}" == 'FAILURE' ]]; then + echo "${status}" + else + echo 'UNKNOWN' + fi + else + echo 'NOT FOUND' + fi +} + +# Gets the status of all tests. Prints the number of tests that are running. +# Returns a non-zero code when all tests reach a final state. +function check_progress() { + local in_progress=false + local remaining=0 + + for test in "${we2e_default_tests[@]}"; do + local status + status="$(workflow_status "${test}")" + if [[ "${status}" == 'IN PROGRESS' ]]; then + in_progress=true + (( remaining++ )) + fi + done + + if "${in_progress}"; then + echo "Tests remaining: ${remaining}" + else + return 1 + fi +} + +# Prints the status of all tests. +function get_results() { + for test in "${we2e_default_tests[@]}"; do + local status + status="$(workflow_status "${test}")" + echo "${test} ${status}" + done +} + # Verify that there is a non-zero sized weather model executable. [[ -s "${workspace}/bin/ufs_model" ]] || [[ -s "${workspace}/bin/NEMS.exe" ]] + +# Set test related environment variables and load required modules. +source "${workspace}/etc/lmod-setup.sh" "${SRW_PLATFORM}" +module use "${workspace}/modulefiles" +module load "build_${SRW_PLATFORM}_${SRW_COMPILER}" +module load "wflow_${SRW_PLATFORM}" + +if [[ "${SRW_PLATFORM}" == 'cheyenne' ]]; then + export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}" +else + conda activate regional_workflow +fi + +# Create the experiments/tests base directory. +mkdir "${we2e_experiment_base_dir}" + +# Generate the experiments/tests file. +for test in "${we2e_default_tests[@]}"; do + echo "${test}" >> "${we2e_test_file}" +done + +# Run the end-to-end tests. +"${we2e_test_dir}/run_WE2E_tests.sh" \ + tests_file="${we2e_test_file}" \ + machine="${SRW_PLATFORM}" \ + account="${SRW_PROJECT}" \ + expt_basedir="${we2e_experiment_base_dir}" \ + compiler="${SRW_COMPILER}" + +# Allow the tests to start before checking for status. +# TODO: Create a parameter that sets the initial start delay. +sleep 180 + +# Wait for all tests to complete. +while check_progress; do + # TODO: Create a paremeter that sets the poll frequency. + sleep 60 +done + +# Get test results and write to a file. +results="$(get_results |\ + tee "${workspace}/test_results-${SRW_PLATFORM}-${SRW_COMPILER}.txt")" + +# Check that the number of tests equals the number of successes, otherwise +# exit with a non-zero code that equals the difference. +successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)" +if [[ "${#we2e_default_tests[@]}" -ne "${successes}" ]]; then + exit "$(( "${#we2e_default_tests[@]}" - "${successes}" ))" +fi diff --git a/Externals.cfg b/Externals.cfg index b1573fb46d..108168d5f9 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,9 +1,9 @@ [regional_workflow] protocol = git -repo_url = https://github.com/ufs-community/regional_workflow +repo_url = https://github.com/danielabdi-noaa/regional_workflow # Specify either a branch name or a hash but not both. -#branch = develop -hash = d45d2ac +branch = feature/yaml_configs +#hash = d45d2ac local_path = regional_workflow required = True