[develop] Update Jenkinsfile to run post-build functional tests

.cicd/Jenkinsfile

@@ -127,12 +127,13 @@ pipeline {
                     }
 
                     // Run the unittest functional tests that require an HPC platform
-                    stage('Functional Tests') {
+                    stage('Functional UnitTests') {
                         steps {
                             echo "Running unittest on retrieve_data.py"
                             sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"'
                         }
                     }
+
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
                         steps {
@@ -148,6 +149,14 @@ pipeline {
                         }
                     }
 
+                    // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage
+                    stage('Functional WorkflowTaskTests') {
+                        steps {
+                            echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
+                            sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_ftest.sh"'
+                        }
+                    }
+
                     // Run the unified test script
                     stage('Test') {
                         environment {
@@ -187,7 +196,7 @@ pipeline {
                                 sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/tests/WE2E/log.*'
                                 // Remove the data sets from the experiments directory to conserve disk space
                                 sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf'
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: '*_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }

.cicd/scripts/srw_ftest.sh

@@ -7,7 +7,7 @@
 # We want this following a normal SRW build in an attempt to exercise environment setup, modules,
 # data sets, and workflow scripts, without using too much time nor account resources.
 # We hope to catch any snags that might prevent WE2E fundamental testing, which follows this gate.
-# NOTE: At this time, this script is a placeholder for functional test framework.
+# NOTE:
 # At this time, we are leaving the exercise of graphical plotting for a later stage, perhaps WE2E state.
 #
 # Required:
@@ -71,19 +71,27 @@ echo "DATA_LOCATION=${DATA_LOCATION}"
 sed "s|^task_get_extrn_ics:|task_get_extrn_ics:\n  EXTRN_MDL_SOURCE_BASEDIR_ICS: ${DATA_LOCATION}/FV3GFS/grib2/2019061518|1" -i ush/config.yaml
 sed "s|^task_get_extrn_lbcs:|task_get_extrn_lbcs:\n  EXTRN_MDL_SOURCE_BASEDIR_LBCS: ${DATA_LOCATION}/FV3GFS/grib2/2019061518|1" -i ush/config.yaml
 
+# Use staged data for HPSS supported machines
+sed 's|^platform:|platform:\n  EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml
+
 # Activate the workflow environment ...
 source etc/lmod-setup.sh ${platform,,}
 module use modulefiles
 module load build_${platform,,}_${SRW_COMPILER}
 module load wflow_${platform,,}
 
 [[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate regional_workflow
+conda activate workflow_tools
 set -e -u
 
+export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src
+
+# Adjust for strict limitation of stack size 
+sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml
+
 cd ${workspace}/ush
         # Consistency check ...
-        ./config_utils.py -c ./config.yaml -v ./config_defaults.yaml -k "(\!rocoto\b)"
+        #./config_utils.py -c ./config.yaml -v ./config_defaults.yaml -k "(\!rocoto\b)"
         # Generate workflow files ...
         ./generate_FV3LAM_wflow.py
 cd ${workspace}
@@ -115,16 +123,24 @@ export OMP_NUM_THREADS=1
 )
 set +x
 
-echo "# Try the first few simple SRW tasks ..."
 results_file=${workspace}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt
 rm -f ${results_file}
+
 status=0
-for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
+
+# Limit to machines that are fully ready
+deny_machines=( gaea )
+if [[ ${deny_machines[@]} =~ ${platform,,} ]] ; then
+    echo "# Deny ${platform} - incomplete configuration." | tee -a ${results_file}
+else
+    echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file}
+    for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
                 echo -n "./$task.sh ... "
                 ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))"
                 # stop at the first sign of trouble ...
                 [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file}
-done
+    done
+fi
 
 # Set exit code to number of failures
 set +e