CI Refactoring and STALLED case detection

ci/Jenkinsfile

@@ -4,13 +4,16 @@ def HOME = 'none'
 def caseList = ''
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
+def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
+def STATUS = 'Passed'
 
 pipeline {
+
     agent { label 'built-in' }
 
     options {
         skipDefaultCheckout()
-        parallelsAlwaysFailFast()
+        //parallelsAlwaysFailFast()
     }
 
     stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
@@ -46,10 +49,7 @@ pipeline {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
                         sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
-                        pullRequest.addLabel("CI-${Machine}-Building")
-                        if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
-                            pullRequest.removeLabel("CI-${Machine}-Ready")
-                        }
+                        sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
                     }
                     echo "Building and running on ${Machine} in directory ${HOME}"
                 }
@@ -108,26 +108,23 @@ pipeline {
                                                             catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
                                                         }
                                                     }
-                                                    repo_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
-                                                    gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
                                                     try {
-                                                        pullRequest.comment("Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
+                                                        sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+                                                        gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
                                                     } catch (Exception error_comment) {
                                                         echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                     }
                                                     error("Failed to build system on ${Machine}")
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            // sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {
                                         try {
-                                            if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
-                                               pullRequest.removeLabel("CI-${Machine}-Building")
-                                            }
-                                            pullRequest.addLabel("CI-${Machine}-Running")
+                                            sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
                                         }
@@ -144,6 +141,7 @@ pipeline {
         }
 
         stage('Run Tests') {
+            failFast false 
             matrix {
                 agent { label "${machine}-emc" }
                 axes {
@@ -175,19 +173,22 @@ pipeline {
                         when {
                             expression { return caseList.contains(Case) }
                         }
+                        failFast false
                         steps {
                             script {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
-                                pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+                                def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+                                def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
+                                sh(script: " rm -f ${error_file}")
                                 try {
-                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
+                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
                                 } catch (Exception error_experment) {
-                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
+                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                     ws(HOME) {
                                         def error_logs = ""
                                         def error_logs_message = ""
-                                        if (fileExists("RUNTESTS/error.logs")) {
-                                            def fileContent = readFile 'RUNTESTS/error.logs'
+                                        if (fileExists(error_file)) {
+                                            def fileContent = readFile error_file
                                             def lines = fileContent.readLines()
                                             for (line in lines) {
                                                 echo "archiving: ${line}"
@@ -201,15 +202,21 @@ pipeline {
                                                     }
                                                 }
                                             }
-                                            repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim()
-                                            gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
                                             try {
-                                                pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
+                                                gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+                                                sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+                                                sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
                                             } catch (Exception error_comment) {
                                                 echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                             }
                                         } else {
-                                            echo "No error logs found for failed cases in $HOME/RUNTESTS/error.logs"
+                                            echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
+                                        }
+                                        STATUS = 'Failed'
+                                        try {
+                                            sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+                                        } catch (Exception e) {
+                                            echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                                         }
                                         error("Failed to run experiments ${Case} on ${Machine}")
                                     }
@@ -220,49 +227,23 @@ pipeline {
                 }
             }
         }
-    }
-
-    post {
-        always {
-            script {
-                if(env.CHANGE_ID) {
-                    try {
-                        for (label in pullRequest.labels) {
-                            if (label.contains("${Machine}")) {
-                                pullRequest.removeLabel(label)
-                            }
-                        }
-                    }  catch (Exception e) {
-                    echo "Failed to remove labels: ${e.getMessage()}"
-                    }
-                }
-            }
-        }
-        success {
-            script {
-                if(env.CHANGE_ID) {
-                    try {
-                        pullRequest.addLabel("CI-${Machine}-Passed")
-                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                        pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
-                    } catch (Exception e) {
-                        echo "Failed to add success label or comment: ${e.getMessage()}"
-                    }
+        stage( 'FINALIZE' ) {
+            when { 
+                expression {
+                   STATUS == 'Passed'
                 }
             }
-        }
-        failure {
-            script {
-                if(env.CHANGE_ID) {
+            agent { label "${machine}-emc" }
+            steps {
+                script {
                     try {
-                        pullRequest.addLabel("CI-${Machine}-Failed")
-                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                        pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+                        sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
                     } catch (Exception e) {
-                        echo "Failed to add failure label or comment: ${e.getMessage()}"
+                        echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                     }
                 }
             }
-        }
+        }   
     }
 }

ci/scripts/check_ci.sh

@@ -68,7 +68,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -100,7 +100,7 @@ for pr in ${pr_list}; do
     sed -i "1 i\`\`\`" "${output_ci}"
     sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
@@ -133,28 +133,40 @@ for pr in ${pr_list}; do
     if [[ ! -f "${db}" ]]; then
        continue
     fi
-    rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true
-    num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true
-    num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true
-    # num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
-    echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true
-    num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true
-    if [[ ${num_failed} -ne 0 ]]; then
-      "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
-      error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
-      {
-       echo "Experiment ${pslot}  *** FAILED *** on ${MACHINE_ID^}"
-       echo "Experiment ${pslot}  with ${num_failed} tasks failed at $(date +'%D %r')" || true
-       echo "Error logs:"
-       echo "${error_logs}"
-      } >> "${output_ci}"
-      sed -i "1 i\`\`\`" "${output_ci}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
-      cancel_all_batch_jobs "${pr_dir}/RUNTESTS/"
-      break
+
+    set +e
+    rocoto_state="$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}")"
+    rocoto_error=$?
+    rm -f "${output_ci_single}"
+    if [[ "${rocoto_error}" -ne 0 ]]; then
+        "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
+        if [[ "${rocoto_state}" == "STALLED" ]]; then
+          # shellcheck disable=SC2312
+          "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')"
+          "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+          cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
+          exit "${rocoto_error}"
+        fi
+        error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
+        # shellcheck disable=SC2086
+        ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
+        # shellcheck disable=SC2086
+        gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")"
+        {
+          echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" || true
+          echo ""
+          echo "Error logs:"
+          echo "\`\`\`"
+          echo "${error_logs}"
+          echo "\`\`\`"
+          echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})"
+        } >> "${output_ci_single}"
+        "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
+        "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+        cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
+        exit "${rocoto_error}"
     fi
-    if [[ "${num_done}" -eq  "${num_cycles}" ]]; then
+    if [[ "${rocoto_state}" == "DONE" ]]; then
       #Remove Experment cases that completed successfully
       rm -Rf "${pslot_dir}"
       rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
@@ -164,7 +176,6 @@ for pr in ${pr_list}; do
       echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}"
       echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
-
     fi
   done
 done

ci/scripts/clone-build_ci.sh

@@ -48,7 +48,7 @@ git clone "${REPO_URL}"
 cd global-workflow || exit 1
 
 # checkout pull request
-"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules
+"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules -j 4
 HOMEgfs="${PWD}"
 source "${HOMEgfs}/ush/detect_machine.sh"
 
@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -g  >> log.build 2>&1
+./build_all.sh -gk  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')
@@ -83,6 +83,7 @@ if [[ ${build_status} != 0 ]]; then
     echo "Build: *** FAILED ***"
     echo "Build: Failed at ${DATE}"
     cat "${PWD}/log.build"
+    cat "${PWD}/logs/error.logs"
   } >> "${outfile}"
   exit "${build_status}"
 else