Migrate CI Jenkins to role account on Hera

ci/Jenkinsfile

@@ -6,7 +6,7 @@ def CI_CASES = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persistent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', gaeac6: 'Gaeac6-EMC']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/glopara/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/proj-shared/global/CI']
 def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -120,7 +120,7 @@ pipeline {
                             def error_logs_message = ""
                             dir("${HOMEgfs}/sorc") {
                                 try {
-                                    sh(script: './build_compute.sh all')  // build the global-workflow executables
+                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables
                                 } catch (Exception error_build) {
                                     echo "Failed to build global-workflow: ${error_build.getMessage()}"
                                     if ( fileExists("logs/error.logs") ) {

ci/platforms/config.gaeac5

@@ -1,6 +1,7 @@
 #!/usr/bin/bash
-export GFS_CI_ROOT=/gpfs/f5/ufs-ard/world-shared/global/GFS_CI_ROOT
-export ICSDIR_ROOT=/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDI
+
+export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
+export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
 export HPC_ACCOUNT=ufs-ard
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.gaeac6

@@ -4,7 +4,7 @@ export GFS_CI_ROOT=/ncrc/proj/nggps_emc/${USER}/GFS_CI_CD
 export ICSDIR_ROOT=/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR
 
 # CI BASH test directories
-export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/Bash
+export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI
 
 # Jenkins directories
 export JENKINS_AGENT_LANUCH_DIR=${GFS_CI_ROOT}/Jenkins/agent

ci/platforms/config.hera

@@ -1,12 +1,20 @@
 #!/usr/bin/bash
 
-export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
+export GFS_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
 
+# Jenkins directories
+export JENKINS_AGENT_LANUCH_DIR=${GFS_CI_ROOT}/Jenkins/agent
+export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace
+# JENKINS cutome_workspace directory where CI jobs are run
+# /scratch1/NCEPDEV/global/glopara/CI
+
+# CTest functional test directories for pre stagged input data
 export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
-export HPC_ACCOUNT=nems
+
+# CI BASH test directories
+export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI
+
+export HPC_ACCOUNT=fv3-cpu
 export max_concurrent_cases=5
 export max_concurrent_pr=4
-
-export JENKINS_AGENT_LANUCH_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/Jenkins
-export JENKINS_WORK_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness

ci/platforms/config.hercules

@@ -2,9 +2,17 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export HPC_ACCOUNT=fv3-cpu
-export max_concurrent_cases=5
-export max_concurrent_pr=4
 
+# Jenkins directories
 export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
 export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS
+
+# CTest functional test directories for pre stagged input data
+export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
+
+# CI BASH test directories
+export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI
+
+export HPC_ACCOUNT=fv3-cpu
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/platforms/config.orion

@@ -2,10 +2,17 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+
+# Jenkins directories
+export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
+export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS
+
+# CI BASH test directories
+export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI
+
+# CTest functional test directories for pre stagged input data
 export STAGED_TESTS_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_TESTS_DIR
+
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
-
-export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
-export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS

ci/scripts/check_ci.sh

@@ -64,7 +64,7 @@ else
   echo "rocotocheck being used from ${rocotocheck}"
 fi
 
-pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
+pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
@@ -82,10 +82,10 @@ fi
 
 for pr in ${pr_list}; do
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
-  output_ci="${GFS_CI_ROOT}/PR/${pr}/output_runtime_${id}"
-  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_runtime_single.log"
+  output_ci="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_${id}"
+  output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_single.log"
   echo "Processing Pull Request #${pr} and looking for cases"
-  pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}"
 
   # If there is no RUNTESTS dir for this PR then cases have not been made yet
   if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then

ci/scripts/driver.sh

@@ -67,7 +67,7 @@ export GH
 # query repo and get list of open PRs with tags {machine}-CI
 ############################################################
 
-pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
+pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db"
 
 if [[ ! -f "${pr_list_dbfile}" ]]; then
   "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}"
@@ -76,10 +76,8 @@ fi
 pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true
 
 for pr in ${pr_list}; do
-  pr_dir="${GFS_CI_ROOT}/PR/${pr}"
-  if [[ ! -d "${pr_dir}" ]]; then
-      mkdir -p "${pr_dir}"
-  fi
+  pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}"
+  [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}"
   db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
   output_ci_single="${pr_dir}/output_single.log"
   #############################################################
@@ -164,9 +162,9 @@ for pr in ${pr_list}; do
       continue
   fi
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
-  pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}"
   output_ci="${pr_dir}/output_ci_${id}"
-  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log"
+  output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_single.log"
   driver_build_PID=$$
   driver_build_HOST=$(hostname -s)
   "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building"
@@ -284,4 +282,4 @@ done # looping over each open and labeled PR
 # scrub working directory for older files
 ##########################################
 #
-#find "${GFS_CI_ROOT}/PR/*" -maxdepth 1 -mtime +3 -exec rm -rf {} \;
+#find "${GFS_BASH_CI_ROOT}/PR/*" -maxdepth 1 -mtime +3 -exec rm -rf {} \;

ci/scripts/driver_weekly.sh

@@ -61,7 +61,7 @@ set -x
 # Create a new branch from develop and move yaml files
 #########################################################
 branch="weekly_ci_$(date +%Y%m%d)"
-develop_dir="${GFS_CI_ROOT}/develop_weekly"
+develop_dir="${GFS_BASH_CI_ROOT}/develop_weekly"
 echo "Creating new branch ${branch} from develop on ${MACHINE_ID} in ${develop_dir}"
 rm -Rf "${develop_dir}"
 mkdir -p "${develop_dir}"
@@ -114,5 +114,5 @@ for label in "${PULL_REQUEST_LABELS[@]}"
 do
   "${GH}" pr edit --add-label "${label}"
 done
-cd "${GFS_CI_ROOT}"
+cd "${GFS_BASH_CI_ROOT}"
 rm -Rf "${develop_dir}"

ci/scripts/run_ci.sh

@@ -44,7 +44,7 @@ else
   echo "rocotorun being used from ${rocotorun}"
 fi
 
-pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
+pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
@@ -64,7 +64,7 @@ fi
 
 for pr in ${pr_list}; do
   echo "Processing Pull Request #${pr} and looking for cases"
-  pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}"
   # If the directory RUNTESTS is not present then
   # setupexpt.py has no been run yet for this PR
   if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then

ci/scripts/utils/ci_utils.sh

@@ -177,3 +177,10 @@ function cleanup_experiment() {
     rm -Rf "${EXPDIR}/${pslot:?}"
     rm -Rf "${STMP}/RUNDIRS/${pslot:?}"
 }
+
+function build_compute () {
+
+  source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+  "${HOMEgfs}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all
+
+}

ci/scripts/utils/launch_java_agent.sh

@@ -110,8 +110,8 @@ fi
 ${GH} --version
 export GH
 
-check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
-if [[ "${check_mark}" != "✓" ]]; then
+check_token=$("${GH}" auth status 2>&1 | grep "Token:") || true
+if [[ "${check_token}" != *"*****"* ]]; then
   echo "gh not authenticating with emcbot token"
   exit 1
 fi
@@ -124,6 +124,7 @@ else
   exit 1
 fi
 cd "${JENKINS_AGENT_LANUCH_DIR}"
+echo "Entered directory ${PWD}"
 
 if ! [[ -f agent.jar ]]; then
   curl -sO "${controller_url}/jnlpJars/agent.jar"
@@ -134,6 +135,7 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then
    echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist"
    exit 1
 fi
+
 JENKINS_TOKEN=$(cat "${controller_user_auth_token}")
 
 cat << EOF > parse.py
@@ -152,8 +154,12 @@ check_node_online() {
        echo "ERROR: Jenkins controller not reachable. Exiting with error."
        exit 1
     fi
+    if [[ "${curl_response}" == *"Token expired"* ]]; then
+       echo "ERROR: Jenkins Token expired"
+       exit 1
+    fi
     echo -n "${curl_response}" > curl_response
-    ./parse.py curl_response
+    ONLINE_STATUS=$(./parse.py curl_response)
 }
 
 lauch_agent () {
@@ -170,7 +176,8 @@ if [[ "${force_launch}" == "True" ]]; then
   exit
 fi
 
-offline=$(set -e; check_node_online)
+check_node_online
+offline="${ONLINE_STATUS}"
 
 if [[ "${offline}" != "False" ]]; then
    if [[ "${skip_wait}" != "True" ]]; then