diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index e11f253d13d..5069bb18be0 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -6,7 +6,7 @@ def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', gaeac6: 'Gaeac6-EMC'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/proj-shared/global/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/glopara/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -120,7 +120,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - sh(script: './build_compute.sh all') // build the global-workflow executables + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" if ( fileExists("logs/error.logs") ) { diff --git a/ci/platforms/config.gaeac5 b/ci/platforms/config.gaeac5 index 675d1b483b6..1223bb1cbb5 100644 --- a/ci/platforms/config.gaeac5 +++ b/ci/platforms/config.gaeac5 @@ -1,6 +1,7 @@ #!/usr/bin/bash -export GFS_CI_ROOT=/gpfs/f5/ufs-ard/world-shared/global/GFS_CI_ROOT -export ICSDIR_ROOT=/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDI + +export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT +export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR export HPC_ACCOUNT=ufs-ard export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.gaeac6 b/ci/platforms/config.gaeac6 index 2680a0727ef..bda69c6a98d 100644 --- a/ci/platforms/config.gaeac6 +++ b/ci/platforms/config.gaeac6 @@ -4,7 +4,7 @@ export GFS_CI_ROOT=/ncrc/proj/nggps_emc/${USER}/GFS_CI_CD export ICSDIR_ROOT=/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR # CI BASH test directories -export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/Bash +export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI # Jenkins directories export JENKINS_AGENT_LANUCH_DIR=${GFS_CI_ROOT}/Jenkins/agent diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera index 09e2e28ddb3..cf0ca640e8a 100644 --- a/ci/platforms/config.hera +++ b/ci/platforms/config.hera @@ -1,12 +1,20 @@ #!/usr/bin/bash -export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT +export GFS_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR +# Jenkins directories +export JENKINS_AGENT_LANUCH_DIR=${GFS_CI_ROOT}/Jenkins/agent +export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace +# JENKINS custom_workspace directory where CI jobs are run +# /scratch1/NCEPDEV/global/glopara/CI + +# CTest functional test directories for pre stagged input data export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR -export HPC_ACCOUNT=nems + +# CI BASH test directories +export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI + +export HPC_ACCOUNT=fv3-cpu export max_concurrent_cases=5 export max_concurrent_pr=4 - -export JENKINS_AGENT_LANUCH_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/Jenkins -export JENKINS_WORK_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness diff --git a/ci/platforms/config.hercules b/ci/platforms/config.hercules index 4d5af21c786..4e729d3afba 100644 --- a/ci/platforms/config.hercules +++ b/ci/platforms/config.hercules @@ -2,9 +2,17 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR -export HPC_ACCOUNT=fv3-cpu -export max_concurrent_cases=5 -export max_concurrent_pr=4 +# Jenkins directories export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS + +# CTest functional test directories for pre stagged input data +export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR + +# CI BASH test directories +export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI + +export HPC_ACCOUNT=fv3-cpu +export max_concurrent_cases=5 +export max_concurrent_pr=4 diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index dbaad7f3d1d..04c8d1c08c6 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -2,10 +2,17 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR + +# Jenkins directories +export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry +export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS + +# CI BASH test directories +export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI + +# CTest functional test directories for pre stagged input data export STAGED_TESTS_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_TESTS_DIR + export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 - -export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry -export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index a89a6610424..be61690588a 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -64,7 +64,7 @@ else echo "rocotocheck being used from ${rocotocheck}" fi -pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db" +pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then @@ -82,10 +82,10 @@ fi for pr in ${pr_list}; do id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') - output_ci="${GFS_CI_ROOT}/PR/${pr}/output_runtime_${id}" - output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_runtime_single.log" + output_ci="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_${id}" + output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_single.log" echo "Processing Pull Request #${pr} and looking for cases" - pr_dir="${GFS_CI_ROOT}/PR/${pr}" + pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" # If there is no RUNTESTS dir for this PR then cases have not been made yet if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 2698fb41abd..f4cb6cf9ad0 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -67,7 +67,7 @@ export GH # query repo and get list of open PRs with tags {machine}-CI ############################################################ -pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db" +pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" if [[ ! -f "${pr_list_dbfile}" ]]; then "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}" @@ -76,10 +76,8 @@ fi pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true for pr in ${pr_list}; do - pr_dir="${GFS_CI_ROOT}/PR/${pr}" - if [[ ! -d "${pr_dir}" ]]; then - mkdir -p "${pr_dir}" - fi + pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" + [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}" db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}") output_ci_single="${pr_dir}/output_single.log" ############################################################# @@ -164,9 +162,9 @@ for pr in ${pr_list}; do continue fi id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') - pr_dir="${GFS_CI_ROOT}/PR/${pr}" + pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" output_ci="${pr_dir}/output_ci_${id}" - output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log" + output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_single.log" driver_build_PID=$$ driver_build_HOST=$(hostname -s) "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" @@ -284,4 +282,4 @@ done # looping over each open and labeled PR # scrub working directory for older files ########################################## # -#find "${GFS_CI_ROOT}/PR/*" -maxdepth 1 -mtime +3 -exec rm -rf {} \; +#find "${GFS_BASH_CI_ROOT}/PR/*" -maxdepth 1 -mtime +3 -exec rm -rf {} \; diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh index 3193cc98ed1..6fac3d3e917 100755 --- a/ci/scripts/driver_weekly.sh +++ b/ci/scripts/driver_weekly.sh @@ -61,7 +61,7 @@ set -x # Create a new branch from develop and move yaml files ######################################################### branch="weekly_ci_$(date +%Y%m%d)" -develop_dir="${GFS_CI_ROOT}/develop_weekly" +develop_dir="${GFS_BASH_CI_ROOT}/develop_weekly" echo "Creating new branch ${branch} from develop on ${MACHINE_ID} in ${develop_dir}" rm -Rf "${develop_dir}" mkdir -p "${develop_dir}" @@ -114,5 +114,5 @@ for label in "${PULL_REQUEST_LABELS[@]}" do "${GH}" pr edit --add-label "${label}" done -cd "${GFS_CI_ROOT}" +cd "${GFS_BASH_CI_ROOT}" rm -Rf "${develop_dir}" diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index 2da5fa2681f..efbd7efcd99 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -44,7 +44,7 @@ else echo "rocotorun being used from ${rocotorun}" fi -pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db" +pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then @@ -64,7 +64,7 @@ fi for pr in ${pr_list}; do echo "Processing Pull Request #${pr} and looking for cases" - pr_dir="${GFS_CI_ROOT}/PR/${pr}" + pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" # If the directory RUNTESTS is not present then # setupexpt.py has no been run yet for this PR if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 56b0571adca..5922243893b 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -177,3 +177,10 @@ function cleanup_experiment() { rm -Rf "${EXPDIR}/${pslot:?}" rm -Rf "${STMP}/RUNDIRS/${pslot:?}" } + +function build_compute () { + + source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" + "${HOMEgfs}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all + +} diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 539ba4ca99d..003695d9d81 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -110,8 +110,8 @@ fi ${GH} --version export GH -check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true -if [[ "${check_mark}" != "✓" ]]; then +check_token=$("${GH}" auth status 2>&1 | grep "Token:") || true +if [[ "${check_token}" != *"*****"* ]]; then echo "gh not authenticating with emcbot token" exit 1 fi @@ -124,6 +124,7 @@ else exit 1 fi cd "${JENKINS_AGENT_LANUCH_DIR}" +echo "Entered directory ${PWD}" if ! [[ -f agent.jar ]]; then curl -sO "${controller_url}/jnlpJars/agent.jar" @@ -134,6 +135,7 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist" exit 1 fi + JENKINS_TOKEN=$(cat "${controller_user_auth_token}") cat << EOF > parse.py @@ -152,8 +154,12 @@ check_node_online() { echo "ERROR: Jenkins controller not reachable. Exiting with error." exit 1 fi + if [[ "${curl_response}" == *"Token expired"* ]]; then + echo "ERROR: Jenkins Token expired" + exit 1 + fi echo -n "${curl_response}" > curl_response - ./parse.py curl_response + ONLINE_STATUS=$(./parse.py curl_response) } lauch_agent () { @@ -170,7 +176,8 @@ if [[ "${force_launch}" == "True" ]]; then exit fi -offline=$(set -e; check_node_online) +check_node_online +offline="${ONLINE_STATUS}" if [[ "${offline}" != "False" ]]; then if [[ "${skip_wait}" != "True" ]]; then