From 9e9b628671f905754dcf1aaa070d69dd7646a569 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 19:43:51 +0000 Subject: [PATCH 01/10] adding a persistent lanuching tool for Jenkins Java Agents --- ci/scripts/utils/launch_java_agent.sh | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 ci/scripts/utils/launch_java_agent.sh diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh new file mode 100755 index 00000000000..f9567ab9ec8 --- /dev/null +++ b/ci/scripts/utils/launch_java_agent.sh @@ -0,0 +1,58 @@ + +#!/bin/env bash +set -ex + +controller_url="https://jenkins.epic.oarcloud.noaa.gov" +controller_user="terry.mcguinness" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" +host=$(hostname) + +######################################################################### +# Set up runtime environment varibles for accounts on supproted machines +######################################################################### + +source "${HOMEgfs}/ush/detect_machine.sh" +case ${MACHINE_ID} in + hera | orion | hercules | wcoss2) + echo "Launch Jenkins Java Controler on ${MACHINE_ID}" + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac + +LOG=lanuched_agent-$(date +%Y%m%d%M).log +rm -f "${LOG}" + +source "${HOMEgfs}/ush/module-setup.sh" +module use "${HOMEgfs}/modulefiles" +module load "module_gwsetup.${MACHINE_ID} + +JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin +JAVA="${JAVA_HOME}/java" +echo "JAVA VERSION: " +${JAVA} -version + +export GH="${HOME}/bin/gh" +command -v $GH +$GH --version + +cd "${JENKINS_AGENT_LANUCH_DIR}" + +if ! [ -f agent.jar ]; then + curl -sO "${controller_url}/jnlpJars/agent.jar" +fi + +JENKINS_TOKEN=$(cat jenkins_token) + +offline=$(curl --silent -u "${contoller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') + +echo "Jenkins Agent offline setting: ${offline}" + +exit 0 + +command="nohup ${JAVA} -jar agent.jar -jnlpUrl https://jenkins.epic.oarcloud.noaa.gov/computer/Hera%2DEMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir /scratch1/NCEPDEV/global/Terry.McGuinness" +echo -e "Lanuching Jenkins Agent on $host with the command:\n${command}" >& "${LOG}" +${command} >> "${LOG}" 2>&1 & +ps -efx | grep agent.jar >> "${LOG}" 2>&1 From 7b3f9d22ef61b0621d6323d36f7955baa6cd411d Mon Sep 17 00:00:00 2001 From: "Terry.McGuinness" Date: Tue, 28 May 2024 20:23:12 +0000 Subject: [PATCH 02/10] got test node working --- ci/scripts/utils/launch_java_agent.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index f9567ab9ec8..271c00c83ab 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -1,10 +1,9 @@ - #!/bin/env bash -set -ex +set -e controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user="terry.mcguinness" -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) ######################################################################### @@ -27,9 +26,9 @@ rm -f "${LOG}" source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID} +module load "module_gwsetup.${MACHINE_ID}" -JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin +JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin" JAVA="${JAVA_HOME}/java" echo "JAVA VERSION: " ${JAVA} -version @@ -38,6 +37,7 @@ export GH="${HOME}/bin/gh" command -v $GH $GH --version +echo "JENKINS_AGENT_LANUCH_DIR: ${JENKINS_AGENT_LANUCH_DIR}" cd "${JENKINS_AGENT_LANUCH_DIR}" if ! [ -f agent.jar ]; then @@ -46,7 +46,7 @@ fi JENKINS_TOKEN=$(cat jenkins_token) -offline=$(curl --silent -u "${contoller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') +offline=$(curl --silent -u "${controller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') echo "Jenkins Agent offline setting: ${offline}" From 50ae868fc360fe699507bf5a99f3b5f24aa52eaf Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 20:41:54 +0000 Subject: [PATCH 03/10] added machine specifics into config file in ci for lanuching Jenkins --- ci/platforms/config.hera | 3 +++ ci/scripts/utils/launch_java_agent.sh | 18 +++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera index 7e85bb33da8..6d3e43c820f 100644 --- a/ci/platforms/config.hera +++ b/ci/platforms/config.hera @@ -5,3 +5,6 @@ export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 + +export JENKINS_AGENT_LANUCH_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/Jenkins +export JENKINS_WORK_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 271c00c83ab..fd77d15cf9a 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -47,12 +47,16 @@ fi JENKINS_TOKEN=$(cat jenkins_token) offline=$(curl --silent -u "${controller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') - echo "Jenkins Agent offline setting: ${offline}" -exit 0 - -command="nohup ${JAVA} -jar agent.jar -jnlpUrl https://jenkins.epic.oarcloud.noaa.gov/computer/Hera%2DEMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir /scratch1/NCEPDEV/global/Terry.McGuinness" -echo -e "Lanuching Jenkins Agent on $host with the command:\n${command}" >& "${LOG}" -${command} >> "${LOG}" 2>&1 & -ps -efx | grep agent.jar >> "${LOG}" 2>&1 +if [ "${offline}" == "true" ]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on $host" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Lanuching Jenkins Agent on $host with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" +else + echo "Jenkins Agent is online (nothing done)" +fi From 88d7f97e4d0e96e66104e6aaea82b00db56a60b6 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 20:47:11 +0000 Subject: [PATCH 04/10] spelled source wrong --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index fd77d15cf9a..61e46a330e0 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -27,6 +27,7 @@ rm -f "${LOG}" source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" +source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin" JAVA="${JAVA_HOME}/java" @@ -37,7 +38,6 @@ export GH="${HOME}/bin/gh" command -v $GH $GH --version -echo "JENKINS_AGENT_LANUCH_DIR: ${JENKINS_AGENT_LANUCH_DIR}" cd "${JENKINS_AGENT_LANUCH_DIR}" if ! [ -f agent.jar ]; then From e80843acbf71ff65a9151fde2e607e6838ccb5f6 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 21:08:43 +0000 Subject: [PATCH 05/10] added lanuch config for hercules and orion --- ci/platforms/config.hercules | 3 +++ ci/platforms/config.orion | 3 +++ ci/scripts/utils/launch_java_agent.sh | 14 ++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/ci/platforms/config.hercules b/ci/platforms/config.hercules index 12846c18a4b..52bb69a879f 100644 --- a/ci/platforms/config.hercules +++ b/ci/platforms/config.hercules @@ -5,3 +5,6 @@ export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 + +export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry +export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS \ No newline at end of file diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index 5a7596001b4..41ce2def37a 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -5,3 +5,6 @@ export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 + +export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry +export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS \ No newline at end of file diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 61e46a330e0..bc317c899b2 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -30,6 +30,14 @@ module load "module_gwsetup.${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin" +if [ ! -d "${JAVA_HOME}" ]; then + JAVA_HOME=/usr/lib/jvm/jre-17 + if [ ! -d "${JAVA_HOME}" ]; then + echo "ERROR: JAVA_HOME not found. Exiting with error." + exit 1 + fi +fi + JAVA="${JAVA_HOME}/java" echo "JAVA VERSION: " ${JAVA} -version @@ -38,6 +46,12 @@ export GH="${HOME}/bin/gh" command -v $GH $GH --version +if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then + echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" +else + echo "ERROR: Jenkins Agent Lanuch Directory not found. Exiting with error." + exit 1 +fi cd "${JENKINS_AGENT_LANUCH_DIR}" if ! [ -f agent.jar ]; then From c2c79adfcca9cf36dcf4828a474c946467c87efa Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 21:24:53 +0000 Subject: [PATCH 06/10] updated configured path to launch dir --- ci/scripts/utils/launch_java_agent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index bc317c899b2..dee77777ed5 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -29,7 +29,7 @@ module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" -JAVA_HOME="${JENKINS_AGENT_JAVA_HOME}/JAVA/jdk-17.0.10/bin" +JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" if [ ! -d "${JAVA_HOME}" ]; then JAVA_HOME=/usr/lib/jvm/jre-17 if [ ! -d "${JAVA_HOME}" ]; then @@ -38,7 +38,7 @@ if [ ! -d "${JAVA_HOME}" ]; then fi fi -JAVA="${JAVA_HOME}/java" +JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version From 8b41ace456280aa86336f2d204811251b1af8aa8 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 21:45:26 +0000 Subject: [PATCH 07/10] made a few shell norm updates --- ci/scripts/utils/launch_java_agent.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index dee77777ed5..72ff29b8da4 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -43,7 +43,7 @@ echo "JAVA VERSION: " ${JAVA} -version export GH="${HOME}/bin/gh" -command -v $GH +command -v "${GH}" $GH --version if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then @@ -54,19 +54,20 @@ else fi cd "${JENKINS_AGENT_LANUCH_DIR}" -if ! [ -f agent.jar ]; then +if ! [[ -f agent.jar ]]; then curl -sO "${controller_url}/jnlpJars/agent.jar" fi JENKINS_TOKEN=$(cat jenkins_token) -offline=$(curl --silent -u "${controller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') +# +offline=$(curl --silent -u "${controller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true echo "Jenkins Agent offline setting: ${offline}" -if [ "${offline}" == "true" ]; then - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on $host" +if [[ "${offline}" == "true" ]]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on $host with the command:\n${command}" >& "${LOG}" + echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" ${command} >> "${LOG}" 2>&1 & nohup_PID=$! echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 From 0f203a80ed7a980ac739adaf44580fef9aa71d7d Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 21:47:54 +0000 Subject: [PATCH 08/10] made two more shell norm updates --- ci/scripts/utils/launch_java_agent.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 72ff29b8da4..35979ec56e1 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -30,9 +30,9 @@ module load "module_gwsetup.${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" -if [ ! -d "${JAVA_HOME}" ]; then +if [[ ! -d "${JAVA_HOME}" ]]; then JAVA_HOME=/usr/lib/jvm/jre-17 - if [ ! -d "${JAVA_HOME}" ]; then + if [[ ! -d "${JAVA_HOME}" ]]; then echo "ERROR: JAVA_HOME not found. Exiting with error." exit 1 fi @@ -44,7 +44,7 @@ ${JAVA} -version export GH="${HOME}/bin/gh" command -v "${GH}" -$GH --version +${GH} --version if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" @@ -61,7 +61,7 @@ fi JENKINS_TOKEN=$(cat jenkins_token) # -offline=$(curl --silent -u "${controller_user}:$JENKINS_TOKEN" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true +offline=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true echo "Jenkins Agent offline setting: ${offline}" if [[ "${offline}" == "true" ]]; then From ed5b2f7c1299d13084c35d719a07ca84d8596bee Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 28 May 2024 21:49:29 +0000 Subject: [PATCH 09/10] trailing spaces needed --- ci/platforms/config.hercules | 2 +- ci/platforms/config.orion | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/platforms/config.hercules b/ci/platforms/config.hercules index 52bb69a879f..5329adae49b 100644 --- a/ci/platforms/config.hercules +++ b/ci/platforms/config.hercules @@ -7,4 +7,4 @@ export max_concurrent_cases=5 export max_concurrent_pr=4 export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry -export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS \ No newline at end of file +export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index 41ce2def37a..5171373127e 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -7,4 +7,4 @@ export max_concurrent_cases=5 export max_concurrent_pr=4 export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry -export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS \ No newline at end of file +export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS From 3e1358057b2add91f476cbd49d9daef300040b6f Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 29 May 2024 10:44:25 -0400 Subject: [PATCH 10/10] Update ci/scripts/utils/launch_java_agent.sh ooo Co-authored-by: Walter Kolczynski - NOAA --- ci/scripts/utils/launch_java_agent.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 35979ec56e1..a4e664724be 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -13,12 +13,12 @@ host=$(hostname) source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2) - echo "Launch Jenkins Java Controler on ${MACHINE_ID}" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; + echo "Launch Jenkins Java Controler on ${MACHINE_ID}" + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; esac LOG=lanuched_agent-$(date +%Y%m%d%M).log