From 9be62c3d53283092529b71d75f7293251d8dc1a9 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 12 Jul 2024 13:40:38 -0500 Subject: [PATCH 01/29] hardened lauch_java_agent --- ci/scripts/utils/launch_java_agent.sh | 33 +++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index a4e664724be..6d98e8360e7 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -3,6 +3,8 @@ set -e controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user="terry.mcguinness" +controller_user_auth_token="jenkins_token" + HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) @@ -46,6 +48,14 @@ export GH="${HOME}/bin/gh" command -v "${GH}" ${GH} --version +check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') +if [[ "${check_mark}" != "✓" ]]; then + echo "gh not authenticating with emcbot token" + exit 1 +fi +echo "gh authenticating with emcbot TOKEN ok" + + if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" else @@ -56,15 +66,30 @@ cd "${JENKINS_AGENT_LANUCH_DIR}" if ! [[ -f agent.jar ]]; then curl -sO "${controller_url}/jnlpJars/agent.jar" + echo "Updated agent.jar downloaded" fi -JENKINS_TOKEN=$(cat jenkins_token) +if [[ ! -f "${controller_user_auth_token}" ]]; then + echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist" + exit 1 +fi -# -offline=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true +JENKINS_TOKEN=$(cat "${controller_user_auth_token}") +curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true +rm -f curl_response +echo -n "${curl_response}" > curl_response +echo -e "#!/usr/bin/env python +import json,sys +with open(sys.argv[1], 'r') as file: + data = json.load(file) +print(data[\"offline\"]) +" > parse.py +chmod u+x parse.py +offline=$(./parse.py curl_response) echo "Jenkins Agent offline setting: ${offline}" -if [[ "${offline}" == "true" ]]; then +if [[ "${offline}" != "False" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to try again" echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" From 303275f0bd8b14ca619b0875d0e7b49847303c1c Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 14:48:45 -0400 Subject: [PATCH 02/29] put curl api call into a function --- ci/scripts/utils/launch_java_agent.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 6d98e8360e7..998346e8a6f 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -75,9 +75,6 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then fi JENKINS_TOKEN=$(cat "${controller_user_auth_token}") -curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true -rm -f curl_response -echo -n "${curl_response}" > curl_response echo -e "#!/usr/bin/env python import json,sys with open(sys.argv[1], 'r') as file: @@ -85,8 +82,16 @@ with open(sys.argv[1], 'r') as file: print(data[\"offline\"]) " > parse.py chmod u+x parse.py -offline=$(./parse.py curl_response) -echo "Jenkins Agent offline setting: ${offline}" + +parse_json_response() { + curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true + echo -n "${curl_response}" > curl_response + $(./parse.py curl_response) +} + +offline=parse_json_response +echo "Jenkins Agent is offline: ${offline}" +exit if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to try again" From 4035b7026257d11a98d986249ceb929c293b3606 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 12 Jul 2024 14:09:54 -0500 Subject: [PATCH 03/29] added feature to wait five minutes after it has been determined that the node is down and checking again before lanuching agent --- ci/scripts/utils/launch_java_agent.sh | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 998346e8a6f..a085759b0a3 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -83,25 +83,32 @@ print(data[\"offline\"]) " > parse.py chmod u+x parse.py -parse_json_response() { +check_node_online() { + rm -p curl_response curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true echo -n "${curl_response}" > curl_response - $(./parse.py curl_response) + ./parse.py curl_response } -offline=parse_json_response -echo "Jenkins Agent is offline: ${offline}" -exit +offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then - echo "Jenkins Agent is offline. Waiting 5 more minutes to try again" - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" + if [[ "${1}" != "now" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" + sleep 300 + fi + offline=$(check_node_online) + if [[ "${offline}" != "False" ]]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" + else + echo "Jenkins Agent is online (nothing done)" + fi else echo "Jenkins Agent is online (nothing done)" fi From eec33698e746979627882eeb3bcbd94fe20ed514 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 15:21:19 -0400 Subject: [PATCH 04/29] added headher to launch script --- ci/scripts/utils/launch_java_agent.sh | 100 +++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index a085759b0a3..fdda9e91cd7 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -1,4 +1,30 @@ -#!/bin/env bash +#!/bin/bash + +# ============================================================================== +# Script Name: launch_java_agent.sh +# +# Description: This script automates the process of launching a Jenkins agent +# on a specified machine. It ensures that the necessary +# prerequisites are met, such as the availability of JAVA_HOME, +# the Jenkins agent launch directory, and proper authentication +# with GitHub. +# +# It then proceeds to check if the Jenkins node is online and +# decides whether to launch the Jenkins agent based on the node's +# status. The agent is launched in the background, +# and its PID is logged for reference. +# +# Prerequisites: JAVA_HOME must be set to a valid JDK installation. +# GitHub CLI (gh) must be installed and authenticated. +# Jenkins agent launch directory must exist and be specified. +# Jenkins controller URL and authentication token must be provided. +# +# Usage: ./launch_java_agent.sh [now] +# The optional 'now' argument forces the script to launch the Jenkins +# agent immediately without waiting, even if the node is detected as offline. +# +# ============================================================================== + set -e controller_url="https://jenkins.epic.oarcloud.noaa.gov" @@ -92,6 +118,78 @@ check_node_online() { offline=$(check_node_online) +if [[ "${offline}" != "False" ]]; then + if [[ "${1}" != "now" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" + sleep 300 + fi + offline=$(check_node_online) + if [[ "${offline}" != "False" ]]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" + else + echo "Jenkins Agent is online (nothing done)" + fi +else + echo "Jenkins Agent is online (nothing done)" +fi ./parse.py curl_response +} + +offline=$(check_node_online) + +if [[ "${offline}" != "False" ]]; then + if [[ "${1}" != "now" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" + sleep 300 + fi + offline=$(check_node_online) + if [[ "${offline}" != "False" ]]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" + else + echo "Jenkins Agent is online (nothing done)" + fi +else + echo "Jenkins Agent is online (nothing done)" +fi ./parse.py curl_response +} + +offline=$(check_node_online) + +if [[ "${offline}" != "False" ]]; then + if [[ "${1}" != "now" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" + sleep 300 + fi + offline=$(check_node_online) + if [[ "${offline}" != "False" ]]; then + echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" + else + echo "Jenkins Agent is online (nothing done)" + fi +else + echo "Jenkins Agent is online (nothing done)" +fi ./parse.py curl_response +} + +offline=$(check_node_online) + if [[ "${offline}" != "False" ]]; then if [[ "${1}" != "now" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" From a740da1da15badd67a657b86a18f9b098537afc0 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 15:42:52 -0400 Subject: [PATCH 05/29] added shell norm suggestions --- ci/scripts/utils/launch_java_agent.sh | 81 ++------------------------- 1 file changed, 6 insertions(+), 75 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index fdda9e91cd7..6b677bff4fb 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -74,14 +74,13 @@ export GH="${HOME}/bin/gh" command -v "${GH}" ${GH} --version -check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') +check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then echo "gh not authenticating with emcbot token" exit 1 fi echo "gh authenticating with emcbot TOKEN ok" - if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" else @@ -116,14 +115,18 @@ check_node_online() { ./parse.py curl_response } +set +e offline=$(check_node_online) +set -e if [[ "${offline}" != "False" ]]; then if [[ "${1}" != "now" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi + set +e offline=$(check_node_online) + set -e if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" @@ -137,76 +140,4 @@ if [[ "${offline}" != "False" ]]; then fi else echo "Jenkins Agent is online (nothing done)" -fi ./parse.py curl_response -} - -offline=$(check_node_online) - -if [[ "${offline}" != "False" ]]; then - if [[ "${1}" != "now" ]]; then - echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" - sleep 300 - fi - offline=$(check_node_online) - if [[ "${offline}" != "False" ]]; then - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" - else - echo "Jenkins Agent is online (nothing done)" - fi -else - echo "Jenkins Agent is online (nothing done)" -fi ./parse.py curl_response -} - -offline=$(check_node_online) - -if [[ "${offline}" != "False" ]]; then - if [[ "${1}" != "now" ]]; then - echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" - sleep 300 - fi - offline=$(check_node_online) - if [[ "${offline}" != "False" ]]; then - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" - else - echo "Jenkins Agent is online (nothing done)" - fi -else - echo "Jenkins Agent is online (nothing done)" -fi ./parse.py curl_response -} - -offline=$(check_node_online) - -if [[ "${offline}" != "False" ]]; then - if [[ "${1}" != "now" ]]; then - echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" - sleep 300 - fi - offline=$(check_node_online) - if [[ "${offline}" != "False" ]]; then - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" - else - echo "Jenkins Agent is online (nothing done)" - fi -else - echo "Jenkins Agent is online (nothing done)" -fi +fi \ No newline at end of file From ea851442d66b12ecdabc43d899e5bc67d88b0902 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 15:54:32 -0400 Subject: [PATCH 06/29] added shell norm for set e --- ci/scripts/utils/launch_java_agent.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 6b677bff4fb..df2c7cbadc4 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -115,18 +115,16 @@ check_node_online() { ./parse.py curl_response } -set +e -offline=$(check_node_online) set -e +offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then if [[ "${1}" != "now" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi - set +e - offline=$(check_node_online) set -e + offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" From 240ed9c1c419b601374c83ce4e0a18e6e8c88e9d Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 15:58:48 -0400 Subject: [PATCH 07/29] added shell suggestion for shopt -s inherit_errexit --- ci/scripts/utils/launch_java_agent.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index df2c7cbadc4..cf516252ff2 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -26,6 +26,7 @@ # ============================================================================== set -e +shopt -s inherit_errexit controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user="terry.mcguinness" @@ -115,7 +116,6 @@ check_node_online() { ./parse.py curl_response } -set -e offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then @@ -123,7 +123,6 @@ if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi - set -e offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" From f548c30f3a93b1df85c8d7e80dca4584af58dc3b Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 12 Jul 2024 16:04:26 -0400 Subject: [PATCH 08/29] fixed syntax error in rm arg and replaced -p with -f --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index cf516252ff2..2a4b6086f95 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -110,7 +110,7 @@ print(data[\"offline\"]) chmod u+x parse.py check_node_online() { - rm -p curl_response + rm -f curl_response curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true echo -n "${curl_response}" > curl_response ./parse.py curl_response From 3e6b671fb339d7d011ef46a7ccb5552b8a4a6ac8 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 10:02:57 -0400 Subject: [PATCH 09/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 2a4b6086f95..57653b63769 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/bin/env bash + +#shellcheck enable=check-set-e-suppressed # ============================================================================== # Script Name: launch_java_agent.sh From 773b07b9c42792ac4310020d031a6042691e3635 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 10:05:18 -0400 Subject: [PATCH 10/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 57653b63769..ea17d8e3ea1 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -107,7 +107,7 @@ echo -e "#!/usr/bin/env python import json,sys with open(sys.argv[1], 'r') as file: data = json.load(file) -print(data[\"offline\"]) +print(data.get('offline','True')) " > parse.py chmod u+x parse.py From b36510b44104282632228b44944662abc2ff3fce Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 10:05:53 -0400 Subject: [PATCH 11/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index ea17d8e3ea1..38363a1cbe5 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -121,7 +121,7 @@ check_node_online() { offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then - if [[ "${1}" != "now" ]]; then + if [[ "${1:-}" != "now" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi From d9e165fa72aef23e674e8e5c4c505d5f8bfc7830 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 10:06:02 -0400 Subject: [PATCH 12/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 38363a1cbe5..44abd3e0da2 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -127,7 +127,7 @@ if [[ "${offline}" != "False" ]]; then fi offline=$(check_node_online) if [[ "${offline}" != "False" ]]; then - echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}" + echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" ${command} >> "${LOG}" 2>&1 & From bd4ca78a2b6ba82af1ec6272624eb8327bf1f6c6 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 10:06:12 -0400 Subject: [PATCH 13/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 44abd3e0da2..3dfd8a64d98 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -129,7 +129,7 @@ if [[ "${offline}" != "False" ]]; then if [[ "${offline}" != "False" ]]; then echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" ${command} >> "${LOG}" 2>&1 & nohup_PID=$! echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 From 41b5af97ce29f969f9b7ec4a4d7fef0cbd1ea92f Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Mon, 15 Jul 2024 14:32:45 +0000 Subject: [PATCH 14/29] updated comprehensive documentation --- ci/scripts/utils/launch_java_agent.sh | 44 ++++++++++++++++++--------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 3dfd8a64d98..2aa4a7e62e2 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -5,21 +5,37 @@ # ============================================================================== # Script Name: launch_java_agent.sh # -# Description: This script automates the process of launching a Jenkins agent -# on a specified machine. It ensures that the necessary -# prerequisites are met, such as the availability of JAVA_HOME, -# the Jenkins agent launch directory, and proper authentication -# with GitHub. +# Description: +# This script automates the process of launching a Jenkins agent +# on a specified machine. It ensures that the necessary +# prerequisites are met, such as the availability of JAVA_HOME, +# the Jenkins agent launch directory, and proper authentication +# with GitHub. # -# It then proceeds to check if the Jenkins node is online and -# decides whether to launch the Jenkins agent based on the node's -# status. The agent is launched in the background, -# and its PID is logged for reference. +# It then proceeds to check if the Jenkins node is online and +# decides whether to launch the Jenkins agent based on the node's +# status. The agent is launched in the background, +# and its PID is logged for reference. # -# Prerequisites: JAVA_HOME must be set to a valid JDK installation. -# GitHub CLI (gh) must be installed and authenticated. -# Jenkins agent launch directory must exist and be specified. -# Jenkins controller URL and authentication token must be provided. +# Prerequisites: +# JAVA_HOME must be set to a valid JDK installation. +# Jenkins agent launch directory must exist and be specified. +# GitHub CLI (gh) must be installed and authenticated for messeging +# from the Jenkins controller to GitHub PR via shell commands. +# Jenkins agent launch directory must exist and be specified. +# TODO: Must use GitHub CLI v2.2.5 (newer versoins have issues) +# Jenkins controller URL and authentication token must be provided. +# jenkins-secret-file: +# Must be present in the Jenkins agent launch directory. +# This file contains the secret key for the Jenkins agent +# established by the Jenkins administrator for each Node. +# jenkins_token: +# Must be present in the Jenkins agent launch directory. +# This file contains the user authentication token for the Jenkins controller +# to use the Remote API. This token can be generated by the user +# on the Jenkins controller. +# controller_user: +# Must be set to the Jenkins controller username corresponing to the jenkins_token. # # Usage: ./launch_java_agent.sh [now] # The optional 'now' argument forces the script to launch the Jenkins @@ -139,4 +155,4 @@ if [[ "${offline}" != "False" ]]; then fi else echo "Jenkins Agent is online (nothing done)" -fi \ No newline at end of file +fi From 68b67de37c3cd6bb3a51836c98dfe386c228deae Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 15 Jul 2024 11:49:53 -0400 Subject: [PATCH 15/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 2aa4a7e62e2..c15e9bb78fe 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -1,7 +1,5 @@ #!/bin/env bash -#shellcheck enable=check-set-e-suppressed - # ============================================================================== # Script Name: launch_java_agent.sh # From fc9826b69f16bcaef4e0e8dcadc76568f4ef2b91 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Mon, 15 Jul 2024 15:59:39 +0000 Subject: [PATCH 16/29] nitpicks for PR review --- ci/scripts/utils/launch_java_agent.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index c15e9bb78fe..ecc2b945399 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -21,7 +21,7 @@ # GitHub CLI (gh) must be installed and authenticated for messeging # from the Jenkins controller to GitHub PR via shell commands. # Jenkins agent launch directory must exist and be specified. -# TODO: Must use GitHub CLI v2.2.5 (newer versoins have issues) +# TODO: Must use GitHub CLI v2.25.1 (newer versoins have issues) # Jenkins controller URL and authentication token must be provided. # jenkins-secret-file: # Must be present in the Jenkins agent launch directory. @@ -35,9 +35,9 @@ # controller_user: # Must be set to the Jenkins controller username corresponing to the jenkins_token. # -# Usage: ./launch_java_agent.sh [now] -# The optional 'now' argument forces the script to launch the Jenkins -# agent immediately without waiting, even if the node is detected as offline. +# Usage: ./launch_java_agent.sh [-f] +# The optional '-f' argument forces the script to launch the Jenkins +# agent without waiting and trying again. # # ============================================================================== @@ -132,10 +132,10 @@ check_node_online() { ./parse.py curl_response } -offline=$(check_node_online) +offline=$(set -e; check_node_online) if [[ "${offline}" != "False" ]]; then - if [[ "${1:-}" != "now" ]]; then + if [[ "${1:-}" != "-f" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi From b243b0d8923ba5429a93fc5850f34a8f997474c8 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Mon, 15 Jul 2024 16:49:51 +0000 Subject: [PATCH 17/29] added two optional swithces to agument the now intention with force --- ci/scripts/utils/launch_java_agent.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index ecc2b945399..d69f2d9fca8 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -35,9 +35,10 @@ # controller_user: # Must be set to the Jenkins controller username corresponing to the jenkins_token. # -# Usage: ./launch_java_agent.sh [-f] -# The optional '-f' argument forces the script to launch the Jenkins -# agent without waiting and trying again. +# Usage: ./launch_java_agent.sh [now] [-f] +# The optional 'now' argument forces the script to launch the Jenkins +# agent without waiting before trying again. +# The optional '-f' argument forces the script to launch the Jenkins regarless of the node status. # # ============================================================================== @@ -135,12 +136,12 @@ check_node_online() { offline=$(set -e; check_node_online) if [[ "${offline}" != "False" ]]; then - if [[ "${1:-}" != "-f" ]]; then + if [[ "${1:-}" != "now" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 fi offline=$(check_node_online) - if [[ "${offline}" != "False" ]]; then + if [[ "${offline}" != "False" ]] || [[ "${1:-}" == "-f" ]]; then echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" From e4651ce81998f08cd6ef8cd15ea073b6fb137ef2 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 16 Jul 2024 16:18:20 +0000 Subject: [PATCH 18/29] addeing get plugins test --- ci/scripts/utils/get_plugins.sh | 165 ++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100755 ci/scripts/utils/get_plugins.sh diff --git a/ci/scripts/utils/get_plugins.sh b/ci/scripts/utils/get_plugins.sh new file mode 100755 index 00000000000..73ef20f5f29 --- /dev/null +++ b/ci/scripts/utils/get_plugins.sh @@ -0,0 +1,165 @@ +#!/bin/env bash + +# ============================================================================== +# Script Name: launch_java_agent.sh +# +# Description: +# This script automates the process of launching a Jenkins agent +# on a specified machine. It ensures that the necessary +# prerequisites are met, such as the availability of JAVA_HOME, +# the Jenkins agent launch directory, and proper authentication +# with GitHub. +# +# It then proceeds to check if the Jenkins node is online and +# decides whether to launch the Jenkins agent based on the node's +# status. The agent is launched in the background, +# and its PID is logged for reference. +# +# Prerequisites: +# JAVA_HOME must be set to a valid JDK installation. +# Jenkins agent launch directory must exist and be specified. +# GitHub CLI (gh) must be installed and authenticated for messeging +# from the Jenkins controller to GitHub PR via shell commands. +# Jenkins agent launch directory must exist and be specified. +# TODO: Must use GitHub CLI v2.25.1 (newer versoins have issues) +# Jenkins controller URL and authentication token must be provided. +# jenkins-secret-file: +# Must be present in the Jenkins agent launch directory. +# This file contains the secret key for the Jenkins agent +# established by the Jenkins administrator for each Node. +# jenkins_token: +# Must be present in the Jenkins agent launch directory. +# This file contains the user authentication token for the Jenkins controller +# to use the Remote API. This token can be generated by the user +# on the Jenkins controller. +# controller_user: +# Must be set to the Jenkins controller username corresponing to the jenkins_token. +# +# Usage: ./launch_java_agent.sh [now] [-f] +# The optional 'now' argument forces the script to launch the Jenkins +# agent without waiting before trying again. +# The optional '-f' argument forces the script to launch the Jenkins regarless of the node status. +# +# ============================================================================== + +set -e +shopt -s inherit_errexit + +controller_url="https://jenkins.epic.oarcloud.noaa.gov" +controller_user="terry.mcguinness" +controller_user_auth_token="jenkins_token" + +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +host=$(hostname) + +######################################################################### +# Set up runtime environment varibles for accounts on supproted machines +######################################################################### + +source "${HOMEgfs}/ush/detect_machine.sh" +case ${MACHINE_ID} in + hera | orion | hercules | wcoss2) + echo "Launch Jenkins Java Controler on ${MACHINE_ID}" + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac + +LOG=lanuched_agent-$(date +%Y%m%d%M).log +rm -f "${LOG}" + +source "${HOMEgfs}/ush/module-setup.sh" +module use "${HOMEgfs}/modulefiles" +module load "module_gwsetup.${MACHINE_ID}" +source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" + +JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" +if [[ ! -d "${JAVA_HOME}" ]]; then + JAVA_HOME=/usr/lib/jvm/jre-17 + if [[ ! -d "${JAVA_HOME}" ]]; then + echo "ERROR: JAVA_HOME not found. Exiting with error." + exit 1 + fi +fi + +JAVA="${JAVA_HOME}/bin/java" +echo "JAVA VERSION: " +${JAVA} -version + +export GH="${HOME}/bin/gh" +command -v "${GH}" +${GH} --version + +check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true +if [[ "${check_mark}" != "✓" ]]; then + echo "gh not authenticating with emcbot token" + exit 1 +fi +echo "gh authenticating with emcbot TOKEN ok" + +if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then + echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" +else + echo "ERROR: Jenkins Agent Lanuch Directory not found. Exiting with error." + exit 1 +fi +cd "${JENKINS_AGENT_LANUCH_DIR}" + +if ! [[ -f agent.jar ]]; then + curl -sO "${controller_url}/jnlpJars/agent.jar" + echo "Updated agent.jar downloaded" +fi + +if [[ ! -f "${controller_user_auth_token}" ]]; then + echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist" + exit 1 +fi + +JENKINS_TOKEN=$(cat "${controller_user_auth_token}") +curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/pluginManager/api/json?pretty=true") || true +if [[ "${curl_response}" == "" ]]; then + echo "ERROR: Jenkins controller not reachable. Exiting with error." + exit 1 +else + echo "${curl_response}" +fi + +echo -e "#!/usr/bin/env python +import json,sys +with open(sys.argv[1], 'r') as file: + data = json.load(file) +print(data.get('offline','True')) +" > parse.py +chmod u+x parse.py + +check_node_online() { + rm -f curl_response + curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true + echo -n "${curl_response}" > curl_response + ./parse.py curl_response +} + +offline=$(set -e; check_node_online) + +if [[ "${offline}" != "False" ]]; then + if [[ "${1:-}" != "now" ]]; then + echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" + sleep 300 + fi + offline=$(check_node_online) + if [[ "${offline}" != "False" ]] || [[ "${1:-}" == "-f" ]]; then + echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 + echo "Java agent running on PID: ${nohup_PID}" + else + echo "Jenkins Agent is online (nothing done)" + fi +else + echo "Jenkins Agent is online (nothing done)" +fi From 26792fe43faa77124ad914a67557f2661ee7d955 Mon Sep 17 00:00:00 2001 From: "Terry.McGuinness" Date: Tue, 16 Jul 2024 16:42:44 +0000 Subject: [PATCH 19/29] remove get_plugins (do not have admin permissions on controler) --- ci/scripts/utils/get_plugins.sh | 165 -------------------------------- 1 file changed, 165 deletions(-) delete mode 100755 ci/scripts/utils/get_plugins.sh diff --git a/ci/scripts/utils/get_plugins.sh b/ci/scripts/utils/get_plugins.sh deleted file mode 100755 index 73ef20f5f29..00000000000 --- a/ci/scripts/utils/get_plugins.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/env bash - -# ============================================================================== -# Script Name: launch_java_agent.sh -# -# Description: -# This script automates the process of launching a Jenkins agent -# on a specified machine. It ensures that the necessary -# prerequisites are met, such as the availability of JAVA_HOME, -# the Jenkins agent launch directory, and proper authentication -# with GitHub. -# -# It then proceeds to check if the Jenkins node is online and -# decides whether to launch the Jenkins agent based on the node's -# status. The agent is launched in the background, -# and its PID is logged for reference. -# -# Prerequisites: -# JAVA_HOME must be set to a valid JDK installation. -# Jenkins agent launch directory must exist and be specified. -# GitHub CLI (gh) must be installed and authenticated for messeging -# from the Jenkins controller to GitHub PR via shell commands. -# Jenkins agent launch directory must exist and be specified. -# TODO: Must use GitHub CLI v2.25.1 (newer versoins have issues) -# Jenkins controller URL and authentication token must be provided. -# jenkins-secret-file: -# Must be present in the Jenkins agent launch directory. -# This file contains the secret key for the Jenkins agent -# established by the Jenkins administrator for each Node. -# jenkins_token: -# Must be present in the Jenkins agent launch directory. -# This file contains the user authentication token for the Jenkins controller -# to use the Remote API. This token can be generated by the user -# on the Jenkins controller. -# controller_user: -# Must be set to the Jenkins controller username corresponing to the jenkins_token. -# -# Usage: ./launch_java_agent.sh [now] [-f] -# The optional 'now' argument forces the script to launch the Jenkins -# agent without waiting before trying again. -# The optional '-f' argument forces the script to launch the Jenkins regarless of the node status. -# -# ============================================================================== - -set -e -shopt -s inherit_errexit - -controller_url="https://jenkins.epic.oarcloud.noaa.gov" -controller_user="terry.mcguinness" -controller_user_auth_token="jenkins_token" - -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" -host=$(hostname) - -######################################################################### -# Set up runtime environment varibles for accounts on supproted machines -######################################################################### - -source "${HOMEgfs}/ush/detect_machine.sh" -case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) - echo "Launch Jenkins Java Controler on ${MACHINE_ID}" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac - -LOG=lanuched_agent-$(date +%Y%m%d%M).log -rm -f "${LOG}" - -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" -source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" - -JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" -if [[ ! -d "${JAVA_HOME}" ]]; then - JAVA_HOME=/usr/lib/jvm/jre-17 - if [[ ! -d "${JAVA_HOME}" ]]; then - echo "ERROR: JAVA_HOME not found. Exiting with error." - exit 1 - fi -fi - -JAVA="${JAVA_HOME}/bin/java" -echo "JAVA VERSION: " -${JAVA} -version - -export GH="${HOME}/bin/gh" -command -v "${GH}" -${GH} --version - -check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true -if [[ "${check_mark}" != "✓" ]]; then - echo "gh not authenticating with emcbot token" - exit 1 -fi -echo "gh authenticating with emcbot TOKEN ok" - -if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then - echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}" -else - echo "ERROR: Jenkins Agent Lanuch Directory not found. Exiting with error." - exit 1 -fi -cd "${JENKINS_AGENT_LANUCH_DIR}" - -if ! [[ -f agent.jar ]]; then - curl -sO "${controller_url}/jnlpJars/agent.jar" - echo "Updated agent.jar downloaded" -fi - -if [[ ! -f "${controller_user_auth_token}" ]]; then - echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist" - exit 1 -fi - -JENKINS_TOKEN=$(cat "${controller_user_auth_token}") -curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/pluginManager/api/json?pretty=true") || true -if [[ "${curl_response}" == "" ]]; then - echo "ERROR: Jenkins controller not reachable. Exiting with error." - exit 1 -else - echo "${curl_response}" -fi - -echo -e "#!/usr/bin/env python -import json,sys -with open(sys.argv[1], 'r') as file: - data = json.load(file) -print(data.get('offline','True')) -" > parse.py -chmod u+x parse.py - -check_node_online() { - rm -f curl_response - curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true - echo -n "${curl_response}" > curl_response - ./parse.py curl_response -} - -offline=$(set -e; check_node_online) - -if [[ "${offline}" != "False" ]]; then - if [[ "${1:-}" != "now" ]]; then - echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" - sleep 300 - fi - offline=$(check_node_online) - if [[ "${offline}" != "False" ]] || [[ "${1:-}" == "-f" ]]; then - echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" - else - echo "Jenkins Agent is online (nothing done)" - fi -else - echo "Jenkins Agent is online (nothing done)" -fi From 0d3f65929e08505b81ae4e97a618051b87b95f4d Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 16 Jul 2024 17:27:19 +0000 Subject: [PATCH 20/29] simplified force/now logic in launch script --- ci/scripts/utils/launch_java_agent.sh | 70 ++++++++++++++++++++------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index d69f2d9fca8..0a0688c4255 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -1,5 +1,7 @@ #!/bin/env bash +set -e + # ============================================================================== # Script Name: launch_java_agent.sh # @@ -38,12 +40,27 @@ # Usage: ./launch_java_agent.sh [now] [-f] # The optional 'now' argument forces the script to launch the Jenkins # agent without waiting before trying again. -# The optional '-f' argument forces the script to launch the Jenkins regarless of the node status. +# The optional 'force' argument forces the script to launch the Jenkins regarless of the node status. # # ============================================================================== -set -e -shopt -s inherit_errexit +if [[ $# -gt 1 || "${1}" == "-h" ]]; then + echo "Usage: ./launch_java_agent.sh [now] [force] +Two mutually exclusive optional arguments: + (now) causes the script to launch the Jenkins agent without waiting before trying again. + (force) forces the script to launch the Jenkins regarless of its connection status." + exit 1 +elif [[ $# -eq 1 ]]; then + if [[ "$1" == "now" ]]; then + skip_wait=true + elif [[ "$1" == "force" ]]; then + force_launch=true + else + echo "Usage: ./launch_java_agent.sh [now] [-f]" + echo "Invalid argument. Use 'now' or '-f'." + exit 1 + fi +fi controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user="terry.mcguinness" @@ -118,6 +135,14 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then fi JENKINS_TOKEN=$(cat "${controller_user_auth_token}") +curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/pluginManager/api/json?pretty=true") || true +if [[ "${curl_response}" == "" ]]; then + echo "ERROR: Jenkins controller not reachable. Exiting with error." + exit 1 +else + echo "${curl_response}" +fi + echo -e "#!/usr/bin/env python import json,sys with open(sys.argv[1], 'r') as file: @@ -133,25 +158,34 @@ check_node_online() { ./parse.py curl_response } +lauch_agent () { + echo "Launching Jenkins Agent on ${host}" + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" + echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" + ${command} >> "${LOG}" 2>&1 & + nohup_PID=$! + echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 +} + + if [[ "${force_launch}" == "true" ]]; then + lauch_agent + exit + fi + offline=$(set -e; check_node_online) -if [[ "${offline}" != "False" ]]; then - if [[ "${1:-}" != "now" ]]; then + if [[ "${offline}" != "False" ]]; then + if [[ "${skip_wait}" != "True" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 - fi - offline=$(check_node_online) - if [[ "${offline}" != "False" ]] || [[ "${1:-}" == "-f" ]]; then - echo "Jenkins Agent is offline. Launching Jenkins Agent on ${host}" - command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" - echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" - ${command} >> "${LOG}" 2>&1 & - nohup_PID=$! - echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 - echo "Java agent running on PID: ${nohup_PID}" - else - echo "Jenkins Agent is online (nothing done)" + offline=$(check_node_online) + fi + if [[ "${offline}" != "False" ]]; then + lauch_agent + else + echo "Jenkins Agent is online (nothing done)" + fi fi else echo "Jenkins Agent is online (nothing done)" -fi +fi \ No newline at end of file From 6e412f3a5708133cae0efb940913593e569441c0 Mon Sep 17 00:00:00 2001 From: "Terry.McGuinness" Date: Tue, 16 Jul 2024 17:46:06 +0000 Subject: [PATCH 21/29] simplified now/force logic and add usage checks --- ci/scripts/utils/launch_java_agent.sh | 31 +++++++++++---------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 0a0688c4255..64c149888c5 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -133,15 +133,7 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then echo "User Jenkins authetication TOKEN to the controller for using the Remote API does not exist" exit 1 fi - JENKINS_TOKEN=$(cat "${controller_user_auth_token}") -curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/pluginManager/api/json?pretty=true") || true -if [[ "${curl_response}" == "" ]]; then - echo "ERROR: Jenkins controller not reachable. Exiting with error." - exit 1 -else - echo "${curl_response}" -fi echo -e "#!/usr/bin/env python import json,sys @@ -154,6 +146,10 @@ chmod u+x parse.py check_node_online() { rm -f curl_response curl_response=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true") || true + if [[ "${curl_response}" == "" ]]; then + echo "ERROR: Jenkins controller not reachable. Exiting with error." + exit 1 + fi echo -n "${curl_response}" > curl_response ./parse.py curl_response } @@ -167,25 +163,24 @@ lauch_agent () { echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 } - if [[ "${force_launch}" == "true" ]]; then - lauch_agent - exit - fi +if [[ "${force_launch}" == "true" ]]; then + lauch_agent + exit +fi offline=$(set -e; check_node_online) - if [[ "${offline}" != "False" ]]; then - if [[ "${skip_wait}" != "True" ]]; then +if [[ "${offline}" != "False" ]]; then + if [[ "${skip_wait}" != "True" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 offline=$(check_node_online) - fi - if [[ "${offline}" != "False" ]]; then + fi + if [[ "${offline}" != "False" ]]; then lauch_agent else echo "Jenkins Agent is online (nothing done)" fi - fi else echo "Jenkins Agent is online (nothing done)" -fi \ No newline at end of file +fi From a8d6b29b276c4f1dd896c94bc1807406e68e4fbc Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 16 Jul 2024 17:54:11 +0000 Subject: [PATCH 22/29] added -d on command line for checking on line function for shellnorms --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 64c149888c5..d9a89d8d7ba 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -174,7 +174,7 @@ if [[ "${offline}" != "False" ]]; then if [[ "${skip_wait}" != "True" ]]; then echo "Jenkins Agent is offline. Waiting 5 more minutes to check again in the event it is a temp network issue" sleep 300 - offline=$(check_node_online) + offline=$(set -e; check_node_online) fi if [[ "${offline}" != "False" ]]; then lauch_agent From ce117b5dd3dae3c737199199193565b2158a6988 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 17 Jul 2024 08:20:46 -0400 Subject: [PATCH 23/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: Walter Kolczynski - NOAA --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index d9a89d8d7ba..e9bba35bf91 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -177,7 +177,7 @@ if [[ "${offline}" != "False" ]]; then offline=$(set -e; check_node_online) fi if [[ "${offline}" != "False" ]]; then - lauch_agent + lauch_agent else echo "Jenkins Agent is online (nothing done)" fi From 308a73835864088616ecab9a7af8a7b1cf0a6f67 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 17 Jul 2024 09:33:13 -0400 Subject: [PATCH 24/29] use getopts --- ci/scripts/utils/launch_java_agent.sh | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index e9bba35bf91..33049f187f3 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -44,23 +44,20 @@ set -e # # ============================================================================== -if [[ $# -gt 1 || "${1}" == "-h" ]]; then - echo "Usage: ./launch_java_agent.sh [now] [force] +force_launch='False' +skip_wait='False' +while getopts 'f' flag; do + case "${flag}" in + force) force_launch='True' ;; + now) skip_wait='True' ;; + h) echo "Usage: ./launch_java_agent.sh [now] [force] Two mutually exclusive optional arguments: (now) causes the script to launch the Jenkins agent without waiting before trying again. (force) forces the script to launch the Jenkins regarless of its connection status." - exit 1 -elif [[ $# -eq 1 ]]; then - if [[ "$1" == "now" ]]; then - skip_wait=true - elif [[ "$1" == "force" ]]; then - force_launch=true - else - echo "Usage: ./launch_java_agent.sh [now] [-f]" - echo "Invalid argument. Use 'now' or '-f'." - exit 1 - fi -fi + *) echo "Unknown flag: ${flag}" + exit 1 ;; + esac +done controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user="terry.mcguinness" From ee06c4b32d998de0a4eb45d2724d89e6e3af90b1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 17 Jul 2024 09:04:44 -0500 Subject: [PATCH 25/29] reverted to getops and single char switches --- ci/scripts/utils/launch_java_agent.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 33049f187f3..83151ea65f5 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -44,18 +44,19 @@ set -e # # ============================================================================== -force_launch='False' -skip_wait='False' -while getopts 'f' flag; do +force_launch="False" +skip_wait="False" +while getopts ":fnh" flag; do case "${flag}" in - force) force_launch='True' ;; - now) skip_wait='True' ;; + f) force_launch="True";; + n) skip_wait="True";; h) echo "Usage: ./launch_java_agent.sh [now] [force] Two mutually exclusive optional arguments: - (now) causes the script to launch the Jenkins agent without waiting before trying again. - (force) forces the script to launch the Jenkins regarless of its connection status." + -n (now) causes the script to launch the Jenkins agent without waiting before trying again. + -f (force) forces the script to launch the Jenkins regarless of its connection status." + exit 0 ;; *) echo "Unknown flag: ${flag}" - exit 1 ;; + exit 1;; esac done @@ -160,7 +161,7 @@ lauch_agent () { echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1 } -if [[ "${force_launch}" == "true" ]]; then +if [[ "${force_launch}" == "True" ]]; then lauch_agent exit fi From 377cfadd9071d72efbc6125d66dd114b8c17f481 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 17 Jul 2024 14:53:50 -0400 Subject: [PATCH 26/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: Walter Kolczynski - NOAA --- ci/scripts/utils/launch_java_agent.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 83151ea65f5..fdc5d612d3e 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -133,12 +133,13 @@ if [[ ! -f "${controller_user_auth_token}" ]]; then fi JENKINS_TOKEN=$(cat "${controller_user_auth_token}") -echo -e "#!/usr/bin/env python +cat << EOF > parse.py +#!/usr/bin/env python3 import json,sys with open(sys.argv[1], 'r') as file: data = json.load(file) print(data.get('offline','True')) -" > parse.py +EOF chmod u+x parse.py check_node_online() { From d2e7efe51b6f8483943b92e94b33ee8325f65f35 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 18 Jul 2024 12:56:35 -0400 Subject: [PATCH 27/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index fdc5d612d3e..a0db1df8249 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -61,7 +61,7 @@ Two mutually exclusive optional arguments: done controller_url="https://jenkins.epic.oarcloud.noaa.gov" -controller_user="terry.mcguinness" +controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" From a242f2c59aa02261f06fde3bb9543addc8dae22a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 2 Aug 2024 14:43:59 -0400 Subject: [PATCH 28/29] Update ci/scripts/utils/launch_java_agent.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index a0db1df8249..650f0072153 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -104,7 +104,7 @@ echo "JAVA VERSION: " ${JAVA} -version export GH="${HOME}/bin/gh" -command -v "${GH}" +[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" ${GH} --version check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true From 677245b4871a49476e31b47006e16058aef645b1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 2 Aug 2024 14:48:48 -0400 Subject: [PATCH 29/29] added url to gh 2.251 --- ci/scripts/utils/launch_java_agent.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 650f0072153..81dbe002b6e 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -24,6 +24,7 @@ set -e # from the Jenkins controller to GitHub PR via shell commands. # Jenkins agent launch directory must exist and be specified. # TODO: Must use GitHub CLI v2.25.1 (newer versoins have issues) +# https://github.com/cli/cli/releases/download/v2.25.1/gh_2.25.1_linux_amd64.tar.gz # Jenkins controller URL and authentication token must be provided. # jenkins-secret-file: # Must be present in the Jenkins agent launch directory. @@ -74,12 +75,10 @@ host=$(hostname) source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2) - echo "Launch Jenkins Java Controler on ${MACHINE_ID}" - ;; + echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; *) echo "Unsupported platform. Exiting with error." - exit 1 - ;; + exit 1;; esac LOG=lanuched_agent-$(date +%Y%m%d%M).log