diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 2fa0a0ea60c..e11f253d13d 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -5,8 +5,8 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', gaeac6: 'Gaeac6-EMC'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', gaeac6: '/gpfs/f6/drsa-precip3/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -79,7 +79,7 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea', 'GaeaC6-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow" @@ -144,9 +144,9 @@ pipeline { """) gist_url=sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID} + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})' """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } @@ -242,9 +242,9 @@ pipeline { try { gist_url = sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID} + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})' """) sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} diff --git a/ci/platforms/config.gaeac6 b/ci/platforms/config.gaeac6 index 45a0e63d260..2680a0727ef 100644 --- a/ci/platforms/config.gaeac6 +++ b/ci/platforms/config.gaeac6 @@ -1,7 +1,18 @@ #!/usr/bin/bash -export GFS_CI_ROOT=/gpfs/f6/drsa-precip3/scratch/${USER}/GFS_CI_ROOT -export ICSDIR_ROOT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR +export GFS_CI_ROOT=/ncrc/proj/nggps_emc/${USER}/GFS_CI_CD +export ICSDIR_ROOT=/gpfs/f6/bil-fire8/world-shared/global/glopara/data/ICSDIR + +# CI BASH test directories +export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/Bash + +# Jenkins directories +export JENKINS_AGENT_LANUCH_DIR=${GFS_CI_ROOT}/Jenkins/agent +export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace + +# CTest functional test directories for pre stagged input data +export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR + export HPC_ACCOUNT=drsa-precip3 export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/scripts/utils/githubpr.py b/ci/scripts/utils/githubpr.py index 5fe0b643eaf..9fda110d79e 100755 --- a/ci/scripts/utils/githubpr.py +++ b/ci/scripts/utils/githubpr.py @@ -3,8 +3,11 @@ import os import re -from github import Github, GithubException, InputFileContent, UnknownObjectException -from wxflow import which +from github import Auth, Github, GithubException, InputFileContent, UnknownObjectException +from wxflow import which, Logger + +# Initialize logger with environment variable for logging level +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) class GitHubDBError(Exception): @@ -54,10 +57,24 @@ def __init__(self, repo_url=None, TOKEN=None): environment variable when repo_url is not provided. """ if TOKEN is None: - gh_cli = which('gh') - gh_cli.add_default_arg(['auth', 'status', '--show-token']) - TOKEN = gh_cli(output=str, error=str).split('\n')[3].split(': ')[1] - super().__init__(TOKEN) + TOKEN = os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN') + if TOKEN: + logger.info("Using TOKEN from environment variable.") + else: + gh_cli = which('gh') + gh_cli.add_default_arg(['auth', 'status', '--show-token']) + gh_output = gh_cli(output=str, error=str) + token_match = re.search(r"Token:\s*([a-zA-Z0-9_]+)", gh_output) + if token_match: + TOKEN = token_match.group(1) + logger.info("Using TOKEN from gh CLI tool.") + else: + raise ValueError("Token not found in gh CLI output.") + else: + logger.info("Using provided TOKEN.") + + auth = Auth.Token(TOKEN) + super().__init__(auth=auth) self.repo = self.get_repo_url(repo_url) self.pulls = self.repo.get_pulls(state='open', sort='updated', direction='desc') diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 3450050c958..5cf0cc0d8d2 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -74,7 +74,7 @@ host=$(hostname) source "${HOMEGFS_}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2 | gaea) + hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 ) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; *) echo "Unsupported platform. Exiting with error." @@ -84,7 +84,7 @@ esac LOG=lanuched_agent-$(date +%Y%m%d%M).log rm -f "${LOG}" -source "${HOMEGFS_}/ush/module-setup.sh" +HOMEgfs="${HOMEGFS_}" source "${HOMEGFS_}/ush/module-setup.sh" module use "${HOMEGFS_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"