From bd9ed1f14684efdf723c4c43db33c5a01a7a94d5 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Wed, 14 Feb 2024 17:39:20 +0000 Subject: [PATCH 01/39] removed env.MACHINE_ID=machine in Jenkins, same as export MACHINE_ID=here as it interfeard with hera GDAS logic --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index be62a205124..0a5e023af15 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,6 @@ pipeline { def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { - env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}
Still doing a checkout to get the latest changes") From fa9990dd0a8736fe7b6acd6ce0c3647fdf3ed438 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 14 Feb 2024 13:15:33 -0500 Subject: [PATCH 02/39] Update build_all.sh complier tag on MACHIN_ID removed for hera in build all scipt --- sorc/build_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index c337374428a..2757659657d 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -129,7 +129,7 @@ build_opts["ww3prepost"]="${_verbose_opt} ${_build_ufs_opt}" # Optional DA builds if [[ "${_build_ufsda}" == "YES" ]]; then - if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera.intel" && "${MACHINE_ID}" != "hercules" ]]; then + if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" ]]; then echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}. Disabling build." else build_jobs["gdas"]=8 From 6901c9cba303cf7f964e7663d9ea018298305e7a Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 13:59:13 -0500 Subject: [PATCH 03/39] used try catch over run experments get archive of error logs and running bacth cancel before leaving pipeline --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0a5e023af15..6536f5cc5e8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -134,12 +134,13 @@ pipeline { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { - pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() - // pullRequest.comment("**Running** experiment: ${Case} on ${Machine}
With the experiment in directory:
`${HOME}/RUNTESTS/${pslot}`") - err = sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}") - if (err != 0) { + pslot = sh(script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() + pullRequest.comment("**Running** experiment: ${Case} on ${Machine}
With the experiment in directory:
`${HOME}/RUNTESTS/${pslot}`") + try { + sh(script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}") + } catch (Exception e) { pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}") - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") + sh(script: "ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") ws(HOME) { if (fileExists('RUNTESTS/error.logs')) { def fileContent = readFile 'RUNTESTS/error.logs' @@ -152,10 +153,9 @@ pipeline { } error("Failed to run experiments ${Case} on ${Machine}") } - // pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}") } + pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}") } - } } } From 699120850ceb6bb728bae8ed0e463b145761288c Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 15:16:05 -0500 Subject: [PATCH 04/39] put HOMEgfs back in the paths to the CI scripts --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6536f5cc5e8..a9bfb33f3d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -134,13 +134,13 @@ pipeline { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { - pslot = sh(script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() + pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() pullRequest.comment("**Running** experiment: ${Case} on ${Machine}
With the experiment in directory:
`${HOME}/RUNTESTS/${pslot}`") try { - sh(script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}") } catch (Exception e) { pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}") - sh(script: "ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") ws(HOME) { if (fileExists('RUNTESTS/error.logs')) { def fileContent = readFile 'RUNTESTS/error.logs' From 481e40703df0532fe4483192299c2c465dcea262 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 15:37:27 -0500 Subject: [PATCH 05/39] trying recusive scm --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9bfb33f3d5..ae691b6b327 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,13 +76,14 @@ pipeline { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}
Still doing a checkout to get the latest changes") - sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules') + checkout scm + //sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules') dir('sorc') { sh(script: './link_workflow.sh') } } else { checkout scm - sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive') + //sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive') def builds_file = readYaml file: 'ci/cases/yamls/build.yaml' def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '') From 3d83f36b6655abff5793def0575dfbd6097aa173 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 16:56:34 -0500 Subject: [PATCH 06/39] just hadded buildDiscarder line --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index ae691b6b327..c8ea9647dbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,6 +10,7 @@ pipeline { options { skipDefaultCheckout() parallelsAlwaysFailFast() + buildDiscarder(logRotator(numToKeepStr: '2')) } stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR From 5840facc8167d44f1ed9d5b355c0b6f2b399e1e4 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 20:40:03 -0500 Subject: [PATCH 07/39] needed w in build all for new tests --- ci/cases/yamls/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 5398fa18891..7ac737c152d 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - gefs: './build_all.sh' - - gfs: './build_all.sh -gu' \ No newline at end of file + - gfs: './build_all.sh -guw' \ No newline at end of file From 1698ff86a8ed8288fb19cb3450e82f5e0d916eec Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Thu, 15 Feb 2024 21:03:39 -0500 Subject: [PATCH 08/39] -w is not really in develop now removeing w from yamls build and removed FMS failing C96_atmsnowDA --- Jenkinsfile | 4 ++-- ci/cases/yamls/build.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c8ea9647dbe..b17062b4c0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" commonworkspace = "${WORKSPACE}" - sh(script: "mkdir -p ${HOME}/RUNTESTS") + sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") pullRequest.addLabel("CI-${Machine}-Building") if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { pullRequest.removeLabel("CI-${Machine}-Ready") @@ -115,7 +115,7 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA' // 'C48mx500_3DVarAOWCDA' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' // 'C96_atmsnowDA', 'C48mx500_3DVarAOWCDA' } } stages { diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 7ac737c152d..5398fa18891 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - gefs: './build_all.sh' - - gfs: './build_all.sh -guw' \ No newline at end of file + - gfs: './build_all.sh -gu' \ No newline at end of file From 8066167e3ba4dd1aefe51cd9b3f07d8a675485a2 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 11:16:01 -0500 Subject: [PATCH 09/39] testing a method for implenting skip host clause --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index b17062b4c0a..f311964762b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ def machine = 'none' def HOME = 'none' def localworkspace = 'none' def commonworkspace = 'none' +def test_cases_list = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] pipeline { agent { label 'built-in' } @@ -120,6 +121,7 @@ pipeline { } stages { stage('Create Experiment') { + when { test_cases_list.contains(Case) } steps { script { sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") @@ -132,6 +134,7 @@ pipeline { } } stage('Run Experiments') { + when { test_cases_list.contains(Case) } steps { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments From 4229ab5cd7e31433369a1e34728befd9bf2f0c21 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 11:16:40 -0500 Subject: [PATCH 10/39] testing a method for implenting skip host clause --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f311964762b..3dce89f9291 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -116,7 +116,7 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' // 'C96_atmsnowDA', 'C48mx500_3DVarAOWCDA' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA', 'C48mx500_3DVarAOWCDA' } } stages { From 4473d9674cfd16d14c67b10124a35ad55611ae2b Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 12:09:34 -0500 Subject: [PATCH 11/39] add python script to just check case skip host --- ci/scripts/utils/check_case_skip.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 ci/scripts/utils/check_case_skip.py diff --git a/ci/scripts/utils/check_case_skip.py b/ci/scripts/utils/check_case_skip.py new file mode 100755 index 00000000000..0490c36660e --- /dev/null +++ b/ci/scripts/utils/check_case_skip.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import os +import sys +from wxflow import AttrDict, parse_j2yaml +from hosts import Host + +_here = os.path.dirname(__file__) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) + +if __name__ == '__main__': + + HOMEgfs = _top + data = AttrDict(HOMEgfs) + host = Host() + + case_name = sys.argv[1] + case_yaml = HOMEgfs + '/ci/pr/'+case_name+'.yaml' + data.update(os.environ) + case_conf = parse_j2yaml(path=case_yaml, data=data) + + if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: + print(f'Skipping creation of case: {case_conf.arguments.pslot} on {host.machine.capitalize()}') + sys.exit(1) From 646002e22fd854c411b603117ea6d64208276048 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 16 Feb 2024 18:02:35 +0000 Subject: [PATCH 12/39] added check case test in python for return false if skip_ci_on_hosts is set for the host it is called from meaning false to the answer shall I run --- ci/scripts/utils/check_case_skip.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/scripts/utils/check_case_skip.py b/ci/scripts/utils/check_case_skip.py index 0490c36660e..648393ed423 100755 --- a/ci/scripts/utils/check_case_skip.py +++ b/ci/scripts/utils/check_case_skip.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 import os import sys +sys.path.insert(0,'...') from wxflow import AttrDict, parse_j2yaml -from hosts import Host +from workflow import hosts _here = os.path.dirname(__file__) _top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) @@ -10,14 +11,17 @@ if __name__ == '__main__': HOMEgfs = _top - data = AttrDict(HOMEgfs) - host = Host() + data = AttrDict(HOMEgfs=_top) + host = hosts.Host() case_name = sys.argv[1] - case_yaml = HOMEgfs + '/ci/pr/'+case_name+'.yaml' + case_yaml = HOMEgfs + '/ci/cases/pr/'+case_name+'.yaml' data.update(os.environ) case_conf = parse_j2yaml(path=case_yaml, data=data) + if 'skip_ci_on_hosts' not in case_conf: + sys.exit(0) + if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: - print(f'Skipping creation of case: {case_conf.arguments.pslot} on {host.machine.capitalize()}') + print(f'Skipping creation of case: {case_name} on {host.machine.capitalize()}') sys.exit(1) From e572cde9c31edf7cb5c0017f68deaefd6a8f73ed Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 13:07:44 -0500 Subject: [PATCH 13/39] added python code to check for skip case on host --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3dce89f9291..b7de7413005 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -121,7 +121,7 @@ pipeline { } stages { stage('Create Experiment') { - when { test_cases_list.contains(Case) } + when { sh '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}' } steps { script { sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") @@ -134,7 +134,7 @@ pipeline { } } stage('Run Experiments') { - when { test_cases_list.contains(Case) } + when { sh '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}' } steps { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments From d111f949de3d098ab3da074857e7565828f1693c Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 13:23:07 -0500 Subject: [PATCH 14/39] check case script now returns true to run and false not to run --- ci/scripts/utils/check_case_skip.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/check_case_skip.py b/ci/scripts/utils/check_case_skip.py index 648393ed423..0f00fe378de 100755 --- a/ci/scripts/utils/check_case_skip.py +++ b/ci/scripts/utils/check_case_skip.py @@ -20,8 +20,12 @@ case_conf = parse_j2yaml(path=case_yaml, data=data) if 'skip_ci_on_hosts' not in case_conf: + print('true') sys.exit(0) if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: - print(f'Skipping creation of case: {case_name} on {host.machine.capitalize()}') - sys.exit(1) + #print(f'Skipping creation of case: {case_name} on {host.machine.capitalize()}') + print('false') + else: + print('true') + From af3f5e0837df9780c392151c576ca4d477c71d7f Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 13:30:57 -0500 Subject: [PATCH 15/39] put in test for skip cases with when expression with env var --- Jenkinsfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7de7413005..3172d692fd5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -120,8 +120,14 @@ pipeline { } } stages { - stage('Create Experiment') { - when { sh '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}' } + + stage('Create Experiments') { + steps { + script { + env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() + } + } + when { expression { env.NOT_SKIP_CASE == 'true' } } steps { script { sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") @@ -133,8 +139,14 @@ pipeline { } } } + stage('Run Experiments') { - when { sh '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}' } + steps { + script { + env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() + } + } + when { expression { env.NOT_SKIP_CASE == 'true' } } steps { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments @@ -163,6 +175,7 @@ pipeline { } } } + } } } From e8cc2ba43a40092c4378ca1581620daad2341cbf Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 13:34:44 -0500 Subject: [PATCH 16/39] make seperate stage to check skip --- Jenkinsfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3172d692fd5..2f2d3c1b8fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -121,12 +121,15 @@ pipeline { } stages { - stage('Create Experiments') { + stage( 'Check Skip') { steps { script { env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() } } + } + + stage('Create Experiments') { when { expression { env.NOT_SKIP_CASE == 'true' } } steps { script { @@ -141,11 +144,6 @@ pipeline { } stage('Run Experiments') { - steps { - script { - env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() - } - } when { expression { env.NOT_SKIP_CASE == 'true' } } steps { script { From e108f1862592d9fd061a1bfc1e75a1d518d9e352 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 14:51:12 -0500 Subject: [PATCH 17/39] add wxflow link in utils dir and another case in test list --- Jenkinsfile | 2 +- ci/scripts/utils/wxflow | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 ci/scripts/utils/wxflow diff --git a/Jenkinsfile b/Jenkinsfile index 2f2d3c1b8fe..f903a1417f5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -116,7 +116,7 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA', 'C48mx500_3DVarAOWCDA' + values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' } } stages { diff --git a/ci/scripts/utils/wxflow b/ci/scripts/utils/wxflow new file mode 120000 index 00000000000..54d0558aba2 --- /dev/null +++ b/ci/scripts/utils/wxflow @@ -0,0 +1 @@ +../../../sorc/wxflow/src/wxflow \ No newline at end of file From f5be771a6745c12d560218408a3b15a2715dad52 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 14:54:36 -0500 Subject: [PATCH 18/39] add env.HOME to point to check script --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index f903a1417f5..8ff658c3376 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -124,6 +124,7 @@ pipeline { stage( 'Check Skip') { steps { script { + env.HOME = HOME env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() } } From 7c3d377d05d647305dd67f8d8b6eca79cb4e4de0 Mon Sep 17 00:00:00 2001 From: "Terry.McGuinness" Date: Fri, 16 Feb 2024 20:32:43 +0000 Subject: [PATCH 19/39] got imports to work by adding links to modules dirs --- ci/scripts/utils/check_case_skip.py | 4 ++-- ci/scripts/utils/workflow | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 ci/scripts/utils/workflow diff --git a/ci/scripts/utils/check_case_skip.py b/ci/scripts/utils/check_case_skip.py index 0f00fe378de..7d41881aaaf 100755 --- a/ci/scripts/utils/check_case_skip.py +++ b/ci/scripts/utils/check_case_skip.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import os import sys -sys.path.insert(0,'...') -from wxflow import AttrDict, parse_j2yaml +from wxflow import parse_j2yaml +from wxflow import AttrDict from workflow import hosts _here = os.path.dirname(__file__) diff --git a/ci/scripts/utils/workflow b/ci/scripts/utils/workflow new file mode 120000 index 00000000000..99a655d2912 --- /dev/null +++ b/ci/scripts/utils/workflow @@ -0,0 +1 @@ +../../../workflow \ No newline at end of file From ea3c1ec95460fe88568990ff29e3cb273efd05c8 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 15:45:03 -0500 Subject: [PATCH 20/39] added echo for result of check case --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 8ff658c3376..70b2f89e7de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -126,6 +126,7 @@ pipeline { script { env.HOME = HOME env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() + echo "NOT_SKIP_CASE: ${Case} ${env.NOT_SKIP_CASE}" } } } From ea338b8f08d217882dd7d8542a188ed0f87de768 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 16:14:11 -0500 Subject: [PATCH 21/39] testing list again --- Jenkinsfile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70b2f89e7de..e1124228f81 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -121,18 +121,10 @@ pipeline { } stages { - stage( 'Check Skip') { - steps { - script { - env.HOME = HOME - env.NOT_SKIP_CASE = sh(script: '${HOME}/gfs/ci/scripts/utils/check_case_skip.py ${Case}', returnStdout: true).trim() - echo "NOT_SKIP_CASE: ${Case} ${env.NOT_SKIP_CASE}" - } - } - } - stage('Create Experiments') { - when { expression { env.NOT_SKIP_CASE == 'true' } } + when { + expression { return test_casts_list.contains(Case) } + } steps { script { sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") @@ -146,7 +138,9 @@ pipeline { } stage('Run Experiments') { - when { expression { env.NOT_SKIP_CASE == 'true' } } + when { + expression { return test_casts_list.contains(Case) } + } steps { script { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments From fb0e4062ed7f21bac0c6f2a7d0374adcbf7bc42c Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 16:22:05 -0500 Subject: [PATCH 22/39] spelled global variable test_cases_list wrong --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e1124228f81..2d3760e69dc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -123,7 +123,7 @@ pipeline { stage('Create Experiments') { when { - expression { return test_casts_list.contains(Case) } + expression { return test_cases_list.contains(Case) } } steps { script { @@ -139,7 +139,7 @@ pipeline { stage('Run Experiments') { when { - expression { return test_casts_list.contains(Case) } + expression { return test_cases_list.contains(Case) } } steps { script { From 37bf83b7b6b97f6851adf7ca5873273a477ed8f9 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 16:47:19 -0500 Subject: [PATCH 23/39] get all cases for host by checking for skips --- ci/scripts/utils/check_case_skip.py | 31 -------------------------- ci/scripts/utils/get_host_case_list.py | 28 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 31 deletions(-) delete mode 100755 ci/scripts/utils/check_case_skip.py create mode 100755 ci/scripts/utils/get_host_case_list.py diff --git a/ci/scripts/utils/check_case_skip.py b/ci/scripts/utils/check_case_skip.py deleted file mode 100755 index 7d41881aaaf..00000000000 --- a/ci/scripts/utils/check_case_skip.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -from wxflow import parse_j2yaml -from wxflow import AttrDict -from workflow import hosts - -_here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) - -if __name__ == '__main__': - - HOMEgfs = _top - data = AttrDict(HOMEgfs=_top) - host = hosts.Host() - - case_name = sys.argv[1] - case_yaml = HOMEgfs + '/ci/cases/pr/'+case_name+'.yaml' - data.update(os.environ) - case_conf = parse_j2yaml(path=case_yaml, data=data) - - if 'skip_ci_on_hosts' not in case_conf: - print('true') - sys.exit(0) - - if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: - #print(f'Skipping creation of case: {case_name} on {host.machine.capitalize()}') - print('false') - else: - print('true') - diff --git a/ci/scripts/utils/get_host_case_list.py b/ci/scripts/utils/get_host_case_list.py new file mode 100755 index 00000000000..81eb17fa888 --- /dev/null +++ b/ci/scripts/utils/get_host_case_list.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import os +from os.path import basename, splitext +import sys +import glob +from wxflow import parse_j2yaml +from wxflow import AttrDict +from workflow import hosts + +_here = os.path.dirname(__file__) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) + +if __name__ == '__main__': + + case_list = [] + host = hosts.Host() + HOMEgfs = _top + data = AttrDict(HOMEgfs=_top) + data.update(os.environ) + + case_files = glob.glob(f'{HOMEgfs}/ci/cases/pr/*.yaml') + for case_yaml in case_files: + case_conf = parse_j2yaml(path=case_yaml, data=data) + if 'skip_ci_on_hosts' in case_conf: + if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: + case_list.append(splitext(basename(case_yaml))[0]) + print(' '.join(case_list)) + From 6b329392f697c309225db9a3b4c041f6fe972f1d Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 16:57:15 -0500 Subject: [PATCH 24/39] added get cases list of cases that are not skipped for that host --- Jenkinsfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2d3760e69dc..ec23a310fc6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def machine = 'none' def HOME = 'none' def localworkspace = 'none' def commonworkspace = 'none' -def test_cases_list = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] +def casesList = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] pipeline { agent { label 'built-in' } @@ -101,6 +101,9 @@ pipeline { } pullRequest.addLabel("CI-${Machine}-Running") } + if (system == 'gfs') { + caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py", returnStdout: true).trim().split() + } } } } @@ -123,7 +126,7 @@ pipeline { stage('Create Experiments') { when { - expression { return test_cases_list.contains(Case) } + expression { return caseList.contains(Case) } } steps { script { @@ -139,7 +142,7 @@ pipeline { stage('Run Experiments') { when { - expression { return test_cases_list.contains(Case) } + expression { return casesList.contains(Case) } } steps { script { From 1aa8d2a515d110df4a8b474bacff53936794f9c5 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 19:07:29 -0500 Subject: [PATCH 25/39] switch list to ones that do run on host from ones that dont --- ci/scripts/utils/get_host_case_list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/get_host_case_list.py b/ci/scripts/utils/get_host_case_list.py index 81eb17fa888..227f69bbded 100755 --- a/ci/scripts/utils/get_host_case_list.py +++ b/ci/scripts/utils/get_host_case_list.py @@ -23,6 +23,7 @@ case_conf = parse_j2yaml(path=case_yaml, data=data) if 'skip_ci_on_hosts' in case_conf: if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: - case_list.append(splitext(basename(case_yaml))[0]) + continue + case_list.append(splitext(basename(case_yaml))[0]) print(' '.join(case_list)) From 371d4f6d3fa8b74d74a827b3d166cf486e867a0d Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Fri, 16 Feb 2024 19:40:16 -0500 Subject: [PATCH 26/39] misspelled caseList and fixed --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec23a310fc6..21096eba19a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def machine = 'none' def HOME = 'none' def localworkspace = 'none' def commonworkspace = 'none' -def casesList = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] +def caseList = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] pipeline { agent { label 'built-in' } @@ -142,7 +142,7 @@ pipeline { stage('Run Experiments') { when { - expression { return casesList.contains(Case) } + expression { return caseList.contains(Case) } } steps { script { From c3d073425ee102f9f87c5c79e7d0041ecc30ff45 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 20 Feb 2024 17:34:27 +0000 Subject: [PATCH 27/39] added skip host orion for C96_atmos snow DA --- Jenkinsfile | 6 ++---- ci/cases/pr/C96_atmsnowDA.yaml | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 21096eba19a..96e4ef15a76 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def machine = 'none' def HOME = 'none' def localworkspace = 'none' def commonworkspace = 'none' -def caseList = ['C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW'] +def caseList = '' pipeline { agent { label 'built-in' } @@ -11,7 +11,7 @@ pipeline { options { skipDefaultCheckout() parallelsAlwaysFailFast() - buildDiscarder(logRotator(numToKeepStr: '2')) + buildDiscarder(logRotator(numToKeepStr: '4')) } stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR @@ -79,13 +79,11 @@ pipeline { sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}
Still doing a checkout to get the latest changes") checkout scm - //sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules') dir('sorc') { sh(script: './link_workflow.sh') } } else { checkout scm - //sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive') def builds_file = readYaml file: 'ci/cases/yamls/build.yaml' def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '') diff --git a/ci/cases/pr/C96_atmsnowDA.yaml b/ci/cases/pr/C96_atmsnowDA.yaml index 35fcc10fb21..22d478e49c4 100644 --- a/ci/cases/pr/C96_atmsnowDA.yaml +++ b/ci/cases/pr/C96_atmsnowDA.yaml @@ -18,4 +18,5 @@ arguments: skip_ci_on_hosts: - orion + - hera - hercules From 6fa270ff1fe09259393910a5260c33aa64470b56 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 15:59:28 -0500 Subject: [PATCH 28/39] trying to reduce workspace paths --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 96e4ef15a76..cde619a52af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,8 +2,8 @@ def Machine = 'none' def machine = 'none' def HOME = 'none' def localworkspace = 'none' -def commonworkspace = 'none' def caseList = '' +def custom_workspace = '/scratch1/NCEPDEV/global/Terry.McGuinness' pipeline { agent { label 'built-in' } @@ -41,12 +41,13 @@ pipeline { } stage('Get Common Workspace') { - agent { label "${machine}-emc" } + agent { + label "${machine}-emc" + customWorkspace custom_workspace steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOME = "${WORKSPACE}/TESTDIR" - commonworkspace = "${WORKSPACE}" + HOME = "${WORKSPACE}" sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") pullRequest.addLabel("CI-${Machine}-Building") if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { From 7f63c5e828f3462cdfa6c9357f4a4808ac06149f Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:06:35 -0500 Subject: [PATCH 29/39] just run bad actor case for datapath issue --- Jenkinsfile | 3 ++- ci/cases/pr/C96_atmsnowDA.yaml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cde619a52af..c0a7e8d4577 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,7 +118,8 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' + // values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' + values 'C96_atmsnowDA' } } stages { diff --git a/ci/cases/pr/C96_atmsnowDA.yaml b/ci/cases/pr/C96_atmsnowDA.yaml index 22d478e49c4..35fcc10fb21 100644 --- a/ci/cases/pr/C96_atmsnowDA.yaml +++ b/ci/cases/pr/C96_atmsnowDA.yaml @@ -18,5 +18,4 @@ arguments: skip_ci_on_hosts: - orion - - hera - hercules From 1fcfc4c7be941a589bb4ee600a051c076fa8305d Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:28:24 -0500 Subject: [PATCH 30/39] just changed numToKeep --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0a7e8d4577..0533b28e0a2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { options { skipDefaultCheckout() parallelsAlwaysFailFast() - buildDiscarder(logRotator(numToKeepStr: '4')) + buildDiscarder(logRotator(numToKeepStr: '2')) } stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR From 25cb1cb4c69a726b96708f4e9c722aed96fe4e50 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:32:04 -0500 Subject: [PATCH 31/39] missed bracket on agent --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0533b28e0a2..f8dcff1c658 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,6 +44,7 @@ pipeline { agent { label "${machine}-emc" customWorkspace custom_workspace + } steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) From da7bc9743cbf80c11b9b92ade9009e96bb7bf8f6 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:37:20 -0500 Subject: [PATCH 32/39] used ws for custom instead --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index f8dcff1c658..c7d1c7efab1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,6 +47,7 @@ pipeline { } steps { script { + ws (custome_workspace) { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}" sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") @@ -55,6 +56,7 @@ pipeline { pullRequest.removeLabel("CI-${Machine}-Ready") } } + } } } From afd77b9bedfcfa109c8817dacba9f960d5c6a7f4 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:38:24 -0500 Subject: [PATCH 33/39] took out in label block --- Jenkinsfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c7d1c7efab1..f3d920735c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,10 +41,7 @@ pipeline { } stage('Get Common Workspace') { - agent { - label "${machine}-emc" - customWorkspace custom_workspace - } + agent { label "${machine}-emc" } steps { script { ws (custome_workspace) { From 689e0ee7853a6b3da4bd8816f09e3d4fc3bbde64 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" Date: Wed, 21 Feb 2024 16:41:08 -0500 Subject: [PATCH 34/39] spell custom_workspace --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f3d920735c1..a57f482e4d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,7 @@ pipeline { agent { label "${machine}-emc" } steps { script { - ws (custome_workspace) { + ws (custom_workspace) { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}" sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") From 56b51b67d9ec5026c5e0275c4ee16a19d15a064c Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 22 Feb 2024 18:32:24 +0000 Subject: [PATCH 35/39] added common workspace with miminum nubmer of chars to path --- Jenkinsfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 96e4ef15a76..5a6cc3b6c6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,8 @@ def Machine = 'none' def machine = 'none' def HOME = 'none' -def localworkspace = 'none' -def commonworkspace = 'none' def caseList = '' +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES'] pipeline { agent { label 'built-in' } @@ -22,7 +21,6 @@ pipeline { agent { label 'built-in' } steps { script { - localworkspace = env.WORKSPACE machine = 'none' for (label in pullRequest.labels) { echo "Label: ${label}" @@ -44,14 +42,16 @@ pipeline { agent { label "${machine}-emc" } steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOME = "${WORKSPACE}/TESTDIR" - commonworkspace = "${WORKSPACE}" - sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") - pullRequest.addLabel("CI-${Machine}-Building") - if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { - pullRequest.removeLabel("CI-${Machine}-Ready") + ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + HOME = "${WORKSPACE}" + sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") + pullRequest.addLabel("CI-${Machine}-Building") + if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { + pullRequest.removeLabel("CI-${Machine}-Ready") + } } + pullRequest.comment("Building and running on ${Machine} in directory ${HOME}") } } } From 3baed5fddd97c44295bc2a8ad8d02ae46d7ef31b Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 22 Feb 2024 21:59:38 +0000 Subject: [PATCH 36/39] put all tests back and removed Keep limiter --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1450a3fd049..708b2421200 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,6 @@ pipeline { options { skipDefaultCheckout() parallelsAlwaysFailFast() - buildDiscarder(logRotator(numToKeepStr: '2')) } stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR @@ -117,8 +116,7 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - // values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' - values 'C96_atmsnowDA' + values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' } } stages { From 93ff847498c93d12a707e6ad96f3d476ba39bf99 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Thu, 22 Feb 2024 23:00:10 +0000 Subject: [PATCH 37/39] python norms needed extra line delteted in get_host_case_list --- ci/scripts/utils/get_host_case_list.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/utils/get_host_case_list.py b/ci/scripts/utils/get_host_case_list.py index 227f69bbded..575d79d0f0b 100755 --- a/ci/scripts/utils/get_host_case_list.py +++ b/ci/scripts/utils/get_host_case_list.py @@ -26,4 +26,3 @@ continue case_list.append(splitext(basename(case_yaml))[0]) print(' '.join(case_list)) - From ab28fca8f988b7dc4969f71c2b7c783129444a48 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 23 Feb 2024 16:18:14 +0000 Subject: [PATCH 38/39] added host name to argument to get_host_case_list.py --- Jenkinsfile | 2 +- ci/scripts/utils/get_host_case_list.py | 10 +++++++--- ci/scripts/utils/workflow | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) delete mode 120000 ci/scripts/utils/workflow diff --git a/Jenkinsfile b/Jenkinsfile index 708b2421200..52175c2b50b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,7 +99,7 @@ pipeline { pullRequest.addLabel("CI-${Machine}-Running") } if (system == 'gfs') { - caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py", returnStdout: true).trim().split() + caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() } } } diff --git a/ci/scripts/utils/get_host_case_list.py b/ci/scripts/utils/get_host_case_list.py index 575d79d0f0b..5909706a679 100755 --- a/ci/scripts/utils/get_host_case_list.py +++ b/ci/scripts/utils/get_host_case_list.py @@ -5,15 +5,19 @@ import glob from wxflow import parse_j2yaml from wxflow import AttrDict -from workflow import hosts _here = os.path.dirname(__file__) _top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) if __name__ == '__main__': + if len(sys.argv) < 2: + print('Usage: get_host_case_list.py ') + sys.exit(1) + + host = sys.argv[1] + case_list = [] - host = hosts.Host() HOMEgfs = _top data = AttrDict(HOMEgfs=_top) data.update(os.environ) @@ -22,7 +26,7 @@ for case_yaml in case_files: case_conf = parse_j2yaml(path=case_yaml, data=data) if 'skip_ci_on_hosts' in case_conf: - if host.machine.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: + if host.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: continue case_list.append(splitext(basename(case_yaml))[0]) print(' '.join(case_list)) diff --git a/ci/scripts/utils/workflow b/ci/scripts/utils/workflow deleted file mode 120000 index 99a655d2912..00000000000 --- a/ci/scripts/utils/workflow +++ /dev/null @@ -1 +0,0 @@ -../../../workflow \ No newline at end of file From fda0610fcbc324656b01e4326b759991fd41473f Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Fri, 23 Feb 2024 16:42:03 +0000 Subject: [PATCH 39/39] fixing python norms with removing white spaces --- ci/scripts/utils/get_host_case_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/get_host_case_list.py b/ci/scripts/utils/get_host_case_list.py index 5909706a679..eb10f29f05f 100755 --- a/ci/scripts/utils/get_host_case_list.py +++ b/ci/scripts/utils/get_host_case_list.py @@ -14,7 +14,7 @@ if len(sys.argv) < 2: print('Usage: get_host_case_list.py ') sys.exit(1) - + host = sys.argv[1] case_list = []