From a6fa1d8356a4354fd9c4b8ddfedaf1463c360c7c Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 16 Apr 2025 16:46:05 -0400 Subject: [PATCH 01/76] ci_ctest_multimodality updates --- CMakeLists.txt | 2 +- dev/ci/.gitlab-ci-ctests.yml | 37 ++++++ dev/ci/.gitlab-ci.yml | 111 ++++++++++++++---- dev/ci/scripts/utils/gitlab/docs/README.md | 105 +++++++++++++++++ .../utils/gitlab/generate_pipelines.py | 9 ++ .../utils/gitlab/gitlab_pipeline_template.yml | 43 +++++++ 6 files changed, 281 insertions(+), 26 deletions(-) create mode 100644 dev/ci/.gitlab-ci-ctests.yml create mode 100644 dev/ci/scripts/utils/gitlab/docs/README.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 5044689f7e2..6470de8ca7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,4 +24,4 @@ endif() # add_subdirectory(sorc) # Setup tests -add_subdirectory(ctests) +add_subdirectory(dev/ctests) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml new file mode 100644 index 00000000000..4c386309078 --- /dev/null +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -0,0 +1,37 @@ +# CTests jobs for GitHub API triggered builds +# This file is included by the main .gitlab-ci.yml when GITHUB_API_TRIGGER=true + +# For each machine, we'll define CTest jobs that depend on the shared build stage +# The shared build already does the CMake setup for ctests + +# Template for CTest jobs that will be used across machines +.run_ctests_template: + extends: .ctests_template + stage: run_tests + parallel: + matrix: + - CTEST: ["C48_ATM_gfs_fcst_seg0", "C48_S2SW_gfs_fcst_seg0", "C48_S2SW_gfs_atmos_prod"] + rules: + - if: $GITHUB_API_TRIGGER == "true" + +# The machine-specific jobs will be dynamically generated and added to the pipeline +# by the generate_pipelines.py script. They will follow this pattern: + +# run_ctests-hera: +# extends: .run_ctests_template +# variables: +# machine: hera +# tags: ["hera"] +# dependencies: +# - build-hera + +# Dummy job to ensure stage exists when no tests are defined +create_ctests: + stage: create_experiments + script: + - echo "CTests setup already handled during build stage" + rules: + - if: $GITHUB_API_TRIGGER == "true" + tags: + - hera + diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 0614b3629b5..5f527fe0ffb 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -15,6 +15,15 @@ variables: GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + # GitHub API trigger detection - defaults to false + GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} + PR_NUMBER: ${PR_NUMBER:-"0"} + +# Include ctests configuration for GitHub API triggered builds +include: + - local: 'dev/ci/.gitlab-ci-ctests.yml' + rules: + - if: $GITHUB_API_TRIGGER == "true" .base_config: variables: @@ -31,6 +40,16 @@ variables: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} + # Add ctests build steps when triggered from GitHub API + - | + if [ "${GITHUB_API_TRIGGER}" == "true" ]; then + echo "Setting up for ctests workflow" + mkdir -p ${HOMEGFS}/ctests + cd ${HOMEGFS}/ctests + mkdir -p build + cd build + cmake ../.. + fi .setup_template: extends: .base_config @@ -41,6 +60,8 @@ variables: - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} + rules: + - if: $GITHUB_API_TRIGGER != "true" .run_tests_template: extends: .base_config @@ -52,6 +73,28 @@ variables: - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} + rules: + - if: $GITHUB_API_TRIGGER != "true" + +.ctests_template: + extends: .base_config + stage: run_tests + allow_failure: true + script: + - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" + - if [ "${PR_NUMBER}" != "0" ]; then + - echo "Testing PR #${PR_NUMBER}" + - else + - echo "Testing standard branch" + - fi + - cd ${HOMEGFS}/ctests/build + - ctest -R test_${CTEST}_setup + - ctest -R test_${CTEST}_stage + - ctest -R test_${CTEST}_execute + dependencies: + - build-${machine} + rules: + - if: $GITHUB_API_TRIGGER == "true" create_gist_from_svg_url: image: ghcr.io/cli/cli:latest @@ -75,6 +118,43 @@ create_gist_from_svg_url: # The following sections are generated for multiple hosts and dynamic case lists +build-gaeac6: + extends: .build_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + +setup_experiments-gaeac6: + extends: .setup_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-gaeac6 + +run_tests-gaeac6: + extends: .run_tests_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 + +# CTests job for GitHub API triggered builds +run_ctests-gaeac6: + extends: .run_ctests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + dependencies: + - build-gaeac6 + build-hera: extends: .build_template variables: @@ -103,32 +183,13 @@ run_tests-hera: dependencies: - setup_experiments-hera -build-gaeac6: - extends: .build_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - -setup_experiments-gaeac6: - extends: .setup_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-gaeac6 - -run_tests-gaeac6: - extends: .run_tests_template +# CTests job for GitHub API triggered builds +run_ctests-hera: + extends: .run_ctests_template variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + machine: hera + tags: ["hera"] dependencies: - - setup_experiments-gaeac6 + - build-hera # End of generated sections diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md new file mode 100644 index 00000000000..fcc33896227 --- /dev/null +++ b/dev/ci/scripts/utils/gitlab/docs/README.md @@ -0,0 +1,105 @@ +# GitLab CI Pipeline Configuration + +This documentation describes the multi-modal GitLab CI pipeline configuration for the global-workflow project. + +## Pipeline Modalities + +The CI pipeline supports two different modalities: + +1. **Standard Pipeline** - The default pipeline that runs when triggered by normal GitLab events (push, merge requests) +2. **CTests Pipeline** - A specialized pipeline that runs when triggered via GitLab API from GitHub + +## Multi-Host Support + +Both pipeline modalities support running on multiple compute hosts using a shared configuration system. The pipeline is dynamically generated by the `generate_pipelines.py` script, which creates jobs for each supported host machine based on the test cases that each host supports. + +## Pipeline Structure + +Both modalities use the following stages: +- `build`: Builds the codebase (shared between modalities) +- `create_experiments`: Sets up experiments or ctests +- `run_tests`: Executes the test workflows +- `finalize`: Updates status badges + +## Standard Pipeline + +The standard pipeline when triggered normally: +- Builds the codebase for each machine +- Sets up experiments using configurations in `dev/ci/cases/pr/` +- Runs tests for each case on the appropriate machine +- Completes with final status reporting + +## CTests Pipeline + +The CTests pipeline when triggered via GitLab API from GitHub: +- Uses the same build jobs as the standard pipeline +- The build stage additionally sets up the CMake environment for ctests +- Skips the standard experiment creation +- Runs specialized CTest test cases (`gfs_atmens`, `gfs_atm`, `gdas`, `gdas_enkf`) on each machine + +## Shared Build Stage + +Both modalities share the same build stage, which: +1. Clones the repository +2. Builds the necessary components +3. Sets up the workspace for testing +4. If triggered by GitHub API, also prepares the CMake environment for ctests + +This shared approach ensures consistent build results while optimizing resource usage. + +## Triggering the CTests Pipeline from GitHub + +To trigger the CTests pipeline from GitHub, use the GitLab API with the following parameters: + +```bash +curl -X POST \ + --header "Content-Type: application/json" \ + --header "PRIVATE-TOKEN: " \ + "https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/api/v4/projects//trigger/pipeline" \ + --data '{ + "ref": "", + "variables": { + "GITHUB_API_TRIGGER": "true", + "PR_NUMBER": "", + "GITHUB_REPO_URL": "" + } + }' +``` + +Replace the following: +- ``: Your GitLab API token with appropriate permissions +- ``: The GitLab project ID for global-workflow +- ``: The branch to run tests on +- ``: The GitHub PR number (use "0" for non-PR runs) +- ``: The GitHub repository URL + +## Pipeline Configuration Files + +The configuration is split across multiple files: + +1. **Template File** (`gitlab_pipeline_template.yml`): + - Contains the core pipeline structure + - Defines job templates for build, setup, and testing + - Includes modality detection and conditional logic + +2. **Generated Pipeline File** (`.gitlab-ci.yml`): + - Generated by `generate_pipelines.py` from the template + - Contains machine-specific jobs for both modalities + - Controls which jobs run based on trigger type + +3. **CTests Configuration** (`.gitlab-ci-ctests.yml`): + - Contains specific configurations for CTests + - Included conditionally when triggered via GitHub API + +## Pipeline Generation Process + +1. The `generate_pipelines.py` script: + - Reads the template file + - Determines which test cases each machine supports + - Generates machine-specific jobs for both standard tests and ctests + - Outputs the complete pipeline configuration + +2. When the pipeline runs: + - GitLab detects if `GITHUB_API_TRIGGER` is set to "true" + - If true, includes the `.gitlab-ci-ctests.yml` file + - Jobs check this variable to determine if they should run \ No newline at end of file diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index d46e77c945b..c30c1bcbcbd 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -134,6 +134,15 @@ def generate_machine_config(machine, case_list): - caseName: {case_list_yaml} dependencies: - setup_experiments-{machine} + +# CTests job for GitHub API triggered builds +run_ctests-{machine}: + extends: .run_ctests_template + variables: + machine: {machine} + tags: ["{machine}"] + dependencies: + - build-{machine} ''' return machine_config diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index dace4fb9095..6dac7a06f19 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -21,6 +21,15 @@ variables: GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + # GitHub API trigger detection - defaults to false + GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} + PR_NUMBER: ${PR_NUMBER:-"0"} + +# Include ctests configuration for GitHub API triggered builds +include: + - local: 'dev/ci/.gitlab-ci-ctests.yml' + rules: + - if: $GITHUB_API_TRIGGER == "true" .base_config: variables: @@ -37,6 +46,16 @@ variables: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} + # Add ctests build steps when triggered from GitHub API + - | + if [ "${GITHUB_API_TRIGGER}" == "true" ]; then + echo "Setting up for ctests workflow" + mkdir -p ${HOMEGFS}/ctests + cd ${HOMEGFS}/ctests + mkdir -p build + cd build + cmake ../.. + fi .setup_template: extends: .base_config @@ -47,6 +66,8 @@ variables: - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} + rules: + - if: $GITHUB_API_TRIGGER != "true" .run_tests_template: extends: .base_config @@ -58,6 +79,28 @@ variables: - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} + rules: + - if: $GITHUB_API_TRIGGER != "true" + +.ctests_template: + extends: .base_config + stage: run_tests + allow_failure: true + script: + - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" + - if [ "${PR_NUMBER}" != "0" ]; then + - echo "Testing PR #${PR_NUMBER}" + - else + - echo "Testing standard branch" + - fi + - cd ${HOMEGFS}/ctests/build + - ctest -R test_${CTEST}_setup + - ctest -R test_${CTEST}_stage + - ctest -R test_${CTEST}_execute + dependencies: + - build-${machine} + rules: + - if: $GITHUB_API_TRIGGER == "true" create_gist_from_svg_url: image: ghcr.io/cli/cli:latest From ed9d2dae62d80f68815cdb1b46dc157535ee1db6 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 16 Apr 2025 17:47:39 -0400 Subject: [PATCH 02/76] completed preliminary design of mixed-modality pipelines --- dev/ci/.gitlab-ci-cases.yml | 75 +++++++++ dev/ci/.gitlab-ci-ctests.yml | 82 ++++++---- dev/ci/.gitlab-ci.yml | 147 ++---------------- dev/ci/scripts/utils/gitlab/docs/README.md | 27 ++-- .../utils/gitlab/generate_pipelines.py | 129 +++++++++++---- .../utils/gitlab/gitlab_pipeline_template.yml | 65 ++------ 6 files changed, 271 insertions(+), 254 deletions(-) create mode 100644 dev/ci/.gitlab-ci-cases.yml diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml new file mode 100644 index 00000000000..30d10ca5e74 --- /dev/null +++ b/dev/ci/.gitlab-ci-cases.yml @@ -0,0 +1,75 @@ +# Standard cases modality configuration for the global-workflow project +# This file defines jobs specific to the standard test cases workflow + +# Base configuration for case jobs +.base_config: + variables: + GIT_STRATEGY: none + +# Template for experiment setup jobs +.setup_template: + extends: .base_config + stage: create_experiments + allow_failure: true + script: + - export RUNTESTS=${RUNTESTS_DIR} + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml + dependencies: + - build-${machine} + +# Template for test execution jobs +.run_tests_template: + extends: .base_config + stage: run_tests + allow_failure: true + script: + - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" + - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + dependencies: + - setup_experiments-${machine} + + +setup_experiments-hera: + extends: .setup_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-hera + +run_tests-hera: + extends: .run_tests_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-hera + +setup_experiments-gaeac6: + extends: .setup_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-gaeac6 + +run_tests-gaeac6: + extends: .run_tests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 4c386309078..f047bcdc784 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -1,8 +1,45 @@ -# CTests jobs for GitHub API triggered builds -# This file is included by the main .gitlab-ci.yml when GITHUB_API_TRIGGER=true +# CTests modality configuration for the global-workflow project +# This file defines jobs specific to the CTests workflow +# It is included when triggered via GitHub API (GITHUB_API_TRIGGER=true) -# For each machine, we'll define CTest jobs that depend on the shared build stage -# The shared build already does the CMake setup for ctests +# Base configuration for ctest jobs +.base_config: + variables: + GIT_STRATEGY: none + +# Setup job for ctests using CMake +create_ctests: + extends: .base_config + stage: create_experiments + script: + - echo "Setting up for ctests workflow" + - mkdir -p ${HOMEGFS}/ctests + - cd ${HOMEGFS}/ctests + - cmake ../.. + - echo "CTests have been created and configured successfully" + dependencies: + - build-${machine} + tags: + - hera + +# Template for CTest execution jobs +.ctests_template: + extends: .base_config + stage: run_tests + allow_failure: true + script: + - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" + - if [ "${PR_NUMBER}" != "0" ]; then + - echo "Testing PR #${PR_NUMBER}" + - else + - echo "Testing standard branch" + - fi + - cd ${HOMEGFS}/ctests/build + - ctest -R test_${CTEST}_setup + - ctest -R test_${CTEST}_stage + - ctest -R test_${CTEST}_execute + dependencies: + - create_ctests # Template for CTest jobs that will be used across machines .run_ctests_template: @@ -10,28 +47,21 @@ stage: run_tests parallel: matrix: - - CTEST: ["C48_ATM_gfs_fcst_seg0", "C48_S2SW_gfs_fcst_seg0", "C48_S2SW_gfs_atmos_prod"] - rules: - - if: $GITHUB_API_TRIGGER == "true" - -# The machine-specific jobs will be dynamically generated and added to the pipeline -# by the generate_pipelines.py script. They will follow this pattern: + - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod'] -# run_ctests-hera: -# extends: .run_ctests_template -# variables: -# machine: hera -# tags: ["hera"] -# dependencies: -# - build-hera -# Dummy job to ensure stage exists when no tests are defined -create_ctests: - stage: create_experiments - script: - - echo "CTests setup already handled during build stage" - rules: - - if: $GITHUB_API_TRIGGER == "true" - tags: - - hera +run_ctests-hera: + extends: .run_ctests_template + variables: + machine: hera + tags: ["hera"] + dependencies: + - create_ctests +run_ctests-gaeac6: + extends: .run_ctests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + dependencies: + - create_ctests diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 5f527fe0ffb..047158fe475 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -1,5 +1,6 @@ -# This file is used to define the GitLab CI/CD pipeline stages: -# building, setting up experiments, and running tests on different machines. +# Main gitlab-ci.yml for the global-workflow project +# This file defines common stages and includes appropriate modality file based on trigger + stages: - build - create_experiments @@ -19,16 +20,18 @@ variables: GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} -# Include ctests configuration for GitHub API triggered builds +# Include the appropriate modality file based on the trigger include: + # Include ctests configuration when triggered via GitHub API - local: 'dev/ci/.gitlab-ci-ctests.yml' rules: - if: $GITHUB_API_TRIGGER == "true" + # Include standard cases configuration when not triggered via GitHub API + - local: 'dev/ci/.gitlab-ci-cases.yml' + rules: + - if: $GITHUB_API_TRIGGER != "true" -.base_config: - variables: - GIT_STRATEGY: none - +# Common build template for all modalities .build_template: variables: GIT_STRATEGY: clone @@ -40,62 +43,8 @@ include: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} - # Add ctests build steps when triggered from GitHub API - - | - if [ "${GITHUB_API_TRIGGER}" == "true" ]; then - echo "Setting up for ctests workflow" - mkdir -p ${HOMEGFS}/ctests - cd ${HOMEGFS}/ctests - mkdir -p build - cd build - cmake ../.. - fi - -.setup_template: - extends: .base_config - stage: create_experiments - allow_failure: true - script: - - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - dependencies: - - build-${machine} - rules: - - if: $GITHUB_API_TRIGGER != "true" - -.run_tests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow - dependencies: - - setup_experiments-${machine} - rules: - - if: $GITHUB_API_TRIGGER != "true" - -.ctests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" - - if [ "${PR_NUMBER}" != "0" ]; then - - echo "Testing PR #${PR_NUMBER}" - - else - - echo "Testing standard branch" - - fi - - cd ${HOMEGFS}/ctests/build - - ctest -R test_${CTEST}_setup - - ctest -R test_${CTEST}_stage - - ctest -R test_${CTEST}_execute - dependencies: - - build-${machine} - rules: - - if: $GITHUB_API_TRIGGER == "true" +# Common finalization job for both modalities create_gist_from_svg_url: image: ghcr.io/cli/cli:latest stage: finalize @@ -116,80 +65,18 @@ create_gist_from_svg_url: variables: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# The following sections are generated for multiple hosts and dynamic case lists - -build-gaeac6: - extends: .build_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - -setup_experiments-gaeac6: - extends: .setup_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-gaeac6 - -run_tests-gaeac6: - extends: .run_tests_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-gaeac6 - -# CTests job for GitHub API triggered builds -run_ctests-gaeac6: - extends: .run_ctests_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - dependencies: - - build-gaeac6 +# The following sections are generated for multiple hosts by the generate_pipelines.py script build-hera: extends: .build_template variables: - mahine: hera - tags: ["hera"] - -setup_experiments-hera: - extends: .setup_template - variables: - mahine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-hera - -run_tests-hera: - extends: .run_tests_template - variables: - mahine: hera + machine: hera tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-hera -# CTests job for GitHub API triggered builds -run_ctests-hera: - extends: .run_ctests_template +build-gaeac6: + extends: .build_template variables: - machine: hera - tags: ["hera"] - dependencies: - - build-hera + machine: gaeac6 + tags: ["gaeac6"] # End of generated sections diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md index fcc33896227..a9b224a612f 100644 --- a/dev/ci/scripts/utils/gitlab/docs/README.md +++ b/dev/ci/scripts/utils/gitlab/docs/README.md @@ -35,7 +35,16 @@ The CTests pipeline when triggered via GitLab API from GitHub: - Uses the same build jobs as the standard pipeline - The build stage additionally sets up the CMake environment for ctests - Skips the standard experiment creation -- Runs specialized CTest test cases (`gfs_atmens`, `gfs_atm`, `gdas`, `gdas_enkf`) on each machine +- Runs specialized CTest test cases on each machine + +### Currently Supported CTests + +The pipeline currently supports the following tests: +- `C48_ATM_gfs_fcst_seg0` +- `C48_S2SW_gfs_fcst_seg0` +- `C48_S2SW_gfs_atmos_prod` + +Additional tests will be added in the future from within the ctest framework in `dev/ctests`. ## Shared Build Stage @@ -77,15 +86,15 @@ Replace the following: The configuration is split across multiple files: -1. **Template File** (`gitlab_pipeline_template.yml`): - - Contains the core pipeline structure - - Defines job templates for build, setup, and testing - - Includes modality detection and conditional logic +1. **Main Pipeline File** (`.gitlab-ci.yml`): + - Contains the common stages and variables + - Includes the appropriate modality file based on trigger + - Contains the build template shared by both modalities + - Contains the finalize stage job -2. **Generated Pipeline File** (`.gitlab-ci.yml`): - - Generated by `generate_pipelines.py` from the template - - Contains machine-specific jobs for both modalities - - Controls which jobs run based on trigger type +2. **Cases Configuration** (`.gitlab-ci-cases.yml`): + - Contains job templates specific to standard test cases + - Includes templates for running experiments 3. **CTests Configuration** (`.gitlab-ci-ctests.yml`): - Contains specific configurations for CTests diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index c30c1bcbcbd..ce159415dc8 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -99,24 +99,31 @@ def generate_machine_config(machine, case_list): Returns ------- - str - YAML configuration for the machine's build, setup, and run_tests jobs. + tuple + A tuple containing (main_config, cases_config, ctests_config) - YAML configurations for: + - main_config: build job for the main pipeline + - cases_config: setup and run_tests jobs for the cases modality + - ctests_config: run_ctests job for the ctests modality """ case_str = '", "'.join(case_list) case_list_yaml = f'["{case_str}"]' - machine_config = f''' + # Build job for the main pipeline + main_config = f''' build-{machine}: extends: .build_template variables: - mahine: {machine} + machine: {machine} tags: ["{machine}"] +''' + # Setup and run_tests jobs for the cases modality + cases_config = f''' setup_experiments-{machine}: extends: .setup_template variables: - mahine: {machine} + machine: {machine} tags: ["{machine}"] parallel: matrix: @@ -127,24 +134,27 @@ def generate_machine_config(machine, case_list): run_tests-{machine}: extends: .run_tests_template variables: - mahine: {machine} + machine: {machine} tags: ["{machine}"] parallel: matrix: - caseName: {case_list_yaml} dependencies: - setup_experiments-{machine} +''' -# CTests job for GitHub API triggered builds + # CTests job for the ctests modality + ctests_config = f''' run_ctests-{machine}: extends: .run_ctests_template variables: machine: {machine} tags: ["{machine}"] dependencies: - - build-{machine} + - create_ctests ''' - return machine_config + + return (main_config, cases_config, ctests_config) def read_template_file(template_path): @@ -198,18 +208,51 @@ def generate_pipeline_config(machines, template_file, output_file=None): ValueError If the template file does not exist. """ - # Set default output file path if not specified - output_file = output_file or os.path.join(_homegfs, 'dev/ci', '.gitlab-ci.yml') - - # Initialize with the template content - if os.path.exists(template_file): - config_content = read_template_file(template_file) + # Set default output file paths if not specified + main_output = output_file or os.path.join(_homegfs, 'dev/ci', '.gitlab-ci.yml') + cases_output = os.path.join(_homegfs, 'dev/ci', '.gitlab-ci-cases.yml') + ctests_output = os.path.join(_homegfs, 'dev/ci', '.gitlab-ci-ctests.yml') + + # Read the current contents of the files to preserve header sections + if os.path.exists(main_output): + with open(main_output, 'r') as f: + main_content = f.read() + # Extract content before the generated sections + main_marker = "# The following sections are generated" + main_parts = main_content.split(main_marker) + main_header = main_parts[0] if len(main_parts) > 0 else main_content else: - raise ValueError(f"Template file {template_file} not found") - - # Add a comment to indicate the start of generated sections - config_content += '\n# The following sections are generated for multiple hosts and dynamic case lists\n' + # If file doesn't exist, use template content + if os.path.exists(template_file): + main_header = read_template_file(template_file) + else: + raise ValueError(f"Template file {template_file} not found") + + # Do the same for cases file + if os.path.exists(cases_output): + with open(cases_output, 'r') as f: + cases_content = f.read() + cases_marker = "# The machine-specific jobs will be generated" + cases_parts = cases_content.split(cases_marker) + cases_header = cases_parts[0] if len(cases_parts) > 0 else cases_content + else: + cases_header = "# Standard cases modality configuration\n\n" + + # And for ctests file + if os.path.exists(ctests_output): + with open(ctests_output, 'r') as f: + ctests_content = f.read() + ctests_marker = "# The machine-specific jobs will be generated" + ctests_parts = ctests_content.split(ctests_marker) + ctests_header = ctests_parts[0] if len(ctests_parts) > 0 else ctests_content + else: + ctests_header = "# CTests modality configuration\n\n" + # Initialize with the headers + main_config = main_header.rstrip() + "\n\n# The following sections are generated for multiple hosts by the generate_pipelines.py script\n" + cases_config = cases_header + ctests_config = ctests_header + # Generate machine-specific configurations for machine in machines: case_list = get_case_list_for_machine(machine) @@ -217,20 +260,40 @@ def generate_pipeline_config(machines, template_file, output_file=None): print(f"Warning: No supported cases found for machine {machine}", file=sys.stderr) continue - machine_config = generate_machine_config(machine, case_list) - config_content += machine_config - - # Add a comment to indicate the end of generated sections - config_content += '\n# End of generated sections\n' - - # Ensure no blank line at the top of the resulting pipeline - config_content = config_content.lstrip() - - # Write the complete configuration to the output file - with open(output_file, 'w') as f: - f.write(config_content) - - print(f"GitLab CI pipeline configuration generated at {output_file}") + main_part, cases_part, ctests_part = generate_machine_config(machine, case_list) + main_config += main_part + # Only add cases and ctests parts if they're non-empty + if cases_part.strip(): + cases_config += cases_part + if ctests_part.strip(): + ctests_config += ctests_part + + # Add comments to indicate the end of generated sections + main_config += '\n# End of generated sections\n' + + # Ensure no blank line at the top of the resulting pipelines + main_config = main_config.lstrip() + cases_config = cases_config.lstrip() + ctests_config = ctests_config.lstrip() + + # Write the complete configurations to the output files + with open(main_output, 'w') as f: + f.write(main_config) + + # Only write to cases and ctests files if they don't contain markers + # This allows us to preserve template sections in these files + if "# The machine-specific jobs will be generated" in cases_content: + with open(cases_output, 'w') as f: + f.write(cases_config) + + if "# The machine-specific jobs will be generated" in ctests_content: + with open(ctests_output, 'w') as f: + f.write(ctests_config) + + print(f"GitLab CI pipeline configurations generated:") + print(f" - Main pipeline: {main_output}") + print(f" - Cases modality: {cases_output}") + print(f" - CTests modality: {ctests_output}") def main(): diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 6dac7a06f19..7c9c1c87a45 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -25,16 +25,23 @@ variables: GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} -# Include ctests configuration for GitHub API triggered builds +# Include the appropriate modality file based on the trigger include: + # Include ctests configuration when triggered via GitHub API - local: 'dev/ci/.gitlab-ci-ctests.yml' rules: - if: $GITHUB_API_TRIGGER == "true" + # Include standard cases configuration when not triggered via GitHub API + - local: 'dev/ci/.gitlab-ci-cases.yml' + rules: + - if: $GITHUB_API_TRIGGER != "true" +# Base configuration .base_config: variables: GIT_STRATEGY: none +# Common build template for all modalities .build_template: variables: GIT_STRATEGY: clone @@ -46,62 +53,8 @@ include: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} - # Add ctests build steps when triggered from GitHub API - - | - if [ "${GITHUB_API_TRIGGER}" == "true" ]; then - echo "Setting up for ctests workflow" - mkdir -p ${HOMEGFS}/ctests - cd ${HOMEGFS}/ctests - mkdir -p build - cd build - cmake ../.. - fi - -.setup_template: - extends: .base_config - stage: create_experiments - allow_failure: true - script: - - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - dependencies: - - build-${machine} - rules: - - if: $GITHUB_API_TRIGGER != "true" - -.run_tests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow - dependencies: - - setup_experiments-${machine} - rules: - - if: $GITHUB_API_TRIGGER != "true" - -.ctests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" - - if [ "${PR_NUMBER}" != "0" ]; then - - echo "Testing PR #${PR_NUMBER}" - - else - - echo "Testing standard branch" - - fi - - cd ${HOMEGFS}/ctests/build - - ctest -R test_${CTEST}_setup - - ctest -R test_${CTEST}_stage - - ctest -R test_${CTEST}_execute - dependencies: - - build-${machine} - rules: - - if: $GITHUB_API_TRIGGER == "true" +# Common finalization job for both modalities create_gist_from_svg_url: image: ghcr.io/cli/cli:latest stage: finalize From eeb78e7fedb0a7bee3f4d7d41b1ea94f834b23e5 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 05:19:31 -0400 Subject: [PATCH 03/76] removed badge update exploration and created a generlized version as a seperate pipeline for a proposed emc-bot project in GitLab --- dev/ci/.gitlab-ci.yml | 22 +----- .../utils/gitlab/badge-updater-pipeline.yml | 79 +++++++++++++++++++ 2 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 dev/ci/scripts/utils/gitlab/badge-updater-pipeline.yml diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 047158fe475..16f621ec1f0 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -44,26 +44,8 @@ include: - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} -# Common finalization job for both modalities -create_gist_from_svg_url: - image: ghcr.io/cli/cli:latest - stage: finalize - allow_failure: true - when: always - script: - - | - # Set your SVG URL, output filename, and Gist ID - SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/develop/pipeline.svg" - SVG_FILE="badge.svg" - GIST_ID="ab937691224bdf50427cbeca666bf67b" - # Download the SVG (add authentication if needed) - curl -L "$SVG_URL" -o "$SVG_FILE" - # Authenticate GitHub CLI (requires GITHUB_TOKEN env variable) - echo "$GITHUB_TOKEN" | gh auth login --with-token - # Update the existing public Gist with the SVG file - gh gist edit "$GIST_ID" "$SVG_FILE" - variables: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# Note: Badge updating has been moved to a separate CI/CD pipeline +# See the repository: global-workflow-badges # The following sections are generated for multiple hosts by the generate_pipelines.py script diff --git a/dev/ci/scripts/utils/gitlab/badge-updater-pipeline.yml b/dev/ci/scripts/utils/gitlab/badge-updater-pipeline.yml new file mode 100644 index 00000000000..b7bd31e3752 --- /dev/null +++ b/dev/ci/scripts/utils/gitlab/badge-updater-pipeline.yml @@ -0,0 +1,79 @@ +# EMC-Bot Badge Updater Pipeline +# This pipeline is designed to be triggered by webhooks when other pipelines complete +# It handles updating GitHub Gists with pipeline status badges for NOAA's Global Workflow + +stages: + - update_badges + +update_all_badges: + image: bitnami/git:latest + stage: update_badges + script: + - | + # Install GitHub CLI and other required tools + apt-get update -qq && apt-get install -y -qq curl jq + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null + apt-get update -qq && apt-get install -y -qq gh + + # Authenticate GitHub CLI + echo "$GITHUB_TOKEN" | gh auth login --with-token + + # Create a temporary directory for badges + mkdir -p badges + + # Get information about the triggering pipeline + echo "Processing pipeline event data..." + PIPELINE_PROJECT_ID="${CI_PIPELINE_SOURCE_PROJECT_ID:-none}" + PIPELINE_BRANCH="${CI_PIPELINE_SOURCE_BRANCH:-develop}" + + # Default to global-workflow project if not specified + if [ "$PIPELINE_PROJECT_ID" == "none" ]; then + PROJECT_PATH="NWS/Operations/NCEP/EMC/global-workflow" + else + # You could look up the project path from the ID if needed + PROJECT_PATH="NWS/Operations/NCEP/EMC/global-workflow" + fi + + # Update main develop branch pipeline badge + echo "Updating develop branch badge..." + curl -L "https://vlab.noaa.gov/gitlab-licensed/${PROJECT_PATH}/badges/${PIPELINE_BRANCH}/pipeline.svg" -o "badges/main_pipeline.svg" + gh gist edit "ab937691224bdf50427cbeca666bf67b" "badges/main_pipeline.svg" + + # Update machine-specific badges + # These badges enhance visibility into the operational status of the global weather modeling system + # across NOAA's high-performance computing environments + + # HERA - NOAA Research HPC + echo "Updating HERA badge..." + curl -L "https://vlab.noaa.gov/gitlab-licensed/${PROJECT_PATH}/badges/${PIPELINE_BRANCH}/pipeline.svg?job=build-hera" -o "badges/hera_badge.svg" + gh gist edit "hera_gist_id" "badges/hera_badge.svg" + + # GAEA-C6 - Department of Energy HPC at Oak Ridge + echo "Updating GAEA-C6 badge..." + curl -L "https://vlab.noaa.gov/gitlab-licensed/${PROJECT_PATH}/badges/${PIPELINE_BRANCH}/pipeline.svg?job=build-gaeac6" -o "badges/gaeac6_badge.svg" + gh gist edit "gaeac6_gist_id" "badges/gaeac6_badge.svg" + + # WCOSS2 - NOAA Operational Weather and Climate Supercomputing System + echo "Updating WCOSS2 badge..." + curl -L "https://vlab.noaa.gov/gitlab-licensed/${PROJECT_PATH}/badges/${PIPELINE_BRANCH}/pipeline.svg?job=build-wcoss2" -o "badges/wcoss2_badge.svg" + gh gist edit "wcoss2_gist_id" "badges/wcoss2_badge.svg" + + # ORION - Navy HPC system + echo "Updating ORION badge..." + curl -L "https://vlab.noaa.gov/gitlab-licensed/${PROJECT_PATH}/badges/${PIPELINE_BRANCH}/pipeline.svg?job=build-orion" -o "badges/orion_badge.svg" + gh gist edit "orion_gist_id" "badges/orion_badge.svg" + + # Add future HPC systems here as they're integrated into the workflow + # This modular design allows for easy extension as computational resources expand + + echo "All badges updated successfully!" + variables: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # This allows the job to run even if GitLab can't access the gists for verification + allow_failure: true + + # Only run this when triggered by a webhook or manually + rules: + - if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "web" From d23d787b54a0757085ab14228b4624d840779833 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 06:32:28 -0400 Subject: [PATCH 04/76] simplifing the demarcation of generated host/case sections of main multi-modal pipeline step one --- dev/ci/.gitlab-ci-cases-hosts.yml | 46 ++++++++++++++ dev/ci/.gitlab-ci-cases.yml | 47 +------------- dev/ci/.gitlab-ci-hosts.yml | 101 ++++++++++++++++++++++++++++++ dev/ci/.gitlab-ci.yml | 21 +------ 4 files changed, 152 insertions(+), 63 deletions(-) create mode 100644 dev/ci/.gitlab-ci-cases-hosts.yml create mode 100644 dev/ci/.gitlab-ci-hosts.yml diff --git a/dev/ci/.gitlab-ci-cases-hosts.yml b/dev/ci/.gitlab-ci-cases-hosts.yml new file mode 100644 index 00000000000..1518f65e75f --- /dev/null +++ b/dev/ci/.gitlab-ci-cases-hosts.yml @@ -0,0 +1,46 @@ +# Host-specific case configurations for the global-workflow project +# This file defines host-specific setup and test execution jobs + +setup_experiments-hera: + extends: .setup_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-hera + +run_tests-hera: + extends: .run_tests_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-hera + +setup_experiments-gaeac6: + extends: .setup_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-gaeac6 + +run_tests-gaeac6: + extends: .run_tests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml index 30d10ca5e74..7109a68c065 100644 --- a/dev/ci/.gitlab-ci-cases.yml +++ b/dev/ci/.gitlab-ci-cases.yml @@ -29,47 +29,6 @@ dependencies: - setup_experiments-${machine} - -setup_experiments-hera: - extends: .setup_template - variables: - machine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-hera - -run_tests-hera: - extends: .run_tests_template - variables: - machine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-hera - -setup_experiments-gaeac6: - extends: .setup_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-gaeac6 - -run_tests-gaeac6: - extends: .run_tests_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-gaeac6 +# Include host-specific job configurations +include: + - local: 'dev/ci/.gitlab-ci-cases-hosts.yml' diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml new file mode 100644 index 00000000000..09fa73ce1cc --- /dev/null +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -0,0 +1,101 @@ +# Host-specific configurations for the global-workflow project +# This file contains host-specific configurations for both standard cases and CTests modes +# It is designed to be generated automatically by the pipeline generator + +# Standard Cases configurations by host +# ======================================= + +# Host: Hera - Standard Cases +setup_experiments-hera: + extends: .setup_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-hera + rules: + - if: $GITHUB_API_TRIGGER != "true" + +run_tests-hera: + extends: .run_tests_template + variables: + machine: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-hera + rules: + - if: $GITHUB_API_TRIGGER != "true" + +# Host: GAEAC6 - Standard Cases +setup_experiments-gaeac6: + extends: .setup_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-gaeac6 + rules: + - if: $GITHUB_API_TRIGGER != "true" + +run_tests-gaeac6: + extends: .run_tests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 + rules: + - if: $GITHUB_API_TRIGGER != "true" + +# CTests configurations by host +# ======================================= + +# Host: Hera - CTests +run_ctests-hera: + extends: .run_ctests_template + variables: + machine: hera + tags: ["hera"] + dependencies: + - create_ctests + rules: + - if: $GITHUB_API_TRIGGER == "true" + +# Host: GAEAC6 - CTests +run_ctests-gaeac6: + extends: .run_ctests_template + variables: + machine: gaeac6 + tags: ["gaeac6"] + dependencies: + - create_ctests + rules: + - if: $GITHUB_API_TRIGGER == "true" + +# ======================================= +# Common build configurations by host +# These will always be included, regardless of modality + +build-hera: + extends: .build_template + variables: + machine: hera + tags: ["hera"] + +build-gaeac6: + extends: .build_template + variables: + machine: gaeac6 + tags: ["gaeac6"] diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 16f621ec1f0..5d6c4842f61 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -30,6 +30,8 @@ include: - local: 'dev/ci/.gitlab-ci-cases.yml' rules: - if: $GITHUB_API_TRIGGER != "true" + # Always include host-specific configurations + - local: 'dev/ci/.gitlab-ci-hosts.yml' # Common build template for all modalities .build_template: @@ -43,22 +45,3 @@ include: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} - -# Note: Badge updating has been moved to a separate CI/CD pipeline -# See the repository: global-workflow-badges - -# The following sections are generated for multiple hosts by the generate_pipelines.py script - -build-hera: - extends: .build_template - variables: - machine: hera - tags: ["hera"] - -build-gaeac6: - extends: .build_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - -# End of generated sections From 427b72f2150748b3c7c8707d28c405939a8fbe6c Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 08:00:30 -0400 Subject: [PATCH 05/76] finaly have base mix-modality framework with each mode in each pipeline with a common build in the main caller, now all the per-host-case/ctest stuff in one single but seperate file --- dev/ci/.gitlab-ci-cases.yml | 6 +---- dev/ci/.gitlab-ci-ctests.yml | 48 +++++++++--------------------------- dev/ci/.gitlab-ci-hosts.yml | 19 +++++++------- 3 files changed, 22 insertions(+), 51 deletions(-) diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml index 7109a68c065..f0f0ac3c155 100644 --- a/dev/ci/.gitlab-ci-cases.yml +++ b/dev/ci/.gitlab-ci-cases.yml @@ -27,8 +27,4 @@ - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - - setup_experiments-${machine} - -# Include host-specific job configurations -include: - - local: 'dev/ci/.gitlab-ci-cases-hosts.yml' + - setup_experiments-${machine} \ No newline at end of file diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index f047bcdc784..dd0cc65f1dd 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -1,14 +1,9 @@ -# CTests modality configuration for the global-workflow project -# This file defines jobs specific to the CTests workflow -# It is included when triggered via GitHub API (GITHUB_API_TRIGGER=true) - -# Base configuration for ctest jobs -.base_config: - variables: - GIT_STRATEGY: none +# ======================================= +# Templates for CTests +# ======================================= # Setup job for ctests using CMake -create_ctests: +.create_ctests: extends: .base_config stage: create_experiments script: @@ -20,10 +15,12 @@ create_ctests: dependencies: - build-${machine} tags: - - hera + - ${machine} + rules: + - if: $GITHUB_API_TRIGGER == "true" -# Template for CTest execution jobs -.ctests_template: +# Main template for CTest execution jobs +.run_ctests_template: extends: .base_config stage: run_tests allow_failure: true @@ -40,28 +37,5 @@ create_ctests: - ctest -R test_${CTEST}_execute dependencies: - create_ctests - -# Template for CTest jobs that will be used across machines -.run_ctests_template: - extends: .ctests_template - stage: run_tests - parallel: - matrix: - - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod'] - - -run_ctests-hera: - extends: .run_ctests_template - variables: - machine: hera - tags: ["hera"] - dependencies: - - create_ctests - -run_ctests-gaeac6: - extends: .run_ctests_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - dependencies: - - create_ctests + rules: + - if: $GITHUB_API_TRIGGER == "true" diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 09fa73ce1cc..8c851aad6e2 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -1,7 +1,4 @@ -# Host-specific configurations for the global-workflow project -# This file contains host-specific configurations for both standard cases and CTests modes -# It is designed to be generated automatically by the pipeline generator - +# ======================================= # Standard Cases configurations by host # ======================================= @@ -62,11 +59,17 @@ run_tests-gaeac6: # CTests configurations by host # ======================================= +# Template for CTest jobs that will be used across machines +.run_ctests_cases_template: + extends: .ctests_template + stage: run_tests + parallel: + matrix: + - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod'] + # Host: Hera - CTests run_ctests-hera: - extends: .run_ctests_template - variables: - machine: hera + extends: .run_ctests_cases_template tags: ["hera"] dependencies: - create_ctests @@ -76,8 +79,6 @@ run_ctests-hera: # Host: GAEAC6 - CTests run_ctests-gaeac6: extends: .run_ctests_template - variables: - machine: gaeac6 tags: ["gaeac6"] dependencies: - create_ctests From 7e5ce32ef05843f8f47521f2d875aab6e020f0ed Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 14 Apr 2025 14:56:47 -0400 Subject: [PATCH 06/76] updated multi-host with semi-dynamic case lists for GitLab pipelines for schecheduled CI runs to work with new virtical structure --- dev/ci/.gitlab-ci.yml | 117 +++++++--- dev/ci/scripts/utils/get_host_case_list.py | 43 +++- .../utils/gitlab/generate_pipelines.py | 215 ++++++++++++++++++ .../utils/gitlab/gitlab_pipeline_template.yml | 58 +++++ .../{ => gitlab}/launch_gitlab_runner.sh | 0 5 files changed, 387 insertions(+), 46 deletions(-) create mode 100755 dev/ci/scripts/utils/gitlab/generate_pipelines.py create mode 100644 dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml rename dev/ci/scripts/utils/{ => gitlab}/launch_gitlab_runner.sh (100%) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 810334c0ace..e344ecbc71f 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -1,69 +1,112 @@ +# This file is used to define the GitLab CI/CD pipeline stages: +# building, setting up experiments, and running tests on different machines. stages: - build - create_experiments - run_tests -# Global variables variables: - BUILD: 'TODAY' - GIT_CLONE_PATH: '${CI_BUILDS_DIR}/${BUILD}/global-workflow' + CI_RUN_PATH: ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA} + GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} - RUNTESTS_DIR: ${CI_BUILDS_DIR}/${BUILD}/RUNTESTS + RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h +.base_config: + variables: + GIT_STRATEGY: none -# Build stage for the global workflow on compute nodes -build: +.build_template: variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive stage: build + allow_failure: true script: - - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" + - echo "Using build directory ${HOMEGFS}" - dev/ci/scripts/utils/ci_utils_wrapper.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} - # TODO - Add more machines to the list and make (next PR) - #parallel: - # matrix: - # - MACHINE: ["gaeac6"] - tags: - - gaeac6 - -# Create experiments stage from a fixed list of cases in $HOMEGFS/dev/ci/cases/pr -# TODO: Next PR has simi-dynamic caseName lists -setup_experiments: - variables: - GIT_STRATEGY: none +.setup_template: + extends: .base_config stage: create_experiments + allow_failure: true script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - parallel: - matrix: - - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] - tags: - - gaeac6 + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml dependencies: - - build + - build-${machine} -# Running the list of experiments created in the previous stage -# using the run-check_ci.sh script from $HOMEgfs/dev/ci/scripts directory -run_tests: - variables: - GIT_STRATEGY: none +.run_tests_template: + extends: .base_config stage: run_tests + allow_failure: true script: - - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" + - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_BUILDS_DIR}/${BUILD} ${pslot} global-workflow + - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + dependencies: + - setup_experiments-${machine} + +# The following sections are generated for multiple hosts and dynamic case lists + +build-gaeac6: + extends: .build_template + variables: + MACHINE: gaeac6 + tags: ["gaeac6"] + +setup_experiments-gaeac6: + extends: .setup_template + variables: + MACHINE: gaeac6 + tags: ["gaeac6"] parallel: matrix: - - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] - tags: - - gaeac6 + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] dependencies: - - setup_experiments + - build-gaeac6 + +run_tests-gaeac6: + extends: .run_tests_template + variables: + MACHINE: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 + +build-hera: + extends: .build_template + variables: + MACHINE: hera + tags: ["hera"] + +setup_experiments-hera: + extends: .setup_template + variables: + MACHINE: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-hera + +run_tests-hera: + extends: .run_tests_template + variables: + MACHINE: hera + tags: ["hera"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-hera + +# End of generated sections diff --git a/dev/ci/scripts/utils/get_host_case_list.py b/dev/ci/scripts/utils/get_host_case_list.py index 59c62539181..78e1d6148cf 100755 --- a/dev/ci/scripts/utils/get_host_case_list.py +++ b/dev/ci/scripts/utils/get_host_case_list.py @@ -9,24 +9,49 @@ _here = os.path.dirname(__file__) _top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) -if __name__ == '__main__': - if len(sys.argv) < 2: - print('Usage: get_host_case_list.py ') - sys.exit(1) +def get_host_cases(host, homegfs=None): + """ + Get list of test cases supported on a host - host = sys.argv[1] + Args: + host (str): Host name to check + homegfs (str, optional): Path to the global-workflow repository root directory + Returns: + list: List of case names (without extension) supported on the host + """ + homegfs = homegfs or _top case_list = [] - HOMEgfs = _top - data = AttrDict(HOMEgfs=_top) + + # Set up data for template rendering + data = AttrDict(HOMEgfs=homegfs) data.update(os.environ) - case_files = glob.glob(f'{HOMEgfs}/dev/ci/cases/pr/*.yaml') + # Get all case files + case_files = glob.glob(f'{homegfs}/dev/ci/cases/pr/*.yaml') + for case_yaml in case_files: + # Parse the case configuration case_conf = parse_j2yaml(path=case_yaml, data=data) + + # Skip cases that don't support this host if 'skip_ci_on_hosts' in case_conf: if host.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]: continue + + # Add the case name (without extension) to the list case_list.append(splitext(basename(case_yaml))[0]) - print(' '.join(case_list)) + + return case_list + + +if __name__ == '__main__': + # When run as a script, maintain the original behavior + if len(sys.argv) < 2: + print('Usage: get_host_case_list.py ') + sys.exit(1) + + host = sys.argv[1] + cases = get_host_cases(host) + print(' '.join(cases)) diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py new file mode 100755 index 00000000000..b37c5516c17 --- /dev/null +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +Script to generate GitLab CI pipeline configuration based on supported test cases +for each machine. + +This script uses get_host_case_list.py to determine which test cases are supported +on each machine and generates the complete .gitlab-ci.yml configuration by combining +a template with machine-specific sections. +""" + +import os +import sys +import argparse +from pathlib import Path + +# update sys.path to include libs in the parent directory +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from get_host_case_list import get_host_cases + +# Get the path to the top directory of the repository +# to place the generated .gitlab-ci.yml file in the default location +_here = os.path.dirname(os.path.abspath(__file__)) +_top = os.path.abspath(os.path.join(_here, '../../../../..')) + +def get_case_list_for_machine(machine): + """ + Get the list of supported cases for the given machine. + + Parameters + ---------- + machine : str + The name of the machine to get supported cases for. + + Returns + ------- + list + A list of test case names supported on the specified machine. + """ + + cases = get_host_cases(machine, homegfs=_top) + return cases + + +def generate_machine_config(machine, case_list): + """ + Generate the machine-specific configuration sections for GitLab CI. + + Parameters + ---------- + machine : str + The name of the machine for which to generate configuration. + case_list : list + A list of test case names supported on the machine. + + Returns + ------- + str + YAML configuration for the machine's build, setup, and run_tests jobs. + """ + + case_str = '", "'.join(case_list) + case_list_yaml = f'["{case_str}"]' + + machine_config = f''' +build-{machine}: + extends: .build_template + variables: + MACHINE: {machine} + tags: ["{machine}"] + +setup_experiments-{machine}: + extends: .setup_template + variables: + MACHINE: {machine} + tags: ["{machine}"] + parallel: + matrix: + - caseName: {case_list_yaml} + dependencies: + - build-{machine} + +run_tests-{machine}: + extends: .run_tests_template + variables: + MACHINE: {machine} + tags: ["{machine}"] + parallel: + matrix: + - caseName: {case_list_yaml} + dependencies: + - setup_experiments-{machine} +''' + return machine_config + + +def read_template_file(template_path): + """ + Read the template file and extract the content below the demarcation line. + + Parameters + ---------- + template_path : str + Path to the template file containing the base GitLab CI configuration. + + Returns + ------- + str + Content of the template file below the demarcation line. + """ + with open(template_path, 'r') as f: + lines = f.readlines() + + # Find the demarcation line that separates the header from the usable template + marker_line = "# ------------------------------------------------------------" + + for i, line in enumerate(lines): + if line.strip() == marker_line: + return ''.join(lines[i + 1:]) + + # If marker line not found, return an empty string + return '' + + +def generate_pipeline_config(machines, template_file, output_file=None): + """ + Generate the complete GitLab CI pipeline configuration. + + This function combines a template file with machine-specific configurations + based on the supported test cases for each machine. + + Parameters + ---------- + machines : list + List of machine names to include in the pipeline configuration. + template_file : str + Path to the template file containing the base configuration. + The template should end with a marker line: "# Machine-specific jobs generated from template:" + output_file : str, optional + Path where the generated configuration will be written. + If not provided, defaults to ci/.gitlab-ci.yml in the repository root. + + Raises + ------ + ValueError + If the template file does not exist. + """ + # Set default output file path if not specified + output_file = output_file or os.path.join(_top, 'dev/ci', '.gitlab-ci.yml') + + # Initialize with the template content + if os.path.exists(template_file): + config_content = read_template_file(template_file) + else: + raise ValueError(f"Template file {template_file} not found") + + # Add a comment to indicate the start of generated sections + config_content += '\n# The following sections are generated for multiple hosts and dynamic case lists\n' + + # Generate machine-specific configurations + for machine in machines: + case_list = get_case_list_for_machine(machine) + if not case_list: + print(f"Warning: No supported cases found for machine {machine}", file=sys.stderr) + continue + + machine_config = generate_machine_config(machine, case_list) + config_content += machine_config + + # Add a comment to indicate the end of generated sections + config_content += '\n# End of generated sections\n' + + # Ensure no blank line at the top of the resulting pipeline + config_content = config_content.lstrip() + + # Write the complete configuration to the output file + with open(output_file, 'w') as f: + f.write(config_content) + + print(f"GitLab CI pipeline configuration generated at {output_file}") + + +def main(): + """ + Parse command line arguments and generate the GitLab CI pipeline configuration. + + This is the main entry point for the script when executed directly. + It parses command line arguments and calls generate_pipeline_config() + with the appropriate parameters. + + Command line arguments: + --machines : str + Comma-separated list of machines to include in the pipeline. + --template : str + Path to the template file containing the base configuration. + --output : str, optional + Path where the generated configuration will be written. + + Returns + ------- + None + """ + + parser = argparse.ArgumentParser(description='Generate GitLab CI pipeline configuration.') + parser.add_argument('--machines', required=True, help='Comma-separated list of machines to include in the pipeline') + parser.add_argument('--template', required=True, help='Path to the template file for the pipeline configuration') + parser.add_argument('--output', default=None, help='Path to the output file (default: ci/.gitlab-ci.yml)') + + args = parser.parse_args() + + machines = [machine.strip() for machine in args.machines.split(',')] + generate_pipeline_config(machines, args.template, args.output) + + +if __name__ == '__main__': + main() diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml new file mode 100644 index 00000000000..a036a2886a8 --- /dev/null +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -0,0 +1,58 @@ +# THIS FILE IS THE CORE TEMPLATE FOR THE GITLAB CI/CD PIPELINES +# Lines below the following demarcation line will be included in the resulting pipeline. +# Do not edit the generated sections in the resulting .gitlab-ci.yml file when updating +# the system but here instead (in this file). +# ------------------------------------------------------------ + +# This file is used to define the GitLab CI/CD pipeline stages: +# building, setting up experiments, and running tests on different machines. +stages: + - build + - create_experiments + - run_tests + +variables: + CI_RUN_PATH: ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA} + GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' + HOMEGFS: ${GIT_CLONE_PATH} + RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS + GIT_DEPTH: 1 + RUNNER_SCRIPT_TIMEOUT: 6h + RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + +.base_config: + variables: + GIT_STRATEGY: none + +.build_template: + variables: + GIT_STRATEGY: clone + GIT_SUBMODULE_STRATEGY: recursive + stage: build + allow_failure: true + script: + - echo "Using build directory ${HOMEGFS}" + - dev/ci/scripts/utils/ci_utils_wrapper.sh build_compute + - sorc/link_workflow.sh + - mkdir -p ${RUNTESTS_DIR} + +.setup_template: + extends: .base_config + stage: create_experiments + allow_failure: true + script: + - export RUNTESTS=${RUNTESTS_DIR} + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + dependencies: + - build-${machine} + +.run_tests_template: + extends: .base_config + stage: run_tests + allow_failure: true + script: + - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" + - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + dependencies: + - setup_experiments-${machine} diff --git a/dev/ci/scripts/utils/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh similarity index 100% rename from dev/ci/scripts/utils/launch_gitlab_runner.sh rename to dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh From b3ddc8cc3b4b06e8eeea07b9a848b27d2e6a08d1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 14 Apr 2025 17:59:43 -0400 Subject: [PATCH 07/76] added BUILD back into the build path for GitLab CI jobs which vaule can be minipuplated withing the GitLab controller --- dev/ci/.gitlab-ci.yml | 3 ++- dev/ci/scripts/utils/gitlab/generate_pipelines.py | 1 + dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index e344ecbc71f..053e2c07cf7 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -6,7 +6,8 @@ stages: - run_tests variables: - CI_RUN_PATH: ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA} + BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' + CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index b37c5516c17..986f085c89a 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -22,6 +22,7 @@ _here = os.path.dirname(os.path.abspath(__file__)) _top = os.path.abspath(os.path.join(_here, '../../../../..')) + def get_case_list_for_machine(machine): """ Get the list of supported cases for the given machine. diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index a036a2886a8..131bad0409b 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -12,7 +12,8 @@ stages: - run_tests variables: - CI_RUN_PATH: ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA} + BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' + CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS From b2602161babf47f1bc819d029d6f3a5adc49520f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 14 Apr 2025 19:09:21 -0400 Subject: [PATCH 08/76] folded ci_utils_wrapper into ci_utils because that was redundent to have it in two files --- dev/ci/.gitlab-ci.yml | 6 +-- dev/ci/Jenkinsfile | 43 +++++++++---------- dev/ci/Jenkinsfile4AWS | 14 +++--- dev/ci/scripts/check_ci.sh | 2 +- dev/ci/scripts/driver.sh | 2 +- dev/ci/scripts/utils/ci_utils.sh | 26 +++++++++++ dev/ci/scripts/utils/ci_utils_wrapper.sh | 9 ---- .../utils/gitlab/gitlab_pipeline_template.yml | 6 +-- 8 files changed, 62 insertions(+), 46 deletions(-) delete mode 100755 dev/ci/scripts/utils/ci_utils_wrapper.sh diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 053e2c07cf7..0072cacae05 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -27,7 +27,7 @@ variables: allow_failure: true script: - echo "Using build directory ${HOMEGFS}" - - dev/ci/scripts/utils/ci_utils_wrapper.sh build_compute + - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} @@ -37,7 +37,7 @@ variables: allow_failure: true script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} @@ -47,7 +47,7 @@ variables: allow_failure: true script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index 3d31079e110..d65c3dc9704 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables + sh(script: "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" if ( fileExists("logs/error.logs") ) { @@ -148,12 +148,12 @@ pipeline { } try { sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} """) gist_url=sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) } catch (Exception error_comment) { @@ -176,8 +176,8 @@ pipeline { } // Get a list of CI cases to run CI_CASES = sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/get_host_case_list.py ${machine} + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/get_host_case_list.py ${machine} """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } @@ -202,8 +202,8 @@ pipeline { def error_output = "" try { error_output = sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs_dev}/ci/cases/pr/${caseName}.yaml + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body '${caseName} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n```\n${error_output}```' """) @@ -215,22 +215,22 @@ pipeline { stage("Running ${caseName}") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { - def pslot = sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() + def pslot = sh(script: "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} """) } catch (Exception error_experment) { sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot} """) ws(CUSTOM_WORKSPACE) { def error_logs = "" @@ -252,13 +252,13 @@ pipeline { } try { gist_url = sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) sh(script: """ - source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + source ${HOMEgfs}/dev/workflow/gw_setup.sh + ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -282,9 +282,8 @@ pipeline { }] } parallel parallelStages + [failFast: true] - } + } } - } } stage( '5. Finalize' ) { diff --git a/dev/ci/Jenkinsfile4AWS b/dev/ci/Jenkinsfile4AWS index 899558087cb..b77d7cbb945 100644 --- a/dev/ci/Jenkinsfile4AWS +++ b/dev/ci/Jenkinsfile4AWS @@ -134,7 +134,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables + // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables sh(script: './build_compute.sh -A ${USER} gfs gefs sfs') // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" @@ -205,10 +205,10 @@ pipeline { script { env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { - error_output = sh(script: """ + error_output = sh(script: \"\"\" source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml - """, returnStdout: true).trim() + ${HOMEgfs}/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml + \"\"\", returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) error("Case ${caseName} failed to create experiment directory") @@ -219,7 +219,7 @@ pipeline { stage("Running ${caseName}") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { - def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() + def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { @@ -229,12 +229,12 @@ pipeline { """) sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + ${HOMEgfs}/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} """) } catch (Exception error_experment) { sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + ${HOMEgfs}/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot} """) ws(CUSTOM_WORKSPACE) { def error_logs = "" diff --git a/dev/ci/scripts/check_ci.sh b/dev/ci/scripts/check_ci.sh index b1a8fa05d9b..0dd7a2807ba 100755 --- a/dev/ci/scripts/check_ci.sh +++ b/dev/ci/scripts/check_ci.sh @@ -168,7 +168,7 @@ for pr in ${pr_list}; do fi if [[ "${rocoto_state}" == "DONE" ]]; then #Remove Experment cases that completed successfully - "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}" + "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" cleanup_experiment "${pslot_dir}" rm -f "${output_ci_single}" # echo "\`\`\`" > "${output_ci_single}" DATE=$(date +'%D %r') diff --git a/dev/ci/scripts/driver.sh b/dev/ci/scripts/driver.sh index fd1d1a66c7b..7821d2aad66 100755 --- a/dev/ci/scripts/driver.sh +++ b/dev/ci/scripts/driver.sh @@ -266,7 +266,7 @@ for pr in ${pr_list}; do "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" if [[ -f "${HOMEgfs}/sorc/logs/error.logs" ]]; then - gist_URL=$("${ROOT_DIR}/dev/ci/scripts/utils/ci_utils_wrapper.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs") + gist_URL=$("${ROOT_DIR}/dev/ci/scripts/utils/ci_utils.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs") { echo -e "\nError logs from build" echo "Gist URL: ${gist_URL}" diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 5bc6da43368..0841dd29c12 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -1,5 +1,13 @@ #!/bin/env bash +# Determine HOMEgfs and source machine detection early +if [[ -z "${HOMEgfs}" ]]; then + HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd)" +fi +source "${HOMEgfs}/ush/detect_machine.sh" + +# --- Existing functions --- + function determine_scheduler() { if command -v sbatch &> /dev/null; then echo "slurm"; @@ -198,3 +206,21 @@ function build_compute () { fi } + +# --- Dispatch logic --- + +# Check if the script is being executed directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + # Script is being executed directly + utility_function="${1}" + shift # Remove the function name from the arguments list + + # Check if the first argument corresponds to a defined function + if [[ $(type -t "$utility_function") == "function" ]]; then + # Call the function with the remaining arguments + "$utility_function" "$@" + else + echo "ERROR: Utility function '$utility_function' not found or not a function in ${BASH_SOURCE[0]}" >&2 + exit 1 + fi +fi diff --git a/dev/ci/scripts/utils/ci_utils_wrapper.sh b/dev/ci/scripts/utils/ci_utils_wrapper.sh deleted file mode 100755 index 28e76459704..00000000000 --- a/dev/ci/scripts/utils/ci_utils_wrapper.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" -source "${HOMEgfs}/ush/detect_machine.sh" - -utility_function="${1}" - -source "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh" -${utility_function} "${@:2}" diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 131bad0409b..b0759e3d45e 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -33,7 +33,7 @@ variables: allow_failure: true script: - echo "Using build directory ${HOMEGFS}" - - dev/ci/scripts/utils/ci_utils_wrapper.sh build_compute + - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} @@ -43,7 +43,7 @@ variables: allow_failure: true script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} @@ -53,7 +53,7 @@ variables: allow_failure: true script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} From 9a1006e5b546962d2d3e3761e9bc41737b53b1e5 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 14 Apr 2025 19:39:34 -0400 Subject: [PATCH 09/76] replaced _here/_top crap with find_homegfs() --- dev/ci/.gitlab-ci.yml | 30 ++++---- .../scripts/tests/test_create_experiment.py | 11 +-- dev/ci/scripts/tests/test_setup.py | 8 +- dev/ci/scripts/utils/find_homegfs.py | 73 +++++++++++++++++++ dev/ci/scripts/utils/get_host_case_list.py | 6 +- .../utils/gitlab/generate_pipelines.py | 8 +- dev/ci/scripts/utils/parse_yaml.py | 9 +-- 7 files changed, 109 insertions(+), 36 deletions(-) create mode 100644 dev/ci/scripts/utils/find_homegfs.py diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 0072cacae05..690fd6fc5f8 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -54,33 +54,33 @@ variables: # The following sections are generated for multiple hosts and dynamic case lists -build-gaeac6: +build-gaeac: extends: .build_template variables: - MACHINE: gaeac6 - tags: ["gaeac6"] + MACHINE: gaeac + tags: ["gaeac"] -setup_experiments-gaeac6: +setup_experiments-gaeac: extends: .setup_template variables: - MACHINE: gaeac6 - tags: ["gaeac6"] + MACHINE: gaeac + tags: ["gaeac"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C96C48_ufs_hybatmDA", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C48_S2SW_extended", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar_extended", "C48mx500_3DVarAOWCDA"] dependencies: - - build-gaeac6 + - build-gaeac -run_tests-gaeac6: +run_tests-gaeac: extends: .run_tests_template variables: - MACHINE: gaeac6 - tags: ["gaeac6"] + MACHINE: gaeac + tags: ["gaeac"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C96C48_ufs_hybatmDA", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C48_S2SW_extended", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar_extended", "C48mx500_3DVarAOWCDA"] dependencies: - - setup_experiments-gaeac6 + - setup_experiments-gaeac build-hera: extends: .build_template @@ -95,7 +95,7 @@ setup_experiments-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C48mx500_3DVarAOWCDA"] dependencies: - build-hera @@ -106,7 +106,7 @@ run_tests-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C48mx500_3DVarAOWCDA"] dependencies: - setup_experiments-hera diff --git a/dev/ci/scripts/tests/test_create_experiment.py b/dev/ci/scripts/tests/test_create_experiment.py index 6f1e1390f29..4a124be1fcf 100644 --- a/dev/ci/scripts/tests/test_create_experiment.py +++ b/dev/ci/scripts/tests/test_create_experiment.py @@ -2,12 +2,13 @@ from shutil import rmtree from wxflow import Executable +from find_homegfs import find_homegfs -_here = os.path.dirname(__file__) - -HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) -RUNDIR_FAKE = os.path.join(_here, 'testdata/RUNDIR') -ICSDIR_FAKE = os.path.join(_here, 'testdata/ICSDIR') +HOMEgfs = find_homegfs() +# Get the current directory using the path of this file instead of _here +current_dir = os.path.dirname(os.path.abspath(__file__)) +RUNDIR_FAKE = os.path.join(current_dir, 'testdata/RUNDIR') +ICSDIR_FAKE = os.path.join(current_dir, 'testdata/ICSDIR') def test_create_experiment(): diff --git a/dev/ci/scripts/tests/test_setup.py b/dev/ci/scripts/tests/test_setup.py index 38f14e11459..26ff4430225 100755 --- a/dev/ci/scripts/tests/test_setup.py +++ b/dev/ci/scripts/tests/test_setup.py @@ -3,10 +3,12 @@ from shutil import rmtree from wxflow import Executable, Configuration, ProcessError +from find_homegfs import find_homegfs -_here = os.path.dirname(__file__) -HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) -RUNDIR = os.path.join(_here, 'testdata/RUNDIR') +HOMEgfs = find_homegfs() +# Get the current directory using the path of this file instead of _here +current_dir = os.path.dirname(os.path.abspath(__file__)) +RUNDIR = os.path.join(current_dir, 'testdata/RUNDIR') pslot = "C48_ATM" account = "fv3-cpu" foobar = "foobar" diff --git a/dev/ci/scripts/utils/find_homegfs.py b/dev/ci/scripts/utils/find_homegfs.py new file mode 100644 index 00000000000..ee9d2dc5c75 --- /dev/null +++ b/dev/ci/scripts/utils/find_homegfs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Utility module to find the HOMEgfs (repository root) directory by traversing up +the file system until finding a directory that contains the .github subdirectory. +""" + +import os +from pathlib import Path + + +def find_homegfs(start_path=None): + """ + Find the HOMEgfs directory by traversing up the file system until + finding a directory that contains the .github subdirectory. + + Parameters + ---------- + start_path : str or Path, optional + The path to start searching from. If not provided, the current + directory will be used. + + Returns + ------- + Path + The full path to the HOMEgfs directory. + + Raises + ------ + ValueError + If the HOMEgfs directory cannot be found. + """ + # If start_path is not provided, use the directory of the calling script + if start_path is None: + # Get the path of the calling script + import inspect + frame = inspect.currentframe().f_back + start_path = os.path.dirname(os.path.abspath(frame.f_code.co_filename)) + + # Convert to Path object if it's a string + if isinstance(start_path, str): + start_path = Path(start_path) + else: + start_path = Path(start_path).resolve() + + # Start traversing up from the current directory + current_dir = start_path + + # Traverse up until we find .github directory or reach the filesystem root + while True: + # Check if .github exists in the current directory + if (current_dir / '.github').is_dir(): + return current_dir + + # Go up one level + parent_dir = current_dir.parent + + # If we've reached the root directory and haven't found .github + if parent_dir == current_dir: + raise ValueError( + "Could not find HOMEgfs directory. " + "Traversed up to the root without finding a .github directory." + ) + + current_dir = parent_dir + + +if __name__ == '__main__': + # Example usage when run as a script + try: + homegfs = find_homegfs() + print(f"Found HOMEgfs at: {homegfs}") + except ValueError as e: + print(f"Error: {e}") diff --git a/dev/ci/scripts/utils/get_host_case_list.py b/dev/ci/scripts/utils/get_host_case_list.py index 78e1d6148cf..22b423331c5 100755 --- a/dev/ci/scripts/utils/get_host_case_list.py +++ b/dev/ci/scripts/utils/get_host_case_list.py @@ -5,9 +5,7 @@ import glob from wxflow import parse_j2yaml from wxflow import AttrDict - -_here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) +from find_homegfs import find_homegfs def get_host_cases(host, homegfs=None): @@ -21,7 +19,7 @@ def get_host_cases(host, homegfs=None): Returns: list: List of case names (without extension) supported on the host """ - homegfs = homegfs or _top + homegfs = homegfs or find_homegfs() case_list = [] # Set up data for template rendering diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index 986f085c89a..c911f116b2f 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -16,11 +16,11 @@ # update sys.path to include libs in the parent directory sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from get_host_case_list import get_host_cases +from find_homegfs import find_homegfs # Get the path to the top directory of the repository # to place the generated .gitlab-ci.yml file in the default location -_here = os.path.dirname(os.path.abspath(__file__)) -_top = os.path.abspath(os.path.join(_here, '../../../../..')) +_homegfs = find_homegfs(os.path.dirname(os.path.abspath(__file__))) def get_case_list_for_machine(machine): @@ -38,7 +38,7 @@ def get_case_list_for_machine(machine): A list of test case names supported on the specified machine. """ - cases = get_host_cases(machine, homegfs=_top) + cases = get_host_cases(machine, homegfs=_homegfs) return cases @@ -146,7 +146,7 @@ def generate_pipeline_config(machines, template_file, output_file=None): If the template file does not exist. """ # Set default output file path if not specified - output_file = output_file or os.path.join(_top, 'dev/ci', '.gitlab-ci.yml') + output_file = output_file or os.path.join(_homegfs, 'dev/ci', '.gitlab-ci.yml') # Initialize with the template content if os.path.exists(template_file): diff --git a/dev/ci/scripts/utils/parse_yaml.py b/dev/ci/scripts/utils/parse_yaml.py index c22ad69983e..fdf6e60c8ae 100755 --- a/dev/ci/scripts/utils/parse_yaml.py +++ b/dev/ci/scripts/utils/parse_yaml.py @@ -9,9 +9,7 @@ from wxflow import AttrDict, parse_j2yaml from argparse import ArgumentParser from pathlib import Path - -_here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) +from find_homegfs import find_homegfs description = """parse yaml file and return value of key""" @@ -43,8 +41,9 @@ def yq(yamlfile, key): The value of the specified key in the yaml file. """ - data = AttrDict(HOMEgfs=_top) - data.update({'HOMEgfs': _top}) + HOMEgfs = find_homegfs() + data = AttrDict(HOMEgfs=HOMEgfs) + data.update({'HOMEgfs': HOMEgfs}) ydict = parse_j2yaml(path=yamlfile, data=data) if key == 'all': return ydict From 1e59d5f8361cbdc3422723f5c394e89f58a686a8 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 09:52:52 -0400 Subject: [PATCH 10/76] added detailed inline documentation for the intent and usage of the generate_pipeline python script --- .../utils/gitlab/generate_pipelines.py | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index c911f116b2f..d1242494409 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -3,9 +3,52 @@ Script to generate GitLab CI pipeline configuration based on supported test cases for each machine. -This script uses get_host_case_list.py to determine which test cases are supported -on each machine and generates the complete .gitlab-ci.yml configuration by combining -a template with machine-specific sections. +Overview +-------- +This script automates the creation of a `.gitlab-ci.yml` file tailored for the +current repository and its supported test environments. Its main role is to +dynamically generate the GitLab CI pipeline configuration, ensuring that the +pipeline reflects the actual set of supported test cases for each target machine. + +Process +------- +1. **Discovery of Supported Test Cases**: + - The script imports and uses `get_host_case_list.py` (via `get_host_cases`) to + determine which test cases are supported on each specified machine. + - The list of machines is provided via command-line arguments. + +2. **Template-Based Configuration**: + - A template YAML file is provided (via `--template` argument) that contains + the static, reusable parts of the pipeline configuration. + - The script reads this template, extracting the relevant content below a + demarcation marker. + +3. **Dynamic Section Generation**: + - For each machine, the script generates YAML configuration sections for + build, setup, and test jobs, using the discovered test cases as a matrix. + - These sections are appended to the template content. + +4. **Output**: + - The final, combined configuration is written to `.gitlab-ci.yml` in the + repository's CI directory (or to a user-specified output path). + - This output file is what GitLab CI will use to define and run the pipeline. + +Role in Workflow +---------------- +- This script is intended to be run whenever the set of supported test cases or + machines changes, or when the pipeline template is updated. +- It ensures that the CI pipeline is always in sync with the actual capabilities + of the codebase and test infrastructure, reducing manual maintenance and errors. +- The generated `.gitlab-ci.yml` is the authoritative pipeline definition for + GitLab CI/CD in this repository. + +Usage +----- +Run this script with the required arguments: + python generate_pipelines.py --machines --template [--output ] + +This will produce or update the `.gitlab-ci.yml` file for use by GitLab CI. + """ import os From 6bbd386b8b9f588ec560c1905f6abdc859d45d6f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 13:16:51 -0400 Subject: [PATCH 11/76] added a finilize stage to the common pipeline, using this for pushing out a public svg image to reflect the status of the scheduled runs as badges on GitHub --- .../utils/gitlab/gitlab_pipeline_template.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index b0759e3d45e..6bf5aa69874 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -10,6 +10,7 @@ stages: - build - create_experiments - run_tests + - finalize variables: BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' @@ -57,3 +58,23 @@ variables: - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} + +.create_gist_from_svg_url: + image: ghcr.io/cli/cli:latest + stage: finalize + allow_failure: true + when: always + script: + - | + # Set your SVG URL, output filename, and Gist ID + SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/develop/pipeline.svg" + SVG_FILE="badge.svg" + GIST_ID="ab937691224bdf50427cbeca666bf67b" + # Download the SVG (add authentication if needed) + curl -L "$SVG_URL" -o "$SVG_FILE" + # Authenticate GitHub CLI (requires GITHUB_TOKEN env variable) + echo "$GITHUB_TOKEN" | gh auth login --with-token + # Update the existing public Gist with the SVG file + gh gist edit "$GIST_ID" "$SVG_FILE" + variables: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5b9b85bd35772a83acd9fe1ac6b93bcbf9699bce Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 13:22:59 -0400 Subject: [PATCH 12/76] pushing out generated Gitlab Pipeline --- dev/ci/.gitlab-ci.yml | 51 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 690fd6fc5f8..ab5277a583c 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - build - create_experiments - run_tests + - finalize variables: BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' @@ -52,35 +53,27 @@ variables: dependencies: - setup_experiments-${machine} -# The following sections are generated for multiple hosts and dynamic case lists - -build-gaeac: - extends: .build_template +.create_gist_from_svg_url: + image: ghcr.io/cli/cli:latest + stage: finalize + allow_failure: true + when: always + script: + - | + # Set your SVG URL, output filename, and Gist ID + SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/ci_gitlab_multihost_dev/pipeline.svg" + SVG_FILE="badge.svg" + GIST_ID="ab937691224bdf50427cbeca666bf67b" + # Download the SVG (add authentication if needed) + curl -L "$SVG_URL" -o "$SVG_FILE" + # Authenticate GitHub CLI (requires GITHUB_TOKEN env variable) + echo "$GITHUB_TOKEN" | gh auth login --with-token + # Update the existing public Gist with the SVG file + gh gist edit "$GIST_ID" "$SVG_FILE" variables: - MACHINE: gaeac - tags: ["gaeac"] + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -setup_experiments-gaeac: - extends: .setup_template - variables: - MACHINE: gaeac - tags: ["gaeac"] - parallel: - matrix: - - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C96C48_ufs_hybatmDA", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C48_S2SW_extended", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar_extended", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-gaeac - -run_tests-gaeac: - extends: .run_tests_template - variables: - MACHINE: gaeac - tags: ["gaeac"] - parallel: - matrix: - - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C96C48_ufs_hybatmDA", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C48_S2SW_extended", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar_extended", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-gaeac +# The following sections are generated for multiple hosts and dynamic case lists build-hera: extends: .build_template @@ -95,7 +88,7 @@ setup_experiments-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM"] dependencies: - build-hera @@ -106,7 +99,7 @@ run_tests-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48_S2SW", "C48_ATM", "C48_S2SWA_gefs", "C96_atm3DVar", "C48mx500_hybAOWCDA", "C96mx100_S2S", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_S2SW", "C48_ATM"] dependencies: - setup_experiments-hera From 658352bcb17ac14979a946616ba6bd3336c7540a Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 13:48:21 -0400 Subject: [PATCH 13/76] using find_homegfs in GitLab runner launch script --- dev/ci/scripts/utils/find_homegfs.py | 2 +- dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 dev/ci/scripts/utils/find_homegfs.py diff --git a/dev/ci/scripts/utils/find_homegfs.py b/dev/ci/scripts/utils/find_homegfs.py old mode 100644 new mode 100755 index ee9d2dc5c75..608837c55c4 --- a/dev/ci/scripts/utils/find_homegfs.py +++ b/dev/ci/scripts/utils/find_homegfs.py @@ -68,6 +68,6 @@ def find_homegfs(start_path=None): # Example usage when run as a script try: homegfs = find_homegfs() - print(f"Found HOMEgfs at: {homegfs}") + print(homegfs) except ValueError as e: print(f"Error: {e}") diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index 732c3d26f6f..b464894d7f0 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -14,10 +14,13 @@ set -e ######################################################################### # Set the HOMEGFS_ variable to the root directory of the global workflow -HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(../find_homegfs.py)" # Get the hostname of the current machine host="$(hostname)" +echo $host $HOMEGFS_ +exit + ######################################################################### # Set up runtime environment variables for accounts on supported machines ######################################################################### From 0d321641b2ab8fb1e3f3eb5e015f07623197d2a7 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 14:03:22 -0400 Subject: [PATCH 14/76] push test run in dev --- dev/ci/.gitlab-ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index ab5277a583c..59d47ba8d48 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -75,32 +75,32 @@ variables: # The following sections are generated for multiple hosts and dynamic case lists -build-hera: +build-EMC: extends: .build_template variables: - MACHINE: hera - tags: ["hera"] + MACHINE: EMC + tags: ["EMC"] -setup_experiments-hera: +setup_experiments-EMC: extends: .setup_template variables: - MACHINE: hera - tags: ["hera"] + MACHINE: EMC + tags: ["EMC"] parallel: matrix: - caseName: ["C48_S2SW", "C48_ATM"] dependencies: - - build-hera + - build-EMC -run_tests-hera: +run_tests-EMC: extends: .run_tests_template variables: - MACHINE: hera - tags: ["hera"] + MACHINE: EMC + tags: ["EMC"] parallel: matrix: - caseName: ["C48_S2SW", "C48_ATM"] dependencies: - - setup_experiments-hera + - setup_experiments-EMC # End of generated sections From 334243a96f142c4de767a180703be5fef902c1fd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 14:33:50 -0400 Subject: [PATCH 15/76] needed fix with MACHINE not being used for the matrix loop control variable --- dev/ci/scripts/utils/gitlab/generate_pipelines.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index d1242494409..1993c6fbe36 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -109,13 +109,13 @@ def generate_machine_config(machine, case_list): build-{machine}: extends: .build_template variables: - MACHINE: {machine} + mahine: {machine} tags: ["{machine}"] setup_experiments-{machine}: extends: .setup_template variables: - MACHINE: {machine} + mahine: {machine} tags: ["{machine}"] parallel: matrix: @@ -126,7 +126,7 @@ def generate_machine_config(machine, case_list): run_tests-{machine}: extends: .run_tests_template variables: - MACHINE: {machine} + mahine: {machine} tags: ["{machine}"] parallel: matrix: From 774e18b01dac672ecc6647f4771e39f2d494cb01 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 14:38:10 -0400 Subject: [PATCH 16/76] forgot to update the pipeline witht the lcv update --- dev/ci/.gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 59d47ba8d48..ba4fa90805b 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -61,7 +61,7 @@ variables: script: - | # Set your SVG URL, output filename, and Gist ID - SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/ci_gitlab_multihost_dev/pipeline.svg" + SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/develop/pipeline.svg" SVG_FILE="badge.svg" GIST_ID="ab937691224bdf50427cbeca666bf67b" # Download the SVG (add authentication if needed) @@ -78,28 +78,28 @@ variables: build-EMC: extends: .build_template variables: - MACHINE: EMC + mahine: EMC tags: ["EMC"] setup_experiments-EMC: extends: .setup_template variables: - MACHINE: EMC + mahine: EMC tags: ["EMC"] parallel: matrix: - - caseName: ["C48_S2SW", "C48_ATM"] + - caseName: ["C48_ATM"] dependencies: - build-EMC run_tests-EMC: extends: .run_tests_template variables: - MACHINE: EMC + mahine: EMC tags: ["EMC"] parallel: matrix: - - caseName: ["C48_S2SW", "C48_ATM"] + - caseName: ["C48_ATM"] dependencies: - setup_experiments-EMC From f9b8a208285712996fd8b73539db7285f68ce9bb Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 15:26:04 -0400 Subject: [PATCH 17/76] missed a new dev in the path to yaml input case file in the pipeline for create_experment --- dev/ci/.gitlab-ci.yml | 2 +- dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index ba4fa90805b..f5f361bb78d 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -38,7 +38,7 @@ variables: allow_failure: true script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 6bf5aa69874..54f23aade48 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -44,7 +44,7 @@ variables: allow_failure: true script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml dependencies: - build-${machine} From 11f085d5863a36ef40f2ef5bdbf337180ffb05d8 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 16:46:37 -0400 Subject: [PATCH 18/76] removed old ci BASH script and renamed files to elimnate - and renamed tests to unittests for clearity --- .github/workflows/ci_unit_tests.yaml | 4 +- .github/workflows/pw_aws_ci.yaml | 2 +- dev/ci/.gitlab-ci.yml | 2 +- dev/ci/Jenkinsfile | 2 +- dev/ci/Jenkinsfile4AWS | 2 +- dev/ci/scripts/check_ci.sh | 180 ----------- dev/ci/scripts/clone-build_ci.sh | 110 ------- dev/ci/scripts/driver.sh | 285 ------------------ .../{run-check_ci.sh => run_check_ci.sh} | 0 dev/ci/scripts/run_ci.sh | 89 ------ .../test_create_experiment.py | 0 .../{tests => unittests}/test_rocotostat.py | 0 .../{tests => unittests}/test_setup.py | 0 dev/ci/scripts/{tests => unittests}/wxflow | 0 .../utils/gitlab/gitlab_pipeline_template.yml | 2 +- .../parallel_works/UserBootstrap_centos7.txt | 5 - .../utils/parallel_works/provision_runner.sh | 39 --- dev/ci/scripts/utils/pr_list_database.py | 216 ------------- 18 files changed, 7 insertions(+), 931 deletions(-) delete mode 100755 dev/ci/scripts/check_ci.sh delete mode 100755 dev/ci/scripts/clone-build_ci.sh delete mode 100755 dev/ci/scripts/driver.sh rename dev/ci/scripts/{run-check_ci.sh => run_check_ci.sh} (100%) delete mode 100755 dev/ci/scripts/run_ci.sh rename dev/ci/scripts/{tests => unittests}/test_create_experiment.py (100%) rename dev/ci/scripts/{tests => unittests}/test_rocotostat.py (100%) rename dev/ci/scripts/{tests => unittests}/test_setup.py (100%) rename dev/ci/scripts/{tests => unittests}/wxflow (100%) delete mode 100644 dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt delete mode 100755 dev/ci/scripts/utils/parallel_works/provision_runner.sh delete mode 100755 dev/ci/scripts/utils/pr_list_database.py diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index 450d2546b8e..83fbb2c9a40 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -60,7 +60,7 @@ jobs: cd global-workflow/sorc git submodule update --init -j 2 wxflow ufs_model.fd ./link_workflow.sh - cd ../dev/ci/scripts/tests + cd ../dev/ci/scripts/unittests pytest -v --junitxml test-results.xml @@ -68,6 +68,6 @@ jobs: if: always() uses: EnricoMi/publish-unit-test-result-action@v2 with: - files: global-workflow/dev/ci/scripts/tests/test-results.xml + files: global-workflow/dev/ci/scripts/unittests/test-results.xml job_summary: true comment_mode: off diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 490e52cb0fa..e7c05f3e296 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -157,7 +157,7 @@ jobs: - name: Run Experiment ${{ matrix.case }} run: | cd ${{ env.TEST_DIR }}/HOMEgfs - ./dev/ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs + ./dev/ci/scripts/run_check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs clean-up: needs: run-experiments diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index f5f361bb78d..4aed81d3fe5 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -49,7 +49,7 @@ variables: script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index d65c3dc9704..b4208ffb578 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -221,7 +221,7 @@ pipeline { try { sh(script: """ source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + ${HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ source ${HOMEgfs}/dev/workflow/gw_setup.sh diff --git a/dev/ci/Jenkinsfile4AWS b/dev/ci/Jenkinsfile4AWS index b77d7cbb945..91220e83ef6 100644 --- a/dev/ci/Jenkinsfile4AWS +++ b/dev/ci/Jenkinsfile4AWS @@ -225,7 +225,7 @@ pipeline { try { sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + ${HOMEgfs}/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ source ${HOMEgfs}/workflow/gw_setup.sh diff --git a/dev/ci/scripts/check_ci.sh b/dev/ci/scripts/check_ci.sh deleted file mode 100755 index 0dd7a2807ba..00000000000 --- a/dev/ci/scripts/check_ci.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash -set -eux -##################################################################################### -# -# Script description: BASH script for checking for cases in a given PR and -# running rocotostat on each to determine if the experiment has -# succeeded or faild. This script is intended -# to run from within a cron job in the CI Managers account -##################################################################################### - -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" -scriptname=$(basename "${BASH_SOURCE[0]}") -echo "Begin ${scriptname} at $(date -u)" || true -export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' - -REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} - -######################################################################### -# Set up runtime environment varibles for accounts on supproted machines -######################################################################### - -source "${HOMEgfs}/ush/detect_machine.sh" -case ${MACHINE_ID} in - hera | orion | hercules | wcoss2 | gaea) - echo "Running Automated Testing on ${MACHINE_ID}" - source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac -set +x -export HOMEgfs -source "${HOMEgfs}/ush/module-setup.sh" -source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" -module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" -module list -# Load machine specific modules for ci (only wcoss2 is current) -if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - module load "module_gwci.${MACHINE_ID}" -fi -set -x -if ! command -v gh > /dev/null; then - GH="${HOME}/bin/gh" -else - GH=$(command -v gh) -fi -export GH - -rocotostat=$(command -v rocotostat) -if [[ -z ${rocotostat} ]]; then - echo "rocotostat not found on system" - exit 1 -else - echo "rocotostat being used from ${rocotostat}" -fi -rocotocheck=$(command -v rocotocheck) -if [[ -z ${rocotocheck} ]]; then - echo "rocotocheck not found on system" - exit 1 -else - echo "rocotocheck being used from ${rocotocheck}" -fi - -pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" - -pr_list="" -if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true -fi -if [[ -z "${pr_list}" ]]; then - echo "no PRs open and ready to run cases on .. exiting" - exit 0 -fi - -############################################################# -# Loop throu all PRs in PR List and look for expirments in -# the RUNTESTS dir and for each one run runcotorun on them -############################################################# - -for pr in ${pr_list}; do - id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') - output_ci="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_${id}" - output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_runtime_single.log" - echo "Processing Pull Request #${pr} and looking for cases" - pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" - - # If there is no RUNTESTS dir for this PR then cases have not been made yet - if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then - continue - fi - - #Check for PR success when ${pr_dir}/RUNTESTS/EXPDIR is void of subfolders - # since all successfull ones where previously removed - # shellcheck disable=SC2312 - if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed" - sed -i "1 i\`\`\`" "${output_ci}" - sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms - weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}" --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true - if [[ -n "${weekly_labels}" ]]; then - num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l) - passed=0 - for platforms in "${HOMEgfs}"/ci/platforms/config.*; do - machine=$(basename "${platforms}" | cut -d. -f2) - if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then - ((passed=passed+1)) - fi - done - if [[ "${passed}" == "${num_platforms}" ]]; then - "${GH}" pr close --repo "${REPO_URL}" "${pr}" - fi - fi - # Completely remove the PR and its cloned repo on sucess - # of all cases on this platform - rm -Rf "${pr_dir}" - continue - fi - - for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do - pslot=$(basename "${pslot_dir}") || true - if [[ -z "${pslot}" ]]; then - echo "No experiments found in ${pslot_dir} .. exiting" - exit 0 - fi - xml="${pslot_dir}/${pslot}.xml" - db="${pslot_dir}/${pslot}.db" - if [[ ! -f "${db}" ]]; then - continue - fi - - set +e - rocoto_state="$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}")" - rocoto_error=$? - rm -f "${output_ci_single}" - if [[ "${rocoto_error}" -ne 0 ]]; then - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed" - if [[ "${rocoto_state}" == "STALLED" ]]; then - # shellcheck disable=SC2312 - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" - "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - cancel_all_batch_jobs "${pr_dir}/RUNTESTS" - exit "${rocoto_error}" - fi - error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true - # shellcheck disable=SC2086 - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null - # shellcheck disable=SC2086 - gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")" - { - echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" || true - echo "" - echo "Error logs:" - echo "\`\`\`" - echo "${error_logs}" - echo "\`\`\`" - echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})" - } >> "${output_ci_single}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" - "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - cancel_all_batch_jobs "${pr_dir}/RUNTESTS" - exit "${rocoto_error}" - fi - if [[ "${rocoto_state}" == "DONE" ]]; then - #Remove Experment cases that completed successfully - "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" cleanup_experiment "${pslot_dir}" - rm -f "${output_ci_single}" - # echo "\`\`\`" > "${output_ci_single}" - DATE=$(date +'%D %r') - echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}" - echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}" - # "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" - fi - done -done diff --git a/dev/ci/scripts/clone-build_ci.sh b/dev/ci/scripts/clone-build_ci.sh deleted file mode 100755 index 3cef7fc230b..00000000000 --- a/dev/ci/scripts/clone-build_ci.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -set -eux - -##################################################################### -# Usage and arguments for specfifying cloned directgory -##################################################################### -usage() { - set +x - echo - echo "Usage: $0 -p -d -o -h" - echo - echo " -p PR number to clone and build" - echo " -d Full path of of where to clone and build PR" - echo " -o Full path to output message file detailing results of CI tests" - echo " -h display this message and quit" - echo - exit 1 -} - -################################################################ -while getopts "p:d:o:h" opt; do - case ${opt} in - p) - PR=${OPTARG} - ;; - d) - repodir=${OPTARG} - ;; - o) - outfile=${OPTARG} - ;; - h|\?|:) - usage - ;; - *) - echo "Unrecognized option" - usage - ;; - esac -done - -cd "${repodir}" || exit 1 -if [[ -d global-workflow ]]; then - rm -Rf global-workflow -fi - -git clone "${REPO_URL}" -cd global-workflow || exit 1 - -# checkout pull request -"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules -HOMEgfs="${PWD}" -source "${HOMEgfs}/ush/detect_machine.sh" - -#################################################################### -# start output file -{ - echo "Automated global-workflow Testing Results:" - echo '```' - echo "Machine: ${MACHINE_ID^}" - echo "Start: $(date) on $(hostname)" || true - echo "---------------------------------------------------" -} >> "${outfile}" -###################################################################### - -# get commit hash -commit=$(git log --pretty=format:'%h' -n 1) -echo "${commit}" > "../commit" - -# build full cycle -cd sorc || exit 1 -set +e - -source "${HOMEgfs}/ush/module-setup.sh" -export BUILD_JOBS=8 -rm -rf log.build -./build_all.sh -guk >> log.build 2>&1 -build_status=$? - -DATE=$(date +'%D %r') -if [[ ${build_status} != 0 ]]; then - { - echo "Build: *** FAILED ***" - echo "Build: Failed at ${DATE}" - cat "${PWD}/log.build" - cat "${PWD}/logs/error.logs" - } >> "${outfile}" - exit "${build_status}" -else - { - echo "Build: Completed at ${DATE}" - } >> "${outfile}" -fi - -LINK_LOGFILE_PATH=link_workflow.log -rm -f "${LINK_LOGFILE_PATH}" -./link_workflow.sh >> "${LINK_LOGFILE_PATH}" 2>&1 -link_status=$? -if [[ ${link_status} != 0 ]]; then - DATE=$(date +'%D %r') - { - echo "Link: *** FAILED ***" - echo "Link: Failed at ${DATE}" - cat "${LINK_LOGFILE_PATH}" - } >> "${outfile}" - exit "${link_status}" -fi - -echo "check/build/link test completed" -exit "${build_status}" diff --git a/dev/ci/scripts/driver.sh b/dev/ci/scripts/driver.sh deleted file mode 100755 index 7821d2aad66..00000000000 --- a/dev/ci/scripts/driver.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash -set -eux - -##################################################################################### -# -# Script description: Top level driver script for checking PR -# ready for CI regression testing -# -# Abstract: -# -# This script uses GitHub CLI to check for Pull Requests with CI-Ready-${machine} tags on the -# development branch for the global-workflow repo. It then stages tests directories per -# PR number and calls clone-build_ci.sh to perform a clone and full build from the PR. -# It then is ready to run a suite of regression tests with various configurations -####################################################################################### - -export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} - -################################################################ -# Setup the reletive paths to scripts and PS4 for better logging -################################################################ -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" -scriptname=$(basename "${BASH_SOURCE[0]}") -echo "Begin ${scriptname} at $(date +'%D %r')" || true -export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' - -######################################################################### -# Set up runtime environment varibles for accounts on supproted machines -######################################################################### - -source "${ROOT_DIR}/ush/detect_machine.sh" -case ${MACHINE_ID} in - hera | orion | hercules | wcoss2 | gaea) - echo "Running Automated Testing on ${MACHINE_ID}" - source "${ROOT_DIR}/dev/ci/platforms/config.${MACHINE_ID}" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac - -###################################################### -# setup runtime env for correct python install and git -###################################################### -HOMEgfs=${ROOT_DIR} -export HOMEgfs -set +x -source "${ROOT_DIR}/dev/ci/scripts/utils/ci_utils.sh" -source "${ROOT_DIR}/ush/module-setup.sh" -module use "${ROOT_DIR}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" -# Load machine specific modules for ci (only wcoss2 is current) -if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - module load "module_gwci.${MACHINE_ID}" -fi -set -x -unset HOMEgfs -if ! command -v gh > /dev/null; then - GH="${HOME}/bin/gh" -else - GH=$(command -v gh) -fi -export GH - -############################################################ -# query repo and get list of open PRs with tags {machine}-CI -############################################################ - -pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" - -if [[ ! -f "${pr_list_dbfile}" ]]; then - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}" -fi - -pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true - -for pr in ${pr_list}; do - pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" - [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}" - db_list=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}") - output_ci_single="${pr_dir}/output_single.log" - ############################################################# - # Check if a Ready labeled PR has changed back from once set - # and in that case completely kill the previose driver.sh cron - # job and all its decedands as well as removing all previous - # jobs in scheduler and associated files in the PR - ############################################################# - if [[ "${db_list}" == *"already is in list"* ]]; then - # Get the the PID and HOST of the driver.sh cron job - # that is stored int he CI database for this PR - driver_ID=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display "${pr}" | awk '{print $4}') || true - driver_PID=$(echo "${driver_ID}" | cut -d":" -f1) || true - driver_HOST=$(echo "${driver_ID}" | cut -d":" -f2) || true - host_name=$(hostname -s) - rm -f "${output_ci_single}" - { - echo "CI Update on ${MACHINE_ID^} at $(date +'%D %r')" || true - echo "=================================================" - echo "PR:${pr} Reset to ${MACHINE_ID^}-Ready by user and is now restarting CI tests" || true - } >> "${output_ci_single}" - if [[ "${driver_PID}" -ne 0 ]]; then - echo "Driver PID: ${driver_PID} no longer running this build having it killed" - if [[ "${driver_HOST}" == "${host_name}" ]]; then - pstree_out="$(pstree -A -p "${driver_PID}")" - if [[ -n "${pstree_out}" ]]; then - #shellcheck disable=SC2312 - echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill - fi - else - # Check if the driver is still running on the head node; if so, kill it and all child processes - #shellcheck disable=SC2029 - ssh "${driver_HOST}" "pstree -A -p \"${driver_PID}\" | grep -Eow \"[0-9]+\" | xargs kill || echo \"Failed to kill process with PID: ${driver_PID}, it may not be valid.\"" - fi - { - echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}" - echo "Driver PID: has restarted as $$ on ${host_name}" - } >> "${output_ci_single}" - fi - - experiments=$(find "${pr_dir}/RUNTESTS/EXPDIR" -mindepth 1 -maxdepth 1 -type d) || true - if [[ -z "${experiments}" ]]; then - echo "No current experiments to cancel in PR: ${pr} on ${MACHINE_ID^}" >> "${output_ci_single}" - else - for case in ${experiments}; do - case_name=$(basename "${case}") - cancel_batch_jobs "${case_name}" - { - echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}" - } >> "${output_ci_single}" - done - fi - first_line=$(head -n 1 "${output_ci_single}") - if [[ "${first_line}" != '```' ]]; then - sed -i "1 i\`\`\`" "${output_ci_single}" - fi - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}" - fi -done - -pr_list="" -if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true -fi -if [[ -z "${pr_list}" ]]; then - echo "no PRs open and ready for checkout/build .. exiting" - exit 0 -fi - - -############################################################# -# Loop throu all open PRs -# Clone, checkout, build, creat set of cases, for each -############################################################# - -for pr in ${pr_list}; do - # Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron - pr_building=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true - if [[ -n "${pr_building}" ]]; then - continue - fi - id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') - pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" - output_ci="${pr_dir}/output_ci_${id}" - output_ci_single="${GFS_BASH_CI_ROOT}/PR/${pr}/output_single.log" - driver_build_PID=$$ - driver_build_HOST=$(hostname -s) - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building "${driver_build_PID}:${driver_build_HOST}" - rm -Rf "${pr_dir}" - mkdir -p "${pr_dir}" - { - echo "CI Update on ${MACHINE_ID^} at $(date +'%D %r')" || true - echo "============================================" - echo "Cloning and Building global-workflow PR: ${pr}" - echo "with PID: ${driver_build_PID} on host: ${driver_build_HOST}" - echo "" - } >> "${output_ci_single}" - first_line=$(head -n 1 "${output_ci_single}") - if [[ "${first_line}" != '```' ]]; then - sed -i "1 i\`\`\`" "${output_ci_single}" - fi - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" - set +e - "${ROOT_DIR}/dev/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}" - ci_status=$? - ################################################################## - # Checking for special case when Ready label was updated - # but a race condtion caused the clone-build_ci.sh to start - # and this instance fails before it was killed. In th case we - # we need to exit this instance of the driver script - ################################################################# - if [[ ${ci_status} -ne 0 ]]; then - build_PID_check=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true - if [[ "${build_PID_check}" -ne "$$" ]]; then - echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting" - exit 0 - fi - fi - set -e - if [[ ${ci_status} -eq 0 ]]; then - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built "0:0" - #setup space to put an experiment - # export RUNTESTS for yaml case files to pickup - export RUNTESTS="${pr_dir}/RUNTESTS" - rm -Rf "${pr_dir:?}/RUNTESTS/"* - - ############################################################# - # loop over every yaml file in the PR's ci/cases - # and create an run directory for each one for this PR loop - ############################################################# - HOMEgfs="${pr_dir}/global-workflow" - cd "${HOMEgfs}" - pr_sha=$(git rev-parse --short HEAD) - - for yaml_config in "${HOMEgfs}/dev/ci/cases/pr/"*.yaml; do - case=$(basename "${yaml_config}" .yaml) || true - # export pslot for yaml case files to pickup - export pslot="${case}_${pr_sha}" - rm -Rf "${STMP}/RUNDIRS/${pslot}" - set +e - export LOGFILE_PATH="${HOMEgfs}/dev/ci/scripts/create_experiment.log" - rm -f "${LOGFILE_PATH}" - yaml_case_file="${HOMEgfs}/dev/ci/cases/pr/${case}.yaml" - skip_hosts=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" --yaml "${yaml_case_file}" --key skip_ci_on_hosts --string) - if [[ "${skip_hosts}" == *"${MACHINE_ID}"* ]]; then - { - echo "Case setup: Skipped for experiment ${pslot}" || true - } >> "${output_ci}" - continue - fi - "${HOMEgfs}/dev/workflow/create_experiment.py" --yaml "${HOMEgfs}/dev/ci/cases/pr/${case}.yaml" --overwrite > "${LOGFILE_PATH}" 2>&1 - ci_status=$? - set -e - if [[ ${ci_status} -eq 0 ]]; then - { - echo "Case setup: Completed for experiment ${pslot}" || true - } >> "${output_ci}" - else - { - echo "*** Failed *** to create experiment: ${pslot} on ${MACHINE_ID^}" - echo "" - cat "${LOGFILE_PATH}" - } >> "${output_ci}" - "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - exit 1 - fi - done - - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - - else # failed to clone and build - - { - echo "Failed on cloning and building global-workflowi PR: ${pr}" - echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true - } >> "${output_ci}" - - "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" - "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - - if [[ -f "${HOMEgfs}/sorc/logs/error.logs" ]]; then - gist_URL=$("${ROOT_DIR}/dev/ci/scripts/utils/ci_utils.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs") - { - echo -e "\nError logs from build" - echo "Gist URL: ${gist_URL}" - } >> "${output_ci}" - fi - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - - fi - -done # looping over each open and labeled PR - -########################################## -# scrub working directory for older files -########################################## -# -#find "${GFS_BASH_CI_ROOT}/PR/*" -maxdepth 1 -mtime +3 -exec rm -rf {} \; diff --git a/dev/ci/scripts/run-check_ci.sh b/dev/ci/scripts/run_check_ci.sh similarity index 100% rename from dev/ci/scripts/run-check_ci.sh rename to dev/ci/scripts/run_check_ci.sh diff --git a/dev/ci/scripts/run_ci.sh b/dev/ci/scripts/run_ci.sh deleted file mode 100755 index aa77a7dbf6b..00000000000 --- a/dev/ci/scripts/run_ci.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -set -eux - -##################################################################################### -# -# Script description: BASH script for checking for cases in a given PR and -# simply running rocotorun on each. This script is intended -# to run from within a cron job in the CI Managers account -# Abstract TODO -##################################################################################### - -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" -scriptname=$(basename "${BASH_SOURCE[0]}") -echo "Begin ${scriptname} at $(date -u)" || true -export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' - -######################################################################### -# Set up runtime environment varibles for accounts on supproted machines -######################################################################### - -source "${HOMEgfs}/ush/detect_machine.sh" -case ${MACHINE_ID} in - hera | orion | hercules | wcoss2 | gaea) - echo "Running Automated Testing on ${MACHINE_ID}" - source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac -set +x -export HOMEgfs -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" -module list -set -eux -rocotorun=$(command -v rocotorun) -if [[ -z ${rocotorun} ]]; then - echo "rocotorun not found on system" - exit 1 -else - echo "rocotorun being used from ${rocotorun}" -fi - -pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" - -pr_list="" -if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${HOMEgfs}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true - pr_list=$(echo "${pr_list}" | tr ' ' '\n' | head -n "${max_concurrent_pr}" | tr '\n' ' ') || true -fi -if [[ -z "${pr_list}" ]]; then - echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting" - exit 0 -fi - -############################################################# -# Loop throu all PRs in PR List and look for expirments in -# the RUNTESTS dir and for each one run runcotorun on them -# only up to $max_concurrent_cases will advance at a time -############################################################# - -for pr in ${pr_list}; do - echo "Processing Pull Request #${pr} and looking for cases" - pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" - # If the directory RUNTESTS is not present then - # setupexpt.py has no been run yet for this PR - if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then - continue - fi - num_cases=0 - for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do - if [[ ! -d "${pslot_dir}" ]]; then - continue - fi - ((num_cases=num_cases+1)) - # No more than two cases are going forward at a time for each PR - if [[ "${num_cases}" -gt "${max_concurrent_cases}" ]]; then - continue - fi - pslot=$(basename "${pslot_dir}") - xml="${pslot_dir}/${pslot}.xml" - db="${pslot_dir}/${pslot}.db" - echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}" - "${rocotorun}" -v 10 -w "${xml}" -d "${db}" - done -done diff --git a/dev/ci/scripts/tests/test_create_experiment.py b/dev/ci/scripts/unittests/test_create_experiment.py similarity index 100% rename from dev/ci/scripts/tests/test_create_experiment.py rename to dev/ci/scripts/unittests/test_create_experiment.py diff --git a/dev/ci/scripts/tests/test_rocotostat.py b/dev/ci/scripts/unittests/test_rocotostat.py similarity index 100% rename from dev/ci/scripts/tests/test_rocotostat.py rename to dev/ci/scripts/unittests/test_rocotostat.py diff --git a/dev/ci/scripts/tests/test_setup.py b/dev/ci/scripts/unittests/test_setup.py similarity index 100% rename from dev/ci/scripts/tests/test_setup.py rename to dev/ci/scripts/unittests/test_setup.py diff --git a/dev/ci/scripts/tests/wxflow b/dev/ci/scripts/unittests/wxflow similarity index 100% rename from dev/ci/scripts/tests/wxflow rename to dev/ci/scripts/unittests/wxflow diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 54f23aade48..73cce86c445 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -55,7 +55,7 @@ variables: script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} diff --git a/dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt b/dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt deleted file mode 100644 index ddc6b057060..00000000000 --- a/dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt +++ /dev/null @@ -1,5 +0,0 @@ -sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm -sudo yum -y install git -/contrib/Terry.McGuinness/SETUP/provision_runner.sh -ALLNODES -/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh \ No newline at end of file diff --git a/dev/ci/scripts/utils/parallel_works/provision_runner.sh b/dev/ci/scripts/utils/parallel_works/provision_runner.sh deleted file mode 100755 index cac18c93156..00000000000 --- a/dev/ci/scripts/utils/parallel_works/provision_runner.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# This script provisions a GitHub Actions runner on a Rocky or CentOS system. -# It performs the following steps: -# 1. Checks the operating system from /etc/os-release. -# 2. Verifies if the operating system is either Rocky or CentOS. -# 3. Checks if an actions-runner process is already running for the current user. -# 4. Copies the actions-runner tar file from a specified directory to the home directory. -# 5. Extracts the tar file and starts the actions-runner in the background. -# -# The actions-runner tar file contains the necessary binaries and scripts to run -# a GitHub Actions runner. It is specific to the operating system and is expected -# to be located in the /contrib/${CI_USER}/SETUP/ directory. - -CI_USER="Terry.McGuinness" - -# Get the Operating System name from /etc/os-release -OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true - -# Check if the OS is Rocky or CentOS -if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then - echo "Operating System is ${OS_NAME}" -else - echo "Unsupported Operating System: ${OS_NAME}" - exit 1 -fi - -running=$(pgrep -u "${USER}" run-helper -c) || true -if [[ "${running}" -gt 0 ]]; then - echo "actions-runner is already running" - exit -fi - -cp "/contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz" "${HOME}" -cd "${HOME}" || exit -tar -xf "actions-runner_${OS_NAME}.tar.gz" -cd actions-runner || exit -d=$(date +%Y-%m-%d-%H:%M) -nohup ./run.sh >& "run_nohup${d}.log" & diff --git a/dev/ci/scripts/utils/pr_list_database.py b/dev/ci/scripts/utils/pr_list_database.py deleted file mode 100755 index 3b53d211344..00000000000 --- a/dev/ci/scripts/utils/pr_list_database.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from typing import List -from wxflow import SQLiteDB, SQLiteDBError -from githubpr import GitHubPR -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER -import argparse - - -def full_path(string): - """ - full_path Get the absolute path of a file or directory. - Parameters - ---------- - string : str - The relative path of the file or directory. - - Returns - ------- - str - The absolute path of the file or directory. - - Raises - ------ - NotADirectoryError - If the provided string does not represent a valid file or directory. - """ - - if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)): - return os.path.abspath(string) - else: - raise NotADirectoryError(string) - - -def create_table(db: SQLiteDB): - """ - Create a new table in a database. - - Parameters - ---------- - db : SQLiteDB - The database to create. - """ - - db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT']) - - -def add_pr(ci_database: SQLiteDB, pr: str) -> bool: - """ - Add a pull request to the database. - - Parameters - ---------- - ci_database : SQLiteDB - The database to add the pull request to. - - pr : str - The pull request to add. - """ - - entities = (pr, 'Open', 'Ready', 0, 'ci_repo') - try: - ci_database.insert_data('pr_list', entities) - return True - except (SQLiteDBError.IntegrityError) as e: - if 'unique' in str(e).lower(): - print(f"pr {pr} already is in list: nothing added") - return False - - -def update_pr(ci_database: SQLiteDB, args: argparse.Namespace): - """ - Update a pull request in the database. - - Parameters - ---------- - ci_database : SQLiteDB - The database to update the pull request in. - - args : argparse.Namespace - The command line arguments. - """ - - if len(args.update_pr) < 2: - print(f"update_pr must have at least one vaule to update") - sys.exit(0) - - update_list = ['state', 'status', 'reset_id', 'cases'] - for value in args.update_pr[1:]: - update = update_list.pop(0) - ci_database.update_data('pr_list', update, value, 'pr', args.update_pr[0]) - - -def display_db(ci_database: SQLiteDB, display: List[str]) -> list: - """ - Display the database. - - Parameters - ---------- - ci_database : SQLiteDB - The database to display. - - display : argparse.Namespace - The command line arguments. - - Returns - ------- - list - The rows of the database. - """ - - values = [] - if len(display) == 1: - rows = ci_database.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'") - if len(display) == 2: - rows = ci_database.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'") - if len(display) == 0: - rows = ci_database.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases']) - for row in rows: - values.append(' '.join(map(str, row))) - - return values - - -def update_database(ci_database: SQLiteDB) -> list: - """ - Update the database from the GitHub PRs - - only PRs from host machine are added to the database - - if the PR is already in the database it its added to the kill list - - Parameters - ---------- - ci_database : SQLiteDB - The database to update. - - Returns - ------- - list - The kill list of pull requests. - """ - - gh = GitHubPR() - pr_ready_list, pr_kill_list = gh.get_open_pr_list() - for pr in pr_ready_list: - if not add_pr(ci_database, str(pr)): - if pr not in pr_kill_list: - pr_kill_list.append(pr) - pr_kill_list = list(set(pr_kill_list)) - return pr_kill_list - - -def input_args(): - """ - Parse command line arguments. - - Returns - ------- - argparse.Namespace - The parsed command line arguments. - """ - description = """Arguments for creating and updating db file for pr states - """ - - parser = ArgumentParser(description=description, - formatter_class=ArgumentDefaultsHelpFormatter) - - parser.add_argument('--dbfile', help='SQLite3 database file with PR list', type=full_path) - parser.add_argument('--create', help='create sqlite file for pr list status', action='store_true', required=False) - parser.add_argument('--add_pr', nargs=1, metavar='PR', help='add new pr to list (defults to: Open,Ready)', required=False) - parser.add_argument('--remove_pr', nargs=1, metavar='PR', help='removes pr from list', required=False) - parser.add_argument('--update_pr', nargs=REMAINDER, metavar=('pr', 'state', 'status', 'reset_id', 'cases'), - help='updates state and status of a given pr', required=False) - parser.add_argument('--display', nargs='*', help='output pr table', required=False) - parser.add_argument('--list', nargs=2, metavar=('state', 'status'), required=False) - parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list', - action='store_true', required=False) - args = parser.parse_args() - return args - - -if __name__ == '__main__': - - args = input_args() - - if not args.create: - if not os.path.isfile(args.dbfile): - print(f'Error: {args.dbfile} does not exsist') - sys.exit(-1) - - ci_database = SQLiteDB(args.dbfile) - ci_database.connect() - - if args.create: - create_table(ci_database) - if args.add_pr: - add_pr(ci_database, args.add_pr[0]) - if args.update_pr: - update_pr(ci_database, args) - if args.remove_pr: - ci_database.remove_data('pr_list', 'PR', args.remove_pr[0]) - if args.display is not None: - for rows in display_db(ci_database, args.display): - print(rows) - if args.list: - for rows in display_db(ci_database, [args.list[0], args.list[1]]): - print(rows, end=' ') - print() - if args.update_database: - pr_kill_list = update_database(ci_database) - for pr in pr_kill_list: - print(pr, end=' ') - print() - - ci_database.disconnect() From d35ad44bcf5f4ac7d65b3e64bf5d901036eea535 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 17:12:20 -0400 Subject: [PATCH 19/76] replaced all instances of getting HOMEgfs with find_homegfs method and renamed it to HOMEGFS_ in the ci shell scripts --- .../unittests/test_create_experiment.py | 1 - dev/ci/scripts/unittests/test_setup.py | 1 - dev/ci/scripts/utils/ci_utils.sh | 37 ++++++++++--------- .../utils/gitlab/launch_gitlab_runner.sh | 3 +- dev/ci/scripts/utils/launch_java_agent.sh | 4 +- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/dev/ci/scripts/unittests/test_create_experiment.py b/dev/ci/scripts/unittests/test_create_experiment.py index 4a124be1fcf..57795081dac 100644 --- a/dev/ci/scripts/unittests/test_create_experiment.py +++ b/dev/ci/scripts/unittests/test_create_experiment.py @@ -5,7 +5,6 @@ from find_homegfs import find_homegfs HOMEgfs = find_homegfs() -# Get the current directory using the path of this file instead of _here current_dir = os.path.dirname(os.path.abspath(__file__)) RUNDIR_FAKE = os.path.join(current_dir, 'testdata/RUNDIR') ICSDIR_FAKE = os.path.join(current_dir, 'testdata/ICSDIR') diff --git a/dev/ci/scripts/unittests/test_setup.py b/dev/ci/scripts/unittests/test_setup.py index 26ff4430225..d8155d9bfb4 100755 --- a/dev/ci/scripts/unittests/test_setup.py +++ b/dev/ci/scripts/unittests/test_setup.py @@ -6,7 +6,6 @@ from find_homegfs import find_homegfs HOMEgfs = find_homegfs() -# Get the current directory using the path of this file instead of _here current_dir = os.path.dirname(os.path.abspath(__file__)) RUNDIR = os.path.join(current_dir, 'testdata/RUNDIR') pslot = "C48_ATM" diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 0841dd29c12..8ac3fba3a77 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -1,10 +1,11 @@ #!/bin/env bash -# Determine HOMEgfs and source machine detection early -if [[ -z "${HOMEgfs}" ]]; then - HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd)" +# Determine HOMEGFS_ and source machine detection early +if [[ -z "${HOMEGFS_}" ]]; then + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + HOMEGFS_="$(${SCRIPT_DIR}/find_homegfs.py)" fi -source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEGFS_}/ush/detect_machine.sh" # --- Existing functions --- @@ -69,7 +70,7 @@ function get_pr_case_list () { # loop over every yaml file in the PR's ci/cases # and create an run directory for each one for this PR loop ############################################################# - for yaml_config in "${HOMEgfs}/dev/ci/cases/pr/"*.yaml; do + for yaml_config in "${HOMEGFS_}/dev/ci/cases/pr/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true echo "${case}" done @@ -122,29 +123,29 @@ function cancel_all_batch_jobs () { function create_experiment () { local yaml_config="${1}" - cd "${HOMEgfs}" || exit 1 + cd "${HOMEGFS_}" || exit 1 pr_sha=$(git rev-parse --short HEAD) case=$(basename "${yaml_config}" .yaml) || true export pslot=${case}_${pr_sha} if [[ ${MACHINE_ID} == "noaacloud" ]]; then - source "${HOMEgfs}/dev/ci/platforms/config.${PW_CSP}" + source "${HOMEGFS_}/dev/ci/platforms/config.${PW_CSP}" else - source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" + source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" fi - source "${HOMEgfs}/dev/ush/gw_setup.sh" + source "${HOMEGFS_}/dev/ush/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) if [[ ${MACHINE_ID} == "noaacloud" ]]; then - STMP=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/dev/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) + STMP=$("${HOMEGFS_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEGFS_}/dev/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) else - STMP=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/dev/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) + STMP=$("${HOMEGFS_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEGFS_}/dev/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) fi echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" - "${HOMEgfs}/${system}/dev/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" + "${HOMEGFS_}/${system}/dev/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" } @@ -168,8 +169,8 @@ function publish_logs() { if [[ -n "${full_paths}" ]]; then # shellcheck disable=SC2027,SC2086 - ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null - URL="$("${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" + ${HOMEGFS_}/dev/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null + URL="$("${HOMEGFS_}/dev/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" fi echo "${URL}" } @@ -186,7 +187,7 @@ function cleanup_experiment() { pslot=$(basename "${EXPDIR}") # Use the Python utility to get the required variables - read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/dev/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true + read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEGFS_}/dev/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" @@ -197,12 +198,12 @@ function cleanup_experiment() { function build_compute () { - source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" + source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" # TODO: when it's safe to build on C6 compute nodes again, do so if [[ "${MACHINE_ID}" == "gaeac6" ]]; then - "${HOMEgfs}/sorc/build_all.sh" -v -k all + "${HOMEGFS_}/sorc/build_all.sh" -v -k all else - "${HOMEgfs}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all + "${HOMEGFS_}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all fi } diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index b464894d7f0..b325e0fe406 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -14,7 +14,8 @@ set -e ######################################################################### # Set the HOMEGFS_ variable to the root directory of the global workflow -HOMEGFS_="$(../find_homegfs.py)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HOMEGFS_="$("${SCRIPT_DIR}/../find_homegfs.py")" # Get the hostname of the current machine host="$(hostname)" diff --git a/dev/ci/scripts/utils/launch_java_agent.sh b/dev/ci/scripts/utils/launch_java_agent.sh index 8e348d09b60..063d7615f60 100755 --- a/dev/ci/scripts/utils/launch_java_agent.sh +++ b/dev/ci/scripts/utils/launch_java_agent.sh @@ -65,7 +65,9 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" +# Set the HOMEGFS_ variable to the root directory of the global workflow +SCRIPT_DIR="$(cd "$(dirname \"${BASH_SOURCE[0]}\")" && pwd)" +HOMEGFS_="$(${SCRIPT_DIR}/find_homegfs.py)" host=$(hostname) ######################################################################### From 04032a9cce46999339094bf9288e0586045dfcf0 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 17:25:25 -0400 Subject: [PATCH 20/76] just noticed that Rahul had added HOMEgfs_dev so put it back --- dev/ci/Jenkinsfile | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index b4208ffb578..d696c4387db 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - sh(script: "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables + sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" if ( fileExists("logs/error.logs") ) { @@ -148,12 +148,12 @@ pipeline { } try { sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} """) gist_url=sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) } catch (Exception error_comment) { @@ -176,8 +176,8 @@ pipeline { } // Get a list of CI cases to run CI_CASES = sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/get_host_case_list.py ${machine} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/get_host_case_list.py ${machine} """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } @@ -202,8 +202,8 @@ pipeline { def error_output = "" try { error_output = sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs_dev}/ci/cases/pr/${caseName}.yaml """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body '${caseName} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n```\n${error_output}```' """) @@ -215,22 +215,22 @@ pipeline { stage("Running ${caseName}") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { - def pslot = sh(script: "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() + def pslot = sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} """) } catch (Exception error_experment) { sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot} """) ws(CUSTOM_WORKSPACE) { def error_logs = "" @@ -252,13 +252,13 @@ pipeline { } try { gist_url = sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) sh(script: """ - source ${HOMEgfs}/dev/workflow/gw_setup.sh - ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -282,8 +282,9 @@ pipeline { }] } parallel parallelStages + [failFast: true] - } + } } + } } stage( '5. Finalize' ) { From 69bc3c8d19a37d5f2a5b6d1ff66ae72858c54ccd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 17:38:39 -0400 Subject: [PATCH 21/76] adding in latest generated CI GitLab pipeline --- dev/ci/.gitlab-ci.yml | 54 ++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 4aed81d3fe5..efc48088c85 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -75,32 +75,60 @@ variables: # The following sections are generated for multiple hosts and dynamic case lists -build-EMC: +build-hera: extends: .build_template variables: - mahine: EMC - tags: ["EMC"] + mahine: hera + tags: ["hera"] -setup_experiments-EMC: +setup_experiments-hera: extends: .setup_template variables: - mahine: EMC - tags: ["EMC"] + mahine: hera + tags: ["hera"] parallel: matrix: - - caseName: ["C48_ATM"] + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] dependencies: - - build-EMC + - build-hera -run_tests-EMC: +run_tests-hera: extends: .run_tests_template variables: - mahine: EMC - tags: ["EMC"] + mahine: hera + tags: ["hera"] parallel: matrix: - - caseName: ["C48_ATM"] + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] dependencies: - - setup_experiments-EMC + - setup_experiments-hera + +build-gaeac6: + extends: .build_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + +setup_experiments-gaeac6: + extends: .setup_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - build-gaeac6 + +run_tests-gaeac6: + extends: .run_tests_template + variables: + mahine: gaeac6 + tags: ["gaeac6"] + parallel: + matrix: + - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + dependencies: + - setup_experiments-gaeac6 # End of generated sections From 9bfd17e83145776ec8b52c7cebf5f1a09b8c9528 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 17:44:09 -0400 Subject: [PATCH 22/76] added some {} for shell norms --- dev/ci/scripts/utils/ci_utils.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 8ac3fba3a77..6e34f516416 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -3,7 +3,7 @@ # Determine HOMEGFS_ and source machine detection early if [[ -z "${HOMEGFS_}" ]]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - HOMEGFS_="$(${SCRIPT_DIR}/find_homegfs.py)" + HOMEGFS_="$("${SCRIPT_DIR}/find_homegfs.py")" fi source "${HOMEGFS_}/ush/detect_machine.sh" @@ -217,11 +217,11 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then shift # Remove the function name from the arguments list # Check if the first argument corresponds to a defined function - if [[ $(type -t "$utility_function") == "function" ]]; then + if [[ $(type -t "${utility_function}") == "function" ]]; then # Call the function with the remaining arguments - "$utility_function" "$@" + "${utility_function}" "$@" else - echo "ERROR: Utility function '$utility_function' not found or not a function in ${BASH_SOURCE[0]}" >&2 + echo "ERROR: Utility function ${utility_function} not found or not a function in ${BASH_SOURCE[0]}" >&2 exit 1 fi fi From 94e68880916f5007857e76437914b346852b8f7b Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 17:46:08 -0400 Subject: [PATCH 23/76] removed vistigal debug outputs in gitlab runner launch scripts --- dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index b325e0fe406..cf1d9199e38 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -19,9 +19,6 @@ HOMEGFS_="$("${SCRIPT_DIR}/../find_homegfs.py")" # Get the hostname of the current machine host="$(hostname)" -echo $host $HOMEGFS_ -exit - ######################################################################### # Set up runtime environment variables for accounts on supported machines ######################################################################### From fa4fda45b6ad569f2e61e52a63725df7081ec941 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 18:15:41 -0400 Subject: [PATCH 24/76] updated sys.path to import find_homegfs in test_create_experments --- dev/ci/scripts/unittests/test_create_experiment.py | 6 +++++- dev/ci/scripts/utils/get_host_case_list.py | 2 +- dev/ci/scripts/utils/gitlab/generate_pipelines.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/ci/scripts/unittests/test_create_experiment.py b/dev/ci/scripts/unittests/test_create_experiment.py index 57795081dac..a525e481199 100644 --- a/dev/ci/scripts/unittests/test_create_experiment.py +++ b/dev/ci/scripts/unittests/test_create_experiment.py @@ -1,7 +1,11 @@ import os from shutil import rmtree - +from pathlib import Path +import sys from wxflow import Executable +# update sys.path to include libs in the parent directory +# TODO: replace this parent.parent reference with a ci_utils package +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) from find_homegfs import find_homegfs HOMEgfs = find_homegfs() diff --git a/dev/ci/scripts/utils/get_host_case_list.py b/dev/ci/scripts/utils/get_host_case_list.py index 22b423331c5..c78a96397a9 100755 --- a/dev/ci/scripts/utils/get_host_case_list.py +++ b/dev/ci/scripts/utils/get_host_case_list.py @@ -46,7 +46,7 @@ def get_host_cases(host, homegfs=None): if __name__ == '__main__': # When run as a script, maintain the original behavior - if len(sys.argv) < 2: + if len(sys.argv) < 2 or sys.argv[1] in ('-h', '--help'): print('Usage: get_host_case_list.py ') sys.exit(1) diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py index 1993c6fbe36..d46e77c945b 100755 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ b/dev/ci/scripts/utils/gitlab/generate_pipelines.py @@ -57,6 +57,7 @@ from pathlib import Path # update sys.path to include libs in the parent directory +# TODO: replace this parent.parent reference with a ci_utils package sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from get_host_case_list import get_host_cases from find_homegfs import find_homegfs From 623d2528cef731547ef9e57bb60a73a6a0d9ffcd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 18:21:52 -0400 Subject: [PATCH 25/76] bunch of white spaces removed from find_home script --- dev/ci/scripts/utils/find_homegfs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/ci/scripts/utils/find_homegfs.py b/dev/ci/scripts/utils/find_homegfs.py index 608837c55c4..b2b2ac1adfc 100755 --- a/dev/ci/scripts/utils/find_homegfs.py +++ b/dev/ci/scripts/utils/find_homegfs.py @@ -12,18 +12,18 @@ def find_homegfs(start_path=None): """ Find the HOMEgfs directory by traversing up the file system until finding a directory that contains the .github subdirectory. - + Parameters ---------- start_path : str or Path, optional The path to start searching from. If not provided, the current directory will be used. - + Returns ------- Path The full path to the HOMEgfs directory. - + Raises ------ ValueError @@ -35,32 +35,32 @@ def find_homegfs(start_path=None): import inspect frame = inspect.currentframe().f_back start_path = os.path.dirname(os.path.abspath(frame.f_code.co_filename)) - + # Convert to Path object if it's a string if isinstance(start_path, str): start_path = Path(start_path) else: start_path = Path(start_path).resolve() - + # Start traversing up from the current directory current_dir = start_path - + # Traverse up until we find .github directory or reach the filesystem root while True: # Check if .github exists in the current directory if (current_dir / '.github').is_dir(): return current_dir - + # Go up one level parent_dir = current_dir.parent - + # If we've reached the root directory and haven't found .github if parent_dir == current_dir: raise ValueError( "Could not find HOMEgfs directory. " "Traversed up to the root without finding a .github directory." ) - + current_dir = parent_dir From 433d387a15a96bfb190bfb6bd06ad8344b09da20 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 18:33:23 -0400 Subject: [PATCH 26/76] fixed shell norms for splitting up gett type --- dev/ci/scripts/utils/ci_utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 6e34f516416..ffd1a595258 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -217,7 +217,8 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then shift # Remove the function name from the arguments list # Check if the first argument corresponds to a defined function - if [[ $(type -t "${utility_function}") == "function" ]]; then + type_t="$(type -t "${utility_function}")" || true + if [[ "${type_t}" == "function" ]]; then # Call the function with the remaining arguments "${utility_function}" "$@" else From 58c87ce57130c1aa78a35d09b26eaa70ebd49faa Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 18:43:05 -0400 Subject: [PATCH 27/76] tried to fix shellnorms for getting SCRIPT_DIR in java launch script --- dev/ci/scripts/utils/launch_java_agent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/scripts/utils/launch_java_agent.sh b/dev/ci/scripts/utils/launch_java_agent.sh index 063d7615f60..3ae686b7f65 100755 --- a/dev/ci/scripts/utils/launch_java_agent.sh +++ b/dev/ci/scripts/utils/launch_java_agent.sh @@ -66,8 +66,8 @@ controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" # Set the HOMEGFS_ variable to the root directory of the global workflow -SCRIPT_DIR="$(cd "$(dirname \"${BASH_SOURCE[0]}\")" && pwd)" -HOMEGFS_="$(${SCRIPT_DIR}/find_homegfs.py)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HOMEGFS_="$("${SCRIPT_DIR}/find_homegfs.py")" host=$(hostname) ######################################################################### From fa4f1e166a5883e0d871669995ff692e18ffe596 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 18:50:45 -0400 Subject: [PATCH 28/76] add relative path to find the find_homegfs library in test_create+experiment script --- dev/ci/scripts/unittests/test_create_experiment.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dev/ci/scripts/unittests/test_create_experiment.py b/dev/ci/scripts/unittests/test_create_experiment.py index a525e481199..2483f4f3b90 100644 --- a/dev/ci/scripts/unittests/test_create_experiment.py +++ b/dev/ci/scripts/unittests/test_create_experiment.py @@ -3,9 +3,8 @@ from pathlib import Path import sys from wxflow import Executable -# update sys.path to include libs in the parent directory -# TODO: replace this parent.parent reference with a ci_utils package -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +# update sys.path to include the utils directory for find_homegfs +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "utils")) from find_homegfs import find_homegfs HOMEgfs = find_homegfs() From c6e6360d9b45f721a1a560af63225014f9ce6f43 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 20:45:24 -0400 Subject: [PATCH 29/76] fixed run_check_ci.sh as the corret name for the bash scripts that advaces the tests and is checking for complition --- dev/ci/Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index d696c4387db..9f1c4e2c452 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -221,7 +221,7 @@ pipeline { try { sh(script: """ source ${HOMEgfs_dev}/ush/gw_setup.sh - ${HOMEgfs_dev}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + ${HOMEgfs_dev}/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ source ${HOMEgfs_dev}/ush/gw_setup.sh @@ -300,9 +300,9 @@ pipeline { done """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log")) { - sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}
Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true) + if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log")) { + sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}
Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log""", returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log """, returnStatus: true) } if (STATUS == 'Passed') { try { From 8899c0440c4b5f3ac4f778746c032a804a2ebb57 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 15 Apr 2025 21:05:58 -0400 Subject: [PATCH 30/76] added /usr/bin/env bash for all shall scripts under ci --- dev/ci/scripts/driver_weekly.sh | 2 +- dev/ci/scripts/run_check_ci.sh | 2 +- dev/ci/scripts/utils/ci_utils.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci/scripts/driver_weekly.sh b/dev/ci/scripts/driver_weekly.sh index cd11ec1bc03..b5b8201b5ba 100755 --- a/dev/ci/scripts/driver_weekly.sh +++ b/dev/ci/scripts/driver_weekly.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux ############################################################################################## diff --git a/dev/ci/scripts/run_check_ci.sh b/dev/ci/scripts/run_check_ci.sh index fe85d275034..9e843d88517 100755 --- a/dev/ci/scripts/run_check_ci.sh +++ b/dev/ci/scripts/run_check_ci.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index ffd1a595258..3d4e71e8ed6 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -1,4 +1,4 @@ -#!/bin/env bash +#!/usr/bin/env bash # Determine HOMEGFS_ and source machine detection early if [[ -z "${HOMEGFS_}" ]]; then From 4b3396523083b77e07825ddd815f751b1cee07c5 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 16 Apr 2025 13:48:15 -0400 Subject: [PATCH 31/76] had to shorten paths on Hera because fortran pathlen=128 in fv3 jdei IO under the gdas build --- dev/ci/.gitlab-ci.yml | 2 +- dev/ci/platforms/config.hera | 2 +- dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index efc48088c85..93107e51e83 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - finalize variables: - BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' + BUILD: '' CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} diff --git a/dev/ci/platforms/config.hera b/dev/ci/platforms/config.hera index 1b589cf943d..ede02d9a995 100644 --- a/dev/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -42,7 +42,7 @@ export GITLAB_RUNNER_NAME="RDHPCS Hera" # Directory for GitLab builds # Used in launch_gitlab_runner.sh for location of builds -export GITLAB_CI_BUILDS_DIR=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD/CI_WORKSPACES/CI_GITLAB +export GITLAB_CI_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB # Directory for GitLab runner used by launch_gitlab_runner.sh export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 73cce86c445..54ef7c19c87 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -13,7 +13,7 @@ stages: - finalize variables: - BUILD: 'CI_GITLAB_GLOBAL_WORKFLOW_BUILD_DIR' + BUILD: '' CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} From 2d8094c600c3d9f8e2615af7ece8b0ca40460ae1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 16 Apr 2025 14:00:56 -0400 Subject: [PATCH 32/76] the create_gist finalize job was a template and had to be updated to be a regular job to make sure it runs --- dev/ci/.gitlab-ci.yml | 2 +- dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 93107e51e83..0614b3629b5 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -53,7 +53,7 @@ variables: dependencies: - setup_experiments-${machine} -.create_gist_from_svg_url: +create_gist_from_svg_url: image: ghcr.io/cli/cli:latest stage: finalize allow_failure: true diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index 54ef7c19c87..dace4fb9095 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -59,7 +59,7 @@ variables: dependencies: - setup_experiments-${machine} -.create_gist_from_svg_url: +create_gist_from_svg_url: image: ghcr.io/cli/cli:latest stage: finalize allow_failure: true From b462776754bd226a84260ac8de3c2ec03c059e80 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 09:19:41 -0400 Subject: [PATCH 33/76] removed the label update so we can do that next sprint with an emc-bot NWS GitLab project --- dev/ci/.gitlab-ci.yml | 30 ++++--------------- .../utils/gitlab/gitlab_pipeline_template.yml | 21 +------------ 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 0614b3629b5..1d6341fe51a 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -52,27 +52,7 @@ variables: - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} - -create_gist_from_svg_url: - image: ghcr.io/cli/cli:latest - stage: finalize - allow_failure: true - when: always - script: - - | - # Set your SVG URL, output filename, and Gist ID - SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/develop/pipeline.svg" - SVG_FILE="badge.svg" - GIST_ID="ab937691224bdf50427cbeca666bf67b" - # Download the SVG (add authentication if needed) - curl -L "$SVG_URL" -o "$SVG_FILE" - # Authenticate GitHub CLI (requires GITHUB_TOKEN env variable) - echo "$GITHUB_TOKEN" | gh auth login --with-token - # Update the existing public Gist with the SVG file - gh gist edit "$GIST_ID" "$SVG_FILE" - variables: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + # The following sections are generated for multiple hosts and dynamic case lists build-hera: @@ -88,7 +68,7 @@ setup_experiments-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar", "C96mx100_S2S"] dependencies: - build-hera @@ -99,7 +79,7 @@ run_tests-hera: tags: ["hera"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar", "C96mx100_S2S"] dependencies: - setup_experiments-hera @@ -116,7 +96,7 @@ setup_experiments-gaeac6: tags: ["gaeac6"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar"] dependencies: - build-gaeac6 @@ -127,7 +107,7 @@ run_tests-gaeac6: tags: ["gaeac6"] parallel: matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] + - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar"] dependencies: - setup_experiments-gaeac6 diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml index dace4fb9095..3c28147ba18 100644 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml @@ -58,23 +58,4 @@ variables: - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow dependencies: - setup_experiments-${machine} - -create_gist_from_svg_url: - image: ghcr.io/cli/cli:latest - stage: finalize - allow_failure: true - when: always - script: - - | - # Set your SVG URL, output filename, and Gist ID - SVG_URL="https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/badges/develop/pipeline.svg" - SVG_FILE="badge.svg" - GIST_ID="ab937691224bdf50427cbeca666bf67b" - # Download the SVG (add authentication if needed) - curl -L "$SVG_URL" -o "$SVG_FILE" - # Authenticate GitHub CLI (requires GITHUB_TOKEN env variable) - echo "$GITHUB_TOKEN" | gh auth login --with-token - # Update the existing public Gist with the SVG file - gh gist edit "$GIST_ID" "$SVG_FILE" - variables: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + \ No newline at end of file From a2aeda4aeff20f3a2f70ec581596d69af6dd581d Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 09:37:19 -0400 Subject: [PATCH 34/76] remove the old case-host pipeline include file that is now replaced by consolidated version --- dev/ci/.gitlab-ci-cases-hosts.yml | 46 ------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 dev/ci/.gitlab-ci-cases-hosts.yml diff --git a/dev/ci/.gitlab-ci-cases-hosts.yml b/dev/ci/.gitlab-ci-cases-hosts.yml deleted file mode 100644 index 1518f65e75f..00000000000 --- a/dev/ci/.gitlab-ci-cases-hosts.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Host-specific case configurations for the global-workflow project -# This file defines host-specific setup and test execution jobs - -setup_experiments-hera: - extends: .setup_template - variables: - machine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-hera - -run_tests-hera: - extends: .run_tests_template - variables: - machine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-hera - -setup_experiments-gaeac6: - extends: .setup_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - build-gaeac6 - -run_tests-gaeac6: - extends: .run_tests_template - variables: - machine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: - - setup_experiments-gaeac6 From 2171fd2d7f637ea4ec98817b7939d0ecd5b18c54 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 09:49:29 -0400 Subject: [PATCH 35/76] had naming of ctest_case_template inverted with the run_ctests_template in regargs to its extends references --- dev/ci/.gitlab-ci-hosts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 8c851aad6e2..b803db939f5 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -60,8 +60,8 @@ run_tests-gaeac6: # ======================================= # Template for CTest jobs that will be used across machines -.run_ctests_cases_template: - extends: .ctests_template +.ctests_cases_template: + extends: .run_ctests_template stage: run_tests parallel: matrix: From 37adf0c61644d6937cb41be481197c34bcffe5b8 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 10:27:01 -0400 Subject: [PATCH 36/76] still trying to get extends naming aliment set --- dev/ci/.gitlab-ci-hosts.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index b803db939f5..3b71fd0bf08 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -7,7 +7,7 @@ setup_experiments-hera: extends: .setup_template variables: machine: hera - tags: ["hera"] + tags: hera parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] @@ -20,7 +20,7 @@ run_tests-hera: extends: .run_tests_template variables: machine: hera - tags: ["hera"] + tags: hera parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] @@ -34,7 +34,7 @@ setup_experiments-gaeac6: extends: .setup_template variables: machine: gaeac6 - tags: ["gaeac6"] + tags: gaeac6 parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] @@ -47,7 +47,7 @@ run_tests-gaeac6: extends: .run_tests_template variables: machine: gaeac6 - tags: ["gaeac6"] + tags: gaeac6 parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] @@ -69,8 +69,8 @@ run_tests-gaeac6: # Host: Hera - CTests run_ctests-hera: - extends: .run_ctests_cases_template - tags: ["hera"] + extends: .ctests_cases_template + tags: hera dependencies: - create_ctests rules: @@ -78,8 +78,8 @@ run_ctests-hera: # Host: GAEAC6 - CTests run_ctests-gaeac6: - extends: .run_ctests_template - tags: ["gaeac6"] + extends: .ctests_cases_template + tags: gaeac6 dependencies: - create_ctests rules: @@ -93,10 +93,10 @@ build-hera: extends: .build_template variables: machine: hera - tags: ["hera"] + tags: hera build-gaeac6: extends: .build_template variables: machine: gaeac6 - tags: ["gaeac6"] + tags: gaeac6 From e3ece6a6188cef53a28e4f63b8baf8b9e608e063 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 10:31:18 -0400 Subject: [PATCH 37/76] added extra include of ci-hosts in ci-ctests direclty, thought it was enought have in in the main (trying) --- dev/ci/.gitlab-ci-ctests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index dd0cc65f1dd..b1d78185d25 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -2,6 +2,9 @@ # Templates for CTests # ======================================= +include: + - local: 'dev/ci/.gitlab-ci-hosts.yml' + # Setup job for ctests using CMake .create_ctests: extends: .base_config From b1a5c76d45961088e22c84c09782e2bae30d9ffd Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 11:05:23 -0400 Subject: [PATCH 38/76] used an all inclusive approach of including depecancies --- dev/ci/.gitlab-ci-ctests.yml | 3 --- dev/ci/.gitlab-ci.yml | 8 -------- 2 files changed, 11 deletions(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index b1d78185d25..dd0cc65f1dd 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -2,9 +2,6 @@ # Templates for CTests # ======================================= -include: - - local: 'dev/ci/.gitlab-ci-hosts.yml' - # Setup job for ctests using CMake .create_ctests: extends: .base_config diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 5d6c4842f61..e79e1cae4a4 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -20,17 +20,9 @@ variables: GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} -# Include the appropriate modality file based on the trigger include: - # Include ctests configuration when triggered via GitHub API - local: 'dev/ci/.gitlab-ci-ctests.yml' - rules: - - if: $GITHUB_API_TRIGGER == "true" - # Include standard cases configuration when not triggered via GitHub API - local: 'dev/ci/.gitlab-ci-cases.yml' - rules: - - if: $GITHUB_API_TRIGGER != "true" - # Always include host-specific configurations - local: 'dev/ci/.gitlab-ci-hosts.yml' # Common build template for all modalities From c3f298ecbeb28c71dd682ac0d712ec8d88c8fe46 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 11:23:23 -0400 Subject: [PATCH 39/76] had the tags sytax wrong there for a bit in the host include file --- dev/ci/.gitlab-ci-hosts.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 3b71fd0bf08..a8c4e8f6730 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -7,7 +7,8 @@ setup_experiments-hera: extends: .setup_template variables: machine: hera - tags: hera + tags: + - hera parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] @@ -20,7 +21,8 @@ run_tests-hera: extends: .run_tests_template variables: machine: hera - tags: hera + tags: + - hera parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] @@ -34,7 +36,8 @@ setup_experiments-gaeac6: extends: .setup_template variables: machine: gaeac6 - tags: gaeac6 + tags: + - gaeac6 parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] @@ -47,7 +50,8 @@ run_tests-gaeac6: extends: .run_tests_template variables: machine: gaeac6 - tags: gaeac6 + tags: + - gaeac6 parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] @@ -70,7 +74,8 @@ run_tests-gaeac6: # Host: Hera - CTests run_ctests-hera: extends: .ctests_cases_template - tags: hera + tags: + - hera dependencies: - create_ctests rules: @@ -79,7 +84,8 @@ run_ctests-hera: # Host: GAEAC6 - CTests run_ctests-gaeac6: extends: .ctests_cases_template - tags: gaeac6 + tags: + - gaeac6 dependencies: - create_ctests rules: @@ -93,10 +99,12 @@ build-hera: extends: .build_template variables: machine: hera - tags: hera + tags: + - hera build-gaeac6: extends: .build_template variables: machine: gaeac6 - tags: gaeac6 + tags: + - gaeac6 From 6a4e76fcc33353c1727c715b8f56951238b598f5 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 11:53:34 -0400 Subject: [PATCH 40/76] was missing the insstantiation of the create ctests per host --- dev/ci/.gitlab-ci-hosts.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index a8c4e8f6730..05bd4442156 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -71,13 +71,38 @@ run_tests-gaeac6: matrix: - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod'] +# Host-specific CTest setup jobs +create_ctests-hera: + extends: .create_ctests + stage: create_experiments + tags: + - hera + variables: + machine: hera + dependencies: + - build-hera + rules: + - if: $GITHUB_API_TRIGGER == "true" + +create_ctests-gaeac6: + extends: .create_ctests + stage: create_experiments + tags: + - gaeac6 + variables: + machine: gaeac6 + dependencies: + - build-gaeac6 + rules: + - if: $GITHUB_API_TRIGGER == "true" + # Host: Hera - CTests run_ctests-hera: extends: .ctests_cases_template tags: - hera dependencies: - - create_ctests + - create_ctests-hera rules: - if: $GITHUB_API_TRIGGER == "true" @@ -87,10 +112,12 @@ run_ctests-gaeac6: tags: - gaeac6 dependencies: - - create_ctests + - create_ctests-gaeac6 rules: - if: $GITHUB_API_TRIGGER == "true" + + # ======================================= # Common build configurations by host # These will always be included, regardless of modality From 4219a84d1027b491e7420a21859b81e63b5f428f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 12:57:47 -0400 Subject: [PATCH 41/76] updated cmake ctest commands for vertical structgure in GitLab pipeline --- dev/ci/.gitlab-ci-ctests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index dd0cc65f1dd..d033766d4de 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -8,9 +8,9 @@ stage: create_experiments script: - echo "Setting up for ctests workflow" - - mkdir -p ${HOMEGFS}/ctests - - cd ${HOMEGFS}/ctests - - cmake ../.. + - cd ${HOMEGFS}/dev/ctests + - mkdir -p build + - cmake ../../.. - echo "CTests have been created and configured successfully" dependencies: - build-${machine} From 928f84acb53681b86046b6c9a06be11c11a44c93 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 14:27:54 -0400 Subject: [PATCH 42/76] forgot to cd into build for cmake ../../.. --- dev/ci/.gitlab-ci-ctests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index d033766d4de..e7fbd00da2c 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -10,6 +10,7 @@ - echo "Setting up for ctests workflow" - cd ${HOMEGFS}/dev/ctests - mkdir -p build + - cd build - cmake ../../.. - echo "CTests have been created and configured successfully" dependencies: From f9d79c542e842a91a8c57fc25c7a95d4dec4727d Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 17:18:02 -0400 Subject: [PATCH 43/76] fixed up main ctest pipeline since all the tag a depenency logic is in the host file --- dev/ci/.gitlab-ci-ctests.yml | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index e7fbd00da2c..6130ed4c112 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -12,13 +12,10 @@ - mkdir -p build - cd build - cmake ../../.. + - ctest -N + - num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") + - '[ "$num" -gt 0 ] || { echo "No tests found"; exit 1; }' - echo "CTests have been created and configured successfully" - dependencies: - - build-${machine} - tags: - - ${machine} - rules: - - if: $GITHUB_API_TRIGGER == "true" # Main template for CTest execution jobs .run_ctests_template: @@ -26,17 +23,9 @@ stage: run_tests allow_failure: true script: - - echo "Running ${CTEST} tests in ${HOMEGFS}/ctests" - - if [ "${PR_NUMBER}" != "0" ]; then - - echo "Testing PR #${PR_NUMBER}" - - else - - echo "Testing standard branch" - - fi - - cd ${HOMEGFS}/ctests/build + - echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" + - cd ${HOMEGFS}/dev/ctests/build - ctest -R test_${CTEST}_setup - ctest -R test_${CTEST}_stage - ctest -R test_${CTEST}_execute - dependencies: - - create_ctests - rules: - - if: $GITHUB_API_TRIGGER == "true" + - ctest -R test_${CTEST}_validate From 8e6638d2b1fcb34e384bc158b28bbcae60207bf0 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 17:35:27 -0400 Subject: [PATCH 44/76] ctests setup templated bash script was not pointing to create experment in the new virtical structure --- dev/ctests/scripts/setup.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ctests/scripts/setup.sh.in b/dev/ctests/scripts/setup.sh.in index 43ac2b7b01b..10a263fac84 100755 --- a/dev/ctests/scripts/setup.sh.in +++ b/dev/ctests/scripts/setup.sh.in @@ -19,7 +19,7 @@ pslot="${TEST_NAME}" \ RUNTESTS="${RUNTESTS}" \ ICSDIR_ROOT="${ICSDIR_ROOT}" \ HPC_ACCOUNT="${HPC_ACCOUNT}" \ -"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite +"${HOMEgfs}/dev/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite rc=$? if [[ "${rc}" -ne 0 ]]; then set +x From 987ccd6649b0b4e01555d472449d7da120ac7d80 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 19:58:30 -0400 Subject: [PATCH 45/76] removed allow fail as it was not usefull durint the test cases --- dev/ci/.gitlab-ci-cases.yml | 4 ++-- dev/ci/.gitlab-ci-ctests.yml | 10 +++++----- dev/ci/.gitlab-ci-hosts.yml | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml index f0f0ac3c155..090178779c6 100644 --- a/dev/ci/.gitlab-ci-cases.yml +++ b/dev/ci/.gitlab-ci-cases.yml @@ -10,7 +10,7 @@ .setup_template: extends: .base_config stage: create_experiments - allow_failure: true + # allow_failure removed to ensure setup failures are properly reported script: - export RUNTESTS=${RUNTESTS_DIR} - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml @@ -21,7 +21,7 @@ .run_tests_template: extends: .base_config stage: run_tests - allow_failure: true + # allow_failure is removed to ensure test failures are properly reported script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 6130ed4c112..9919bd333b2 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -8,6 +8,7 @@ stage: create_experiments script: - echo "Setting up for ctests workflow" + - source ci/platfroms/config.${machine} - cd ${HOMEGFS}/dev/ctests - mkdir -p build - cd build @@ -21,11 +22,10 @@ .run_ctests_template: extends: .base_config stage: run_tests - allow_failure: true script: - echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" - cd ${HOMEGFS}/dev/ctests/build - - ctest -R test_${CTEST}_setup - - ctest -R test_${CTEST}_stage - - ctest -R test_${CTEST}_execute - - ctest -R test_${CTEST}_validate + - ctest -V -R test_${CTEST}_setup + - ctest -V -R test_${CTEST}_stage + - ctest -V -R test_${CTEST}_execute + - ctest -V -R test_${CTEST}_validate diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 05bd4442156..5caa871552c 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -69,7 +69,9 @@ run_tests-gaeac6: stage: run_tests parallel: matrix: - - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod'] + - CTEST: ['C48_ATM_gfs_fcst_seg0'] +# - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod_f000-f003'] +# TODO only running one test for now # Host-specific CTest setup jobs create_ctests-hera: From edca1cfad1cee20d9d51727a7b458d292e5d7404 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 20:58:25 -0400 Subject: [PATCH 46/76] missed dev again on path to source config. --- dev/ci/.gitlab-ci-ctests.yml | 2 +- dev/ci/.gitlab-ci-hosts.yml | 2 -- dev/ci/.gitlab-ci.yml | 9 ++++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 9919bd333b2..3c8b522222c 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -8,7 +8,7 @@ stage: create_experiments script: - echo "Setting up for ctests workflow" - - source ci/platfroms/config.${machine} + - source dev/ci/platfroms/config.${machine} - cd ${HOMEGFS}/dev/ctests - mkdir -p build - cd build diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 5caa871552c..639964f55fe 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -118,8 +118,6 @@ run_ctests-gaeac6: rules: - if: $GITHUB_API_TRIGGER == "true" - - # ======================================= # Common build configurations by host # These will always be included, regardless of modality diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index e79e1cae4a4..e4c5816d696 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -16,7 +16,6 @@ variables: GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h - # GitHub API trigger detection - defaults to false GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} @@ -25,15 +24,19 @@ include: - local: 'dev/ci/.gitlab-ci-cases.yml' - local: 'dev/ci/.gitlab-ci-hosts.yml' +.base_config: + variables: + GIT_STRATEGY: none + # Common build template for all modalities .build_template: variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive stage: build - allow_failure: true script: - echo "Using build directory ${HOMEGFS}" - - dev/ci/scripts/utils/ci_utils.sh build_compute + #- dev/ci/scripts/utils/ci_utils.sh build_compute + - echo "Skipping build_compute step" - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} From f888d4694fbae525926a39b6ecce748e90a49dfc Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 21:19:04 -0400 Subject: [PATCH 47/76] needed full path because job had GIT_STRATEGY set to none --- dev/ci/.gitlab-ci-ctests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 3c8b522222c..500e61617ce 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -8,7 +8,7 @@ stage: create_experiments script: - echo "Setting up for ctests workflow" - - source dev/ci/platfroms/config.${machine} + - source ${HOMEGFS}/dev/ci/platfroms/config.${machine} - cd ${HOMEGFS}/dev/ctests - mkdir -p build - cd build From 14afd70d4bc102af37ca0706975eaa228a0b0553 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 21:36:50 -0400 Subject: [PATCH 48/76] super imposed spelling of platforms and put dependecies back in at the top level for ctests --- dev/ci/.gitlab-ci-ctests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 500e61617ce..2c1b63b6acb 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -8,7 +8,7 @@ stage: create_experiments script: - echo "Setting up for ctests workflow" - - source ${HOMEGFS}/dev/ci/platfroms/config.${machine} + - source ${HOMEGFS}/dev/ci/platforms/config.${machine} - cd ${HOMEGFS}/dev/ctests - mkdir -p build - cd build @@ -17,6 +17,8 @@ - num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") - '[ "$num" -gt 0 ] || { echo "No tests found"; exit 1; }' - echo "CTests have been created and configured successfully" + dependencies: + - build-${machine} # Main template for CTest execution jobs .run_ctests_template: @@ -29,3 +31,5 @@ - ctest -V -R test_${CTEST}_stage - ctest -V -R test_${CTEST}_execute - ctest -V -R test_${CTEST}_validate + dependencies: + - create_ctests-${machine} From a63463bc9039ad65e6dbe3c99903998f564edb29 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 22:00:19 -0400 Subject: [PATCH 49/76] My coding partner suggested to use a DAG so we can relax the bottle neck of waiting for builds to finish between hosts --- dev/ci/.gitlab-ci-cases.yml | 14 +++----------- dev/ci/.gitlab-ci-ctests.yml | 4 ++-- dev/ci/.gitlab-ci-hosts.yml | 16 ++++++++-------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml index 090178779c6..2e6d82fada0 100644 --- a/dev/ci/.gitlab-ci-cases.yml +++ b/dev/ci/.gitlab-ci-cases.yml @@ -1,30 +1,22 @@ -# Standard cases modality configuration for the global-workflow project -# This file defines jobs specific to the standard test cases workflow - -# Base configuration for case jobs -.base_config: - variables: - GIT_STRATEGY: none +# This file defines jobs to create and run specific to PR cases # Template for experiment setup jobs .setup_template: extends: .base_config stage: create_experiments - # allow_failure removed to ensure setup failures are properly reported script: - export RUNTESTS=${RUNTESTS_DIR} - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - dependencies: + needs: - build-${machine} # Template for test execution jobs .run_tests_template: extends: .base_config stage: run_tests - # allow_failure is removed to ensure test failures are properly reported script: - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow - dependencies: + needs: - setup_experiments-${machine} \ No newline at end of file diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 2c1b63b6acb..50ac2252876 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -17,7 +17,7 @@ - num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") - '[ "$num" -gt 0 ] || { echo "No tests found"; exit 1; }' - echo "CTests have been created and configured successfully" - dependencies: + needs: - build-${machine} # Main template for CTest execution jobs @@ -31,5 +31,5 @@ - ctest -V -R test_${CTEST}_stage - ctest -V -R test_${CTEST}_execute - ctest -V -R test_${CTEST}_validate - dependencies: + needs: - create_ctests-${machine} diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 639964f55fe..49c8092aaaa 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -12,7 +12,7 @@ setup_experiments-hera: parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: + needs: - build-hera rules: - if: $GITHUB_API_TRIGGER != "true" @@ -26,7 +26,7 @@ run_tests-hera: parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C96mx100_S2S", "C48mx500_3DVarAOWCDA"] - dependencies: + needs: - setup_experiments-hera rules: - if: $GITHUB_API_TRIGGER != "true" @@ -41,7 +41,7 @@ setup_experiments-gaeac6: parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: + needs: - build-gaeac6 rules: - if: $GITHUB_API_TRIGGER != "true" @@ -55,7 +55,7 @@ run_tests-gaeac6: parallel: matrix: - caseName: ["C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C48_S2SWA_gefs", "C96C48_hybatmaerosnowDA", "C48_S2SW", "C96_atm3DVar", "C48_ATM", "C48mx500_3DVarAOWCDA"] - dependencies: + needs: - setup_experiments-gaeac6 rules: - if: $GITHUB_API_TRIGGER != "true" @@ -81,7 +81,7 @@ create_ctests-hera: - hera variables: machine: hera - dependencies: + needs: - build-hera rules: - if: $GITHUB_API_TRIGGER == "true" @@ -93,7 +93,7 @@ create_ctests-gaeac6: - gaeac6 variables: machine: gaeac6 - dependencies: + needs: - build-gaeac6 rules: - if: $GITHUB_API_TRIGGER == "true" @@ -103,7 +103,7 @@ run_ctests-hera: extends: .ctests_cases_template tags: - hera - dependencies: + needs: - create_ctests-hera rules: - if: $GITHUB_API_TRIGGER == "true" @@ -113,7 +113,7 @@ run_ctests-gaeac6: extends: .ctests_cases_template tags: - gaeac6 - dependencies: + needs: - create_ctests-gaeac6 rules: - if: $GITHUB_API_TRIGGER == "true" From 4495d1423fb474d9ec248e05724829a3ad6095c1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 18 Apr 2025 22:46:42 -0400 Subject: [PATCH 50/76] put build back in not that it runs through --- dev/ci/.gitlab-ci.yml | 3 +-- dev/ctests/scripts/execute.sh.in | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index e4c5816d696..85390ec2faa 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -36,7 +36,6 @@ include: stage: build script: - echo "Using build directory ${HOMEGFS}" - #- dev/ci/scripts/utils/ci_utils.sh build_compute - - echo "Skipping build_compute step" + - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} diff --git a/dev/ctests/scripts/execute.sh.in b/dev/ctests/scripts/execute.sh.in index 52ae03a43ae..8dc17f2c654 100755 --- a/dev/ctests/scripts/execute.sh.in +++ b/dev/ctests/scripts/execute.sh.in @@ -28,7 +28,7 @@ lack_of_job_count=0 LACK_OF_JOB_LIMIT=5 while true; do - job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1) || true + job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | tr -cd '[:alpha:]' | xargs) || true if [[ -n "${job_status}" ]]; then echo "Job ${job_id} found in sacct." break @@ -46,8 +46,7 @@ done timeout=0 TIMEOUT=60 while true; do - # Trim trailing spaces from job_status - job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | xargs) || true + job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | tr -cd '[:alpha:]' | xargs) || true if [[ "${job_status}" == "COMPLETED" ]]; then echo "Job ${job_id} completed successfully." break From e749491224088b2c93c38390252bb84a1a1828f1 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Sat, 19 Apr 2025 01:20:06 -0400 Subject: [PATCH 51/76] cleaned up few left overs, added all ctests back and used -L per test group --- dev/ci/.gitlab-ci-ctests.yml | 5 +---- dev/ci/.gitlab-ci-hosts.yml | 4 +--- dev/ctests/CMakeLists.txt | 6 +++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 50ac2252876..a30e8f62f96 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -27,9 +27,6 @@ script: - echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" - cd ${HOMEGFS}/dev/ctests/build - - ctest -V -R test_${CTEST}_setup - - ctest -V -R test_${CTEST}_stage - - ctest -V -R test_${CTEST}_execute - - ctest -V -R test_${CTEST}_validate + - ctest -L ${CTEST} needs: - create_ctests-${machine} diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 49c8092aaaa..4eb9bbecccb 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -69,9 +69,7 @@ run_tests-gaeac6: stage: run_tests parallel: matrix: - - CTEST: ['C48_ATM_gfs_fcst_seg0'] -# - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod_f000-f003'] -# TODO only running one test for now + - CTEST: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0', 'C48_S2SW_gfs_atmos_prod_f000-f003'] # Host-specific CTest setup jobs create_ctests-hera: diff --git a/dev/ctests/CMakeLists.txt b/dev/ctests/CMakeLists.txt index 3ec286852be..25de0fce593 100644 --- a/dev/ctests/CMakeLists.txt +++ b/dev/ctests/CMakeLists.txt @@ -85,18 +85,18 @@ function(AddJJOBTest) add_test(NAME test_${TEST_NAME}_stage COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE}_${ARG_JOB}") add_test(NAME test_${TEST_NAME}_execute COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE}_${ARG_JOB}") # TODO - This is a stub for the validation step add_test(NAME test_${TEST_NAME}_validate COMMAND ./validate.sh ${TEST_NAME} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE}_${ARG_JOB}") endfunction() AddJJOBTest( From 7c29e1ffefc83079a7c5568e555ac78a85e51243 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 16:25:28 -0400 Subject: [PATCH 52/76] bringing back new main pipeline from final design. --- dev/ci/.gitlab-ci.yml | 81 ------------------------------------------- 1 file changed, 81 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index ccb37511c60..85390ec2faa 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -39,84 +39,3 @@ include: - dev/ci/scripts/utils/ci_utils.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} - -.setup_template: - extends: .base_config - stage: create_experiments - allow_failure: true - script: - - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - dependencies: - - build-${machine} - -.run_tests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow - dependencies: - - setup_experiments-${machine} - -# The following sections are generated for multiple hosts and dynamic case lists - -build-hera: - extends: .build_template - variables: - mahine: hera - tags: ["hera"] - -setup_experiments-hera: - extends: .setup_template - variables: - mahine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar", "C96mx100_S2S"] - dependencies: - - build-hera - -run_tests-hera: - extends: .run_tests_template - variables: - mahine: hera - tags: ["hera"] - parallel: - matrix: - - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar", "C96mx100_S2S"] - dependencies: - - setup_experiments-hera - -build-gaeac6: - extends: .build_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - -setup_experiments-gaeac6: - extends: .setup_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar"] - dependencies: - - build-gaeac6 - -run_tests-gaeac6: - extends: .run_tests_template - variables: - mahine: gaeac6 - tags: ["gaeac6"] - parallel: - matrix: - - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA", "C96_atm3DVar"] - dependencies: - - setup_experiments-gaeac6 - -# End of generated sections From fda2752128fbd543b17a76ff91f78089e35586f2 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 16:27:16 -0400 Subject: [PATCH 53/76] removed old docs and generate_pipelines --- dev/ci/scripts/utils/gitlab/docs/README.md | 114 ------ .../utils/gitlab/generate_pipelines.py | 332 ------------------ .../utils/gitlab/gitlab_pipeline_template.yml | 77 ---- 3 files changed, 523 deletions(-) delete mode 100644 dev/ci/scripts/utils/gitlab/docs/README.md delete mode 100755 dev/ci/scripts/utils/gitlab/generate_pipelines.py delete mode 100644 dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md deleted file mode 100644 index a9b224a612f..00000000000 --- a/dev/ci/scripts/utils/gitlab/docs/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# GitLab CI Pipeline Configuration - -This documentation describes the multi-modal GitLab CI pipeline configuration for the global-workflow project. - -## Pipeline Modalities - -The CI pipeline supports two different modalities: - -1. **Standard Pipeline** - The default pipeline that runs when triggered by normal GitLab events (push, merge requests) -2. **CTests Pipeline** - A specialized pipeline that runs when triggered via GitLab API from GitHub - -## Multi-Host Support - -Both pipeline modalities support running on multiple compute hosts using a shared configuration system. The pipeline is dynamically generated by the `generate_pipelines.py` script, which creates jobs for each supported host machine based on the test cases that each host supports. - -## Pipeline Structure - -Both modalities use the following stages: -- `build`: Builds the codebase (shared between modalities) -- `create_experiments`: Sets up experiments or ctests -- `run_tests`: Executes the test workflows -- `finalize`: Updates status badges - -## Standard Pipeline - -The standard pipeline when triggered normally: -- Builds the codebase for each machine -- Sets up experiments using configurations in `dev/ci/cases/pr/` -- Runs tests for each case on the appropriate machine -- Completes with final status reporting - -## CTests Pipeline - -The CTests pipeline when triggered via GitLab API from GitHub: -- Uses the same build jobs as the standard pipeline -- The build stage additionally sets up the CMake environment for ctests -- Skips the standard experiment creation -- Runs specialized CTest test cases on each machine - -### Currently Supported CTests - -The pipeline currently supports the following tests: -- `C48_ATM_gfs_fcst_seg0` -- `C48_S2SW_gfs_fcst_seg0` -- `C48_S2SW_gfs_atmos_prod` - -Additional tests will be added in the future from within the ctest framework in `dev/ctests`. - -## Shared Build Stage - -Both modalities share the same build stage, which: -1. Clones the repository -2. Builds the necessary components -3. Sets up the workspace for testing -4. If triggered by GitHub API, also prepares the CMake environment for ctests - -This shared approach ensures consistent build results while optimizing resource usage. - -## Triggering the CTests Pipeline from GitHub - -To trigger the CTests pipeline from GitHub, use the GitLab API with the following parameters: - -```bash -curl -X POST \ - --header "Content-Type: application/json" \ - --header "PRIVATE-TOKEN: " \ - "https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/api/v4/projects//trigger/pipeline" \ - --data '{ - "ref": "", - "variables": { - "GITHUB_API_TRIGGER": "true", - "PR_NUMBER": "", - "GITHUB_REPO_URL": "" - } - }' -``` - -Replace the following: -- ``: Your GitLab API token with appropriate permissions -- ``: The GitLab project ID for global-workflow -- ``: The branch to run tests on -- ``: The GitHub PR number (use "0" for non-PR runs) -- ``: The GitHub repository URL - -## Pipeline Configuration Files - -The configuration is split across multiple files: - -1. **Main Pipeline File** (`.gitlab-ci.yml`): - - Contains the common stages and variables - - Includes the appropriate modality file based on trigger - - Contains the build template shared by both modalities - - Contains the finalize stage job - -2. **Cases Configuration** (`.gitlab-ci-cases.yml`): - - Contains job templates specific to standard test cases - - Includes templates for running experiments - -3. **CTests Configuration** (`.gitlab-ci-ctests.yml`): - - Contains specific configurations for CTests - - Included conditionally when triggered via GitHub API - -## Pipeline Generation Process - -1. The `generate_pipelines.py` script: - - Reads the template file - - Determines which test cases each machine supports - - Generates machine-specific jobs for both standard tests and ctests - - Outputs the complete pipeline configuration - -2. When the pipeline runs: - - GitLab detects if `GITHUB_API_TRIGGER` is set to "true" - - If true, includes the `.gitlab-ci-ctests.yml` file - - Jobs check this variable to determine if they should run \ No newline at end of file diff --git a/dev/ci/scripts/utils/gitlab/generate_pipelines.py b/dev/ci/scripts/utils/gitlab/generate_pipelines.py deleted file mode 100755 index ce159415dc8..00000000000 --- a/dev/ci/scripts/utils/gitlab/generate_pipelines.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to generate GitLab CI pipeline configuration based on supported test cases -for each machine. - -Overview --------- -This script automates the creation of a `.gitlab-ci.yml` file tailored for the -current repository and its supported test environments. Its main role is to -dynamically generate the GitLab CI pipeline configuration, ensuring that the -pipeline reflects the actual set of supported test cases for each target machine. - -Process -------- -1. **Discovery of Supported Test Cases**: - - The script imports and uses `get_host_case_list.py` (via `get_host_cases`) to - determine which test cases are supported on each specified machine. - - The list of machines is provided via command-line arguments. - -2. **Template-Based Configuration**: - - A template YAML file is provided (via `--template` argument) that contains - the static, reusable parts of the pipeline configuration. - - The script reads this template, extracting the relevant content below a - demarcation marker. - -3. **Dynamic Section Generation**: - - For each machine, the script generates YAML configuration sections for - build, setup, and test jobs, using the discovered test cases as a matrix. - - These sections are appended to the template content. - -4. **Output**: - - The final, combined configuration is written to `.gitlab-ci.yml` in the - repository's CI directory (or to a user-specified output path). - - This output file is what GitLab CI will use to define and run the pipeline. - -Role in Workflow ----------------- -- This script is intended to be run whenever the set of supported test cases or - machines changes, or when the pipeline template is updated. -- It ensures that the CI pipeline is always in sync with the actual capabilities - of the codebase and test infrastructure, reducing manual maintenance and errors. -- The generated `.gitlab-ci.yml` is the authoritative pipeline definition for - GitLab CI/CD in this repository. - -Usage ------ -Run this script with the required arguments: - python generate_pipelines.py --machines --template [--output ] - -This will produce or update the `.gitlab-ci.yml` file for use by GitLab CI. - -""" - -import os -import sys -import argparse -from pathlib import Path - -# update sys.path to include libs in the parent directory -# TODO: replace this parent.parent reference with a ci_utils package -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from get_host_case_list import get_host_cases -from find_homegfs import find_homegfs - -# Get the path to the top directory of the repository -# to place the generated .gitlab-ci.yml file in the default location -_homegfs = find_homegfs(os.path.dirname(os.path.abspath(__file__))) - - -def get_case_list_for_machine(machine): - """ - Get the list of supported cases for the given machine. - - Parameters - ---------- - machine : str - The name of the machine to get supported cases for. - - Returns - ------- - list - A list of test case names supported on the specified machine. - """ - - cases = get_host_cases(machine, homegfs=_homegfs) - return cases - - -def generate_machine_config(machine, case_list): - """ - Generate the machine-specific configuration sections for GitLab CI. - - Parameters - ---------- - machine : str - The name of the machine for which to generate configuration. - case_list : list - A list of test case names supported on the machine. - - Returns - ------- - tuple - A tuple containing (main_config, cases_config, ctests_config) - YAML configurations for: - - main_config: build job for the main pipeline - - cases_config: setup and run_tests jobs for the cases modality - - ctests_config: run_ctests job for the ctests modality - """ - - case_str = '", "'.join(case_list) - case_list_yaml = f'["{case_str}"]' - - # Build job for the main pipeline - main_config = f''' -build-{machine}: - extends: .build_template - variables: - machine: {machine} - tags: ["{machine}"] -''' - - # Setup and run_tests jobs for the cases modality - cases_config = f''' -setup_experiments-{machine}: - extends: .setup_template - variables: - machine: {machine} - tags: ["{machine}"] - parallel: - matrix: - - caseName: {case_list_yaml} - dependencies: - - build-{machine} - -run_tests-{machine}: - extends: .run_tests_template - variables: - machine: {machine} - tags: ["{machine}"] - parallel: - matrix: - - caseName: {case_list_yaml} - dependencies: - - setup_experiments-{machine} -''' - - # CTests job for the ctests modality - ctests_config = f''' -run_ctests-{machine}: - extends: .run_ctests_template - variables: - machine: {machine} - tags: ["{machine}"] - dependencies: - - create_ctests -''' - - return (main_config, cases_config, ctests_config) - - -def read_template_file(template_path): - """ - Read the template file and extract the content below the demarcation line. - - Parameters - ---------- - template_path : str - Path to the template file containing the base GitLab CI configuration. - - Returns - ------- - str - Content of the template file below the demarcation line. - """ - with open(template_path, 'r') as f: - lines = f.readlines() - - # Find the demarcation line that separates the header from the usable template - marker_line = "# ------------------------------------------------------------" - - for i, line in enumerate(lines): - if line.strip() == marker_line: - return ''.join(lines[i + 1:]) - - # If marker line not found, return an empty string - return '' - - -def generate_pipeline_config(machines, template_file, output_file=None): - """ - Generate the complete GitLab CI pipeline configuration. - - This function combines a template file with machine-specific configurations - based on the supported test cases for each machine. - - Parameters - ---------- - machines : list - List of machine names to include in the pipeline configuration. - template_file : str - Path to the template file containing the base configuration. - The template should end with a marker line: "# Machine-specific jobs generated from template:" - output_file : str, optional - Path where the generated configuration will be written. - If not provided, defaults to ci/.gitlab-ci.yml in the repository root. - - Raises - ------ - ValueError - If the template file does not exist. - """ - # Set default output file paths if not specified - main_output = output_file or os.path.join(_homegfs, 'dev/ci', '.gitlab-ci.yml') - cases_output = os.path.join(_homegfs, 'dev/ci', '.gitlab-ci-cases.yml') - ctests_output = os.path.join(_homegfs, 'dev/ci', '.gitlab-ci-ctests.yml') - - # Read the current contents of the files to preserve header sections - if os.path.exists(main_output): - with open(main_output, 'r') as f: - main_content = f.read() - # Extract content before the generated sections - main_marker = "# The following sections are generated" - main_parts = main_content.split(main_marker) - main_header = main_parts[0] if len(main_parts) > 0 else main_content - else: - # If file doesn't exist, use template content - if os.path.exists(template_file): - main_header = read_template_file(template_file) - else: - raise ValueError(f"Template file {template_file} not found") - - # Do the same for cases file - if os.path.exists(cases_output): - with open(cases_output, 'r') as f: - cases_content = f.read() - cases_marker = "# The machine-specific jobs will be generated" - cases_parts = cases_content.split(cases_marker) - cases_header = cases_parts[0] if len(cases_parts) > 0 else cases_content - else: - cases_header = "# Standard cases modality configuration\n\n" - - # And for ctests file - if os.path.exists(ctests_output): - with open(ctests_output, 'r') as f: - ctests_content = f.read() - ctests_marker = "# The machine-specific jobs will be generated" - ctests_parts = ctests_content.split(ctests_marker) - ctests_header = ctests_parts[0] if len(ctests_parts) > 0 else ctests_content - else: - ctests_header = "# CTests modality configuration\n\n" - - # Initialize with the headers - main_config = main_header.rstrip() + "\n\n# The following sections are generated for multiple hosts by the generate_pipelines.py script\n" - cases_config = cases_header - ctests_config = ctests_header - - # Generate machine-specific configurations - for machine in machines: - case_list = get_case_list_for_machine(machine) - if not case_list: - print(f"Warning: No supported cases found for machine {machine}", file=sys.stderr) - continue - - main_part, cases_part, ctests_part = generate_machine_config(machine, case_list) - main_config += main_part - # Only add cases and ctests parts if they're non-empty - if cases_part.strip(): - cases_config += cases_part - if ctests_part.strip(): - ctests_config += ctests_part - - # Add comments to indicate the end of generated sections - main_config += '\n# End of generated sections\n' - - # Ensure no blank line at the top of the resulting pipelines - main_config = main_config.lstrip() - cases_config = cases_config.lstrip() - ctests_config = ctests_config.lstrip() - - # Write the complete configurations to the output files - with open(main_output, 'w') as f: - f.write(main_config) - - # Only write to cases and ctests files if they don't contain markers - # This allows us to preserve template sections in these files - if "# The machine-specific jobs will be generated" in cases_content: - with open(cases_output, 'w') as f: - f.write(cases_config) - - if "# The machine-specific jobs will be generated" in ctests_content: - with open(ctests_output, 'w') as f: - f.write(ctests_config) - - print(f"GitLab CI pipeline configurations generated:") - print(f" - Main pipeline: {main_output}") - print(f" - Cases modality: {cases_output}") - print(f" - CTests modality: {ctests_output}") - - -def main(): - """ - Parse command line arguments and generate the GitLab CI pipeline configuration. - - This is the main entry point for the script when executed directly. - It parses command line arguments and calls generate_pipeline_config() - with the appropriate parameters. - - Command line arguments: - --machines : str - Comma-separated list of machines to include in the pipeline. - --template : str - Path to the template file containing the base configuration. - --output : str, optional - Path where the generated configuration will be written. - - Returns - ------- - None - """ - - parser = argparse.ArgumentParser(description='Generate GitLab CI pipeline configuration.') - parser.add_argument('--machines', required=True, help='Comma-separated list of machines to include in the pipeline') - parser.add_argument('--template', required=True, help='Path to the template file for the pipeline configuration') - parser.add_argument('--output', default=None, help='Path to the output file (default: ci/.gitlab-ci.yml)') - - args = parser.parse_args() - - machines = [machine.strip() for machine in args.machines.split(',')] - generate_pipeline_config(machines, args.template, args.output) - - -if __name__ == '__main__': - main() diff --git a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml b/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml deleted file mode 100644 index 1a7ab0f66b6..00000000000 --- a/dev/ci/scripts/utils/gitlab/gitlab_pipeline_template.yml +++ /dev/null @@ -1,77 +0,0 @@ -# THIS FILE IS THE CORE TEMPLATE FOR THE GITLAB CI/CD PIPELINES -# Lines below the following demarcation line will be included in the resulting pipeline. -# Do not edit the generated sections in the resulting .gitlab-ci.yml file when updating -# the system but here instead (in this file). -# ------------------------------------------------------------ - -# This file is used to define the GitLab CI/CD pipeline stages: -# building, setting up experiments, and running tests on different machines. -stages: - - build - - create_experiments - - run_tests - - finalize - -variables: - BUILD: '' - CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} - GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' - HOMEGFS: ${GIT_CLONE_PATH} - RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS - GIT_DEPTH: 1 - RUNNER_SCRIPT_TIMEOUT: 6h - RUNNER_AFTER_SCRIPT_TIMEOUT: 6h - # GitHub API trigger detection - defaults to false - GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} - PR_NUMBER: ${PR_NUMBER:-"0"} - -# Include the appropriate modality file based on the trigger -include: - # Include ctests configuration when triggered via GitHub API - - local: 'dev/ci/.gitlab-ci-ctests.yml' - rules: - - if: $GITHUB_API_TRIGGER == "true" - # Include standard cases configuration when not triggered via GitHub API - - local: 'dev/ci/.gitlab-ci-cases.yml' - rules: - - if: $GITHUB_API_TRIGGER != "true" - -# Base configuration -.base_config: - variables: - GIT_STRATEGY: none - -# Common build template for all modalities -.build_template: - variables: - GIT_STRATEGY: clone - GIT_SUBMODULE_STRATEGY: recursive - stage: build - allow_failure: true - script: - - echo "Using build directory ${HOMEGFS}" - - dev/ci/scripts/utils/ci_utils.sh build_compute - - sorc/link_workflow.sh - - mkdir -p ${RUNTESTS_DIR} - -.setup_template: - extends: .base_config - stage: create_experiments - allow_failure: true - script: - - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml - dependencies: - - build-${machine} - -.run_tests_template: - extends: .base_config - stage: run_tests - allow_failure: true - script: - - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow - dependencies: - - setup_experiments-${machine} - From 03b5bbf9880c5320397bda4bfba863286c267f51 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 18:11:43 -0400 Subject: [PATCH 54/76] added inline documention and a docs README.md in the gitlab ci folder --- dev/ci/.gitlab-ci-cases.yml | 20 +++- dev/ci/.gitlab-ci-ctests.yml | 19 ++++ dev/ci/.gitlab-ci-hosts.yml | 19 ++++ dev/ci/.gitlab-ci.yml | 33 +++++- dev/ci/scripts/utils/gitlab/docs/README.md | 114 +++++++++++++++++++++ 5 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 dev/ci/scripts/utils/gitlab/docs/README.md diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/.gitlab-ci-cases.yml index 2e6d82fada0..4ee8676c0a4 100644 --- a/dev/ci/.gitlab-ci-cases.yml +++ b/dev/ci/.gitlab-ci-cases.yml @@ -1,4 +1,20 @@ -# This file defines jobs to create and run specific to PR cases +# ========================================================================== +# Templates for Standard Test Cases +# ========================================================================== +# +# This file defines templates for setting up and running standard experiment +# test cases for the global-workflow project. These cases represent more +# comprehensive, end-to-end testing scenarios than the CTesting framework. +# +# These templates are used primarily for PR testing when GITHUB_API_TRIGGER=false. +# The actual test case matrix for each host is defined in .gitlab-ci-hosts.yml. +# +# Key templates: +# - .setup_template: Creates experiment directories based on case YAML files +# - .run_tests_template: Executes and validates the experiment workflows +# +# Test cases are defined as YAML files in the dev/ci/cases/pr directory and +# are referenced by name in the host configuration file's test matrix. # Template for experiment setup jobs .setup_template: @@ -19,4 +35,4 @@ - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow needs: - - setup_experiments-${machine} \ No newline at end of file + - setup_experiments-${machine} diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index a30e8f62f96..97ca9297fe1 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -1,3 +1,22 @@ +# ========================================================================== +# Templates and Configuration for CTest-Based Testing +# ========================================================================== +# +# This file defines jobs and templates for running CMake/CTest-based fuctional +# tests in the global-workflow project. The CTests are designed to +# test specific Rocoto Jobs singluarly with predefined input data. +# +# CTests are primarily triggered via GitHub API for quick validation of PRs, +# whereas the full experiment cases are used for more extensive testing. +# +# Key components: +# - .create_ctests: Template for setting up the CMake/CTest environment +# - .run_ctests_template: Template for executing specific CTest labels +# +# CTests are defined in the dev/ctests directory and are categorized by labels, +# which are referenced in the hosts file to determine which tests to run on +# which platforms. + # ======================================= # Templates for CTests # ======================================= diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 4eb9bbecccb..3f48f0881b5 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -1,3 +1,20 @@ +# ========================================================================== +# Host-Specific Configurations for the global-workflow CI Pipeline +# ========================================================================== +# +# This file defines host-specific job configurations for different computing +# platforms supported by the global-workflow testing pipeline. It's designed +# to be easily extendable with new computing platforms. +# +# Key features: +# - Per-host test case matrices that define which tests run on which hosts +# - Separate configurations for standard cases and CTests +# - Host-specific tags and variables for job routing +# - Conditional rules based on trigger type (PR vs API) +# +# The GITHUB_API_TRIGGER variable (set in the main .gitlab-ci.yml) controls +# whether to run CTests (true) or standard cases (false). + # ======================================= # Standard Cases configurations by host # ======================================= @@ -60,6 +77,7 @@ run_tests-gaeac6: rules: - if: $GITHUB_API_TRIGGER != "true" +# ======================================= # CTests configurations by host # ======================================= @@ -119,6 +137,7 @@ run_ctests-gaeac6: # ======================================= # Common build configurations by host # These will always be included, regardless of modality +# ======================================= build-hera: extends: .build_template diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index 85390ec2faa..fa5b34a6003 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -1,5 +1,26 @@ -# Main gitlab-ci.yml for the global-workflow project -# This file defines common stages and includes appropriate modality file based on trigger +# ========================================================================== +# Main GitLab CI Configuration for the global-workflow Project +# ========================================================================== +# +# This is the primary CI/CD configuration file that orchestrates the GitLab CI +# testing framework for the global-workflow project. It defines pipeline stages +# and includes specialized configuration files for different testing aspects. +# +# The pipeline architecture supports: +# 1. Multiple host environments (hera, gaeac6, etc.) +# 2. Two primary testing modes: +# - PR validation via standard test cases +# - CTest-based tests triggered via GitHub API +# +# Included files: +# - .gitlab-ci-ctests.yml: Handles CMake/CTest-based testing framework +# - .gitlab-ci-cases.yml: Defines templates for standard experiment cases +# - .gitlab-ci-hosts.yml: Contains host-specific configurations and test matrices +# that can be extended to support additional hosts +# +# The host configuration in .gitlab-ci-hosts.yml is designed to be easily +# extended with new computing platforms and allows per-host specification +# of which test cases to run. stages: - build @@ -16,13 +37,15 @@ variables: GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + # Controls pipeline behavior: CTests (true) or PR cases (false) GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} +# Include specialized pipeline configuration files include: - - local: 'dev/ci/.gitlab-ci-ctests.yml' - - local: 'dev/ci/.gitlab-ci-cases.yml' - - local: 'dev/ci/.gitlab-ci-hosts.yml' + - local: 'dev/ci/.gitlab-ci-ctests.yml' # CTest framework configuration + - local: 'dev/ci/.gitlab-ci-cases.yml' # Standard test case templates + - local: 'dev/ci/.gitlab-ci-hosts.yml' # Host-specific configurations .base_config: variables: diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md new file mode 100644 index 00000000000..216f5a587f8 --- /dev/null +++ b/dev/ci/scripts/utils/gitlab/docs/README.md @@ -0,0 +1,114 @@ +# GitLab CI Pipeline Configuration + +This documentation describes the multi-modal GitLab CI pipeline configuration for the global-workflow project. + +## Pipeline Modalities + +The CI pipeline supports two different modalities: + +1. **Standard Pipeline** - The default pipeline that runs when triggered by normal GitLab events (push, merge requests) +2. **CTests Pipeline** - A specialized pipeline that runs when triggered via GitLab API from GitHub + +## Multi-Host Support + +Both pipeline modalities support running on multiple compute hosts using a shared configuration system. The pipeline is dynamically generated by the `generate_pipelines.py` script, which creates jobs for each supported host machine based on the test cases that each host supports. + +## Pipeline Structure + +Both modalities use the following stages: +- `build`: Builds the codebase (shared between modalities) +- `create_experiments`: Sets up experiments or ctests +- `run_tests`: Executes the test workflows +- `finalize`: Updates status badges + +## Standard Pipeline + +The standard pipeline when triggered normally: +- Builds the codebase for each machine +- Sets up experiments using configurations in `dev/ci/cases/pr/` +- Runs tests for each case on the appropriate machine +- Completes with final status reporting + +## CTests Pipeline + +The CTests pipeline when triggered via GitLab API from GitHub: +- Uses the same build jobs as the standard pipeline +- The build stage additionally sets up the CMake environment for ctests +- Skips the standard experiment creation +- Runs specialized CTest test cases on each machine + +### Currently Supported CTests + +The pipeline currently supports the following tests: +- `C48_ATM_gfs_fcst_seg0` +- `C48_S2SW_gfs_fcst_seg0` +- `C48_S2SW_gfs_atmos_prod` + +Additional tests will be added in the future from within the ctest framework in `dev/ctests`. + +## Shared Build Stage + +Both modalities share the same build stage, which: +1. Clones the repository +2. Builds the necessary components +3. Sets up the workspace for testing +4. If triggered by GitHub API, also prepares the CMake environment for ctests + +This shared approach ensures consistent build results while optimizing resource usage. + +## Triggering the CTests Pipeline from GitHub + +To trigger the CTests pipeline from GitHub, use the GitLab API with the following parameters: + +```bash +curl -X POST \ + --header "Content-Type: application/json" \ + --header "PRIVATE-TOKEN: " \ + "https://vlab.noaa.gov/gitlab-licensed/NWS/Operations/NCEP/EMC/global-workflow/api/v4/projects//trigger/pipeline" \ + --data '{ + "ref": "", + "variables": { + "GITHUB_API_TRIGGER": "true", + "PR_NUMBER": "", + "GITHUB_REPO_URL": "" + } + }' +``` + +Replace the following: +- ``: Your GitLab API token with appropriate permissions +- ``: The GitLab project ID for global-workflow +- ``: The branch to run tests on +- ``: The GitHub PR number (use "0" for non-PR runs) +- ``: The GitHub repository URL + +## Pipeline Configuration Files + +The configuration is split across multiple files: + +1. **Main Pipeline File** (`.gitlab-ci.yml`): + - Contains the common stages and variables + - Includes the appropriate modality file based on trigger + - Contains the build template shared by both modalities + - Contains the finalize stage job + +2. **Cases Configuration** (`.gitlab-ci-cases.yml`): + - Contains job templates specific to standard test cases + - Includes templates for running experiments + +3. **CTests Configuration** (`.gitlab-ci-ctests.yml`): + - Contains specific configurations for CTests + - Included conditionally when triggered via GitHub API + +## Pipeline Generation Process + +1. The `generate_pipelines.py` script: + - Reads the template file + - Determines which test cases each machine supports + - Generates machine-specific jobs for both standard tests and ctests + - Outputs the complete pipeline configuration + +2. When the pipeline runs: + - GitLab detects if `GITHUB_API_TRIGGER` is set to "true" + - If true, includes the `.gitlab-ci-ctests.yml` file + - Jobs check this variable to determine if they should run From 8c68fd2b4b4219df4d1a33b73d6b32ec7509ba56 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 18:23:20 -0400 Subject: [PATCH 55/76] small refinments in key features for clearity --- dev/ci/.gitlab-ci-hosts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/.gitlab-ci-hosts.yml index 3f48f0881b5..dc4f23c4d57 100644 --- a/dev/ci/.gitlab-ci-hosts.yml +++ b/dev/ci/.gitlab-ci-hosts.yml @@ -8,8 +8,8 @@ # # Key features: # - Per-host test case matrices that define which tests run on which hosts -# - Separate configurations for standard cases and CTests -# - Host-specific tags and variables for job routing +# - Includes two sections supporting the standard PR Cases and CTests +# - Host-specific tags and variables for job routing to the correct runners # - Conditional rules based on trigger type (PR vs API) # # The GITHUB_API_TRIGGER variable (set in the main .gitlab-ci.yml) controls From d48f907a6cf7f8c0948d7b135d0a305a57917d6c Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 19:08:16 -0400 Subject: [PATCH 56/76] removed refence to script generator with is no longer used at this time --- dev/ci/scripts/utils/gitlab/docs/README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md index 216f5a587f8..d4ac3c98706 100644 --- a/dev/ci/scripts/utils/gitlab/docs/README.md +++ b/dev/ci/scripts/utils/gitlab/docs/README.md @@ -100,14 +100,6 @@ The configuration is split across multiple files: - Contains specific configurations for CTests - Included conditionally when triggered via GitHub API -## Pipeline Generation Process - -1. The `generate_pipelines.py` script: - - Reads the template file - - Determines which test cases each machine supports - - Generates machine-specific jobs for both standard tests and ctests - - Outputs the complete pipeline configuration - 2. When the pipeline runs: - GitLab detects if `GITHUB_API_TRIGGER` is set to "true" - If true, includes the `.gitlab-ci-ctests.yml` file From be7a8579f6c4712ee92afd0704657ddc83f52181 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 22 Apr 2025 13:01:33 -0400 Subject: [PATCH 57/76] Update dev/ci/.gitlab-ci-ctests.yml Co-authored-by: Rahul Mahajan --- dev/ci/.gitlab-ci-ctests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/.gitlab-ci-ctests.yml index 97ca9297fe1..86ea771fbb5 100644 --- a/dev/ci/.gitlab-ci-ctests.yml +++ b/dev/ci/.gitlab-ci-ctests.yml @@ -31,7 +31,7 @@ - cd ${HOMEGFS}/dev/ctests - mkdir -p build - cd build - - cmake ../../.. + - cmake -S ${HOMEGFS} - ctest -N - num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") - '[ "$num" -gt 0 ] || { echo "No tests found"; exit 1; }' From d3ab98dbcc317bc741298c653019ba8e58df4d9e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 22 Apr 2025 13:27:03 -0400 Subject: [PATCH 58/76] Update dev/ci/scripts/utils/find_homegfs.py Co-authored-by: Rahul Mahajan --- dev/ci/scripts/utils/find_homegfs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/ci/scripts/utils/find_homegfs.py b/dev/ci/scripts/utils/find_homegfs.py index b2b2ac1adfc..f97f896d6b9 100755 --- a/dev/ci/scripts/utils/find_homegfs.py +++ b/dev/ci/scripts/utils/find_homegfs.py @@ -32,9 +32,7 @@ def find_homegfs(start_path=None): # If start_path is not provided, use the directory of the calling script if start_path is None: # Get the path of the calling script - import inspect - frame = inspect.currentframe().f_back - start_path = os.path.dirname(os.path.abspath(frame.f_code.co_filename)) + start_path = os.getcwd() # Convert to Path object if it's a string if isinstance(start_path, str): From 79236f0e52929188987b8f80fcbad4cf889b1579 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 22 Apr 2025 13:35:59 -0400 Subject: [PATCH 59/76] Update dev/ci/.gitlab-ci.yml Co-authored-by: Rahul Mahajan --- dev/ci/.gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index fa5b34a6003..ede6cf160eb 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -58,7 +58,8 @@ include: GIT_SUBMODULE_STRATEGY: recursive stage: build script: - - echo "Using build directory ${HOMEGFS}" - - dev/ci/scripts/utils/ci_utils.sh build_compute - - sorc/link_workflow.sh - - mkdir -p ${RUNTESTS_DIR} + - | + echo "Using build directory ${HOMEGFS}" + dev/ci/scripts/utils/ci_utils.sh build_compute + sorc/link_workflow.sh + mkdir -p ${RUNTESTS_DIR} From dbb8a6e52b305c551b651e5745ad2c2454a25d0d Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 22 Apr 2025 13:56:52 -0400 Subject: [PATCH 60/76] applied review changes --- dev/ci/.gitlab-ci.yml | 8 ++++---- dev/ci/Jenkinsfile | 2 +- dev/ci/Jenkinsfile4AWS | 2 +- dev/ci/{.gitlab-ci-cases.yml => gitlab-ci-cases.yml} | 0 dev/ci/{.gitlab-ci-ctests.yml => gitlab-ci-ctests.yml} | 0 dev/ci/{.gitlab-ci-hosts.yml => gitlat-ci-hosts.yml} | 0 dev/ci/scripts/utils/ci_utils.sh | 2 +- dev/ci/scripts/utils/parse_yaml.py | 1 - 8 files changed, 7 insertions(+), 8 deletions(-) rename dev/ci/{.gitlab-ci-cases.yml => gitlab-ci-cases.yml} (100%) rename dev/ci/{.gitlab-ci-ctests.yml => gitlab-ci-ctests.yml} (100%) rename dev/ci/{.gitlab-ci-hosts.yml => gitlat-ci-hosts.yml} (100%) diff --git a/dev/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml index ede6cf160eb..10335ec2f11 100644 --- a/dev/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -43,9 +43,9 @@ variables: # Include specialized pipeline configuration files include: - - local: 'dev/ci/.gitlab-ci-ctests.yml' # CTest framework configuration - - local: 'dev/ci/.gitlab-ci-cases.yml' # Standard test case templates - - local: 'dev/ci/.gitlab-ci-hosts.yml' # Host-specific configurations + - local: 'dev/ci/gitlab-ci-ctests.yml' # CTest framework configuration + - local: 'dev/ci/gitlab-ci-cases.yml' # Standard test case templates + - local: 'dev/ci/gitlab-ci-hosts.yml' # Host-specific configurations .base_config: variables: @@ -60,6 +60,6 @@ include: script: - | echo "Using build directory ${HOMEGFS}" - dev/ci/scripts/utils/ci_utils.sh build_compute + dev/ci/scripts/utils/ci_utils.sh build sorc/link_workflow.sh mkdir -p ${RUNTESTS_DIR} diff --git a/dev/ci/Jenkinsfile b/dev/ci/Jenkinsfile index 9f1c4e2c452..5bd5f555f02 100644 --- a/dev/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables + sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh build") // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" if ( fileExists("logs/error.logs") ) { diff --git a/dev/ci/Jenkinsfile4AWS b/dev/ci/Jenkinsfile4AWS index 91220e83ef6..d5b31065f24 100644 --- a/dev/ci/Jenkinsfile4AWS +++ b/dev/ci/Jenkinsfile4AWS @@ -134,7 +134,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh build_compute") // build the global-workflow executables + // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh build") // build the global-workflow executables sh(script: './build_compute.sh -A ${USER} gfs gefs sfs') // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" diff --git a/dev/ci/.gitlab-ci-cases.yml b/dev/ci/gitlab-ci-cases.yml similarity index 100% rename from dev/ci/.gitlab-ci-cases.yml rename to dev/ci/gitlab-ci-cases.yml diff --git a/dev/ci/.gitlab-ci-ctests.yml b/dev/ci/gitlab-ci-ctests.yml similarity index 100% rename from dev/ci/.gitlab-ci-ctests.yml rename to dev/ci/gitlab-ci-ctests.yml diff --git a/dev/ci/.gitlab-ci-hosts.yml b/dev/ci/gitlat-ci-hosts.yml similarity index 100% rename from dev/ci/.gitlab-ci-hosts.yml rename to dev/ci/gitlat-ci-hosts.yml diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index 3d4e71e8ed6..b8c683e8fb3 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -196,7 +196,7 @@ function cleanup_experiment() { rm -Rf "${STMP}/RUNDIRS/${pslot:?}" } -function build_compute () { +function build () { source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" # TODO: when it's safe to build on C6 compute nodes again, do so diff --git a/dev/ci/scripts/utils/parse_yaml.py b/dev/ci/scripts/utils/parse_yaml.py index fdf6e60c8ae..98fa4c66f10 100755 --- a/dev/ci/scripts/utils/parse_yaml.py +++ b/dev/ci/scripts/utils/parse_yaml.py @@ -42,7 +42,6 @@ def yq(yamlfile, key): """ HOMEgfs = find_homegfs() - data = AttrDict(HOMEgfs=HOMEgfs) data.update({'HOMEgfs': HOMEgfs}) ydict = parse_j2yaml(path=yamlfile, data=data) if key == 'all': From 67c5d294d03409af541db08a538d69148394a57f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 22 Apr 2025 15:19:28 -0400 Subject: [PATCH 61/76] added steps to make sure failures are captured to pipeline with ctests fail and imporived readablity by removing single seperated bash lines --- dev/ci/gitlab-ci-ctests.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/dev/ci/gitlab-ci-ctests.yml b/dev/ci/gitlab-ci-ctests.yml index 86ea771fbb5..56ba02ed83b 100644 --- a/dev/ci/gitlab-ci-ctests.yml +++ b/dev/ci/gitlab-ci-ctests.yml @@ -25,17 +25,18 @@ .create_ctests: extends: .base_config stage: create_experiments - script: - - echo "Setting up for ctests workflow" - - source ${HOMEGFS}/dev/ci/platforms/config.${machine} - - cd ${HOMEGFS}/dev/ctests - - mkdir -p build - - cd build - - cmake -S ${HOMEGFS} - - ctest -N - - num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") - - '[ "$num" -gt 0 ] || { echo "No tests found"; exit 1; }' - - echo "CTests have been created and configured successfully" + script: | + set -e # Fail the job if any command fails + echo "Setting up for ctests workflow" + source ${HOMEGFS}/dev/ci/platforms/config.${machine} + cd ${HOMEGFS}/dev/ctests + mkdir -p build + cd build + cmake -S ${HOMEGFS} + ctest -N + num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") + [ "$num" -gt 0 ] || { echo "No tests found"; exit 1; } + echo "CTests have been created and configured successfully" needs: - build-${machine} @@ -43,9 +44,10 @@ .run_ctests_template: extends: .base_config stage: run_tests - script: - - echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" - - cd ${HOMEGFS}/dev/ctests/build - - ctest -L ${CTEST} + script: | + set -e # Fail the job if any command fails + echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" + cd ${HOMEGFS}/dev/ctests/build + ctest -L ${CTEST} --output-on-failure needs: - create_ctests-${machine} From 07335ead994338edcdbc00f5c5c5ada459a09ece Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 22 Apr 2025 16:50:02 -0400 Subject: [PATCH 62/76] Replaced HOMEGFS_ with HOMEgfs_ --- dev/ci/scripts/utils/ci_utils.sh | 36 +++++++++---------- .../utils/gitlab/launch_gitlab_runner.sh | 8 ++--- dev/ci/scripts/utils/launch_java_agent.sh | 14 ++++---- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/dev/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh index b8c683e8fb3..01b83b66a92 100755 --- a/dev/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -# Determine HOMEGFS_ and source machine detection early -if [[ -z "${HOMEGFS_}" ]]; then +# Determine HOMEgfs_ and source machine detection early +if [[ -z "${HOMEgfs_}" ]]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - HOMEGFS_="$("${SCRIPT_DIR}/find_homegfs.py")" + HOMEgfs_="$("${SCRIPT_DIR}/find_homegfs.py")" fi -source "${HOMEGFS_}/ush/detect_machine.sh" +source "${HOMEgfs_}/ush/detect_machine.sh" # --- Existing functions --- @@ -70,7 +70,7 @@ function get_pr_case_list () { # loop over every yaml file in the PR's ci/cases # and create an run directory for each one for this PR loop ############################################################# - for yaml_config in "${HOMEGFS_}/dev/ci/cases/pr/"*.yaml; do + for yaml_config in "${HOMEgfs_}/dev/ci/cases/pr/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true echo "${case}" done @@ -123,29 +123,29 @@ function cancel_all_batch_jobs () { function create_experiment () { local yaml_config="${1}" - cd "${HOMEGFS_}" || exit 1 + cd "${HOMEgfs_}" || exit 1 pr_sha=$(git rev-parse --short HEAD) case=$(basename "${yaml_config}" .yaml) || true export pslot=${case}_${pr_sha} if [[ ${MACHINE_ID} == "noaacloud" ]]; then - source "${HOMEGFS_}/dev/ci/platforms/config.${PW_CSP}" + source "${HOMEgfs_}/dev/ci/platforms/config.${PW_CSP}" else - source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs_}/dev/ci/platforms/config.${MACHINE_ID}" fi - source "${HOMEGFS_}/dev/ush/gw_setup.sh" + source "${HOMEgfs_}/dev/ush/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) if [[ ${MACHINE_ID} == "noaacloud" ]]; then - STMP=$("${HOMEGFS_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEGFS_}/dev/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) + STMP=$("${HOMEgfs_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs_}/dev/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) else - STMP=$("${HOMEGFS_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEGFS_}/dev/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) + STMP=$("${HOMEgfs_}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs_}/dev/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) fi echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" - "${HOMEGFS_}/${system}/dev/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" + "${HOMEgfs_}/${system}/dev/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" } @@ -169,8 +169,8 @@ function publish_logs() { if [[ -n "${full_paths}" ]]; then # shellcheck disable=SC2027,SC2086 - ${HOMEGFS_}/dev/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null - URL="$("${HOMEGFS_}/dev/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" + ${HOMEgfs_}/dev/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null + URL="$("${HOMEgfs_}/dev/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" fi echo "${URL}" } @@ -187,7 +187,7 @@ function cleanup_experiment() { pslot=$(basename "${EXPDIR}") # Use the Python utility to get the required variables - read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEGFS_}/dev/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true + read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs_}/dev/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" @@ -198,12 +198,12 @@ function cleanup_experiment() { function build () { - source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs_}/dev/ci/platforms/config.${MACHINE_ID}" # TODO: when it's safe to build on C6 compute nodes again, do so if [[ "${MACHINE_ID}" == "gaeac6" ]]; then - "${HOMEGFS_}/sorc/build_all.sh" -v -k all + "${HOMEgfs_}/sorc/build_all.sh" -v -k all else - "${HOMEGFS_}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all + "${HOMEgfs_}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all fi } diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index cf1d9199e38..f175fb6e623 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -13,9 +13,9 @@ set -e # Usage: ./launch_gitlab_runner.sh [register|run|unregister] [token] ######################################################################### -# Set the HOMEGFS_ variable to the root directory of the global workflow +# Set the HOMEgfs_ variable to the root directory of the global workflow SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -HOMEGFS_="$("${SCRIPT_DIR}/../find_homegfs.py")" +HOMEgfs_="$("${SCRIPT_DIR}/../find_homegfs.py")" # Get the hostname of the current machine host="$(hostname)" @@ -24,7 +24,7 @@ host="$(hostname)" ######################################################################### # Source the detect_machine.sh script to determine the MACHINE_ID -source "${HOMEGFS_}/ush/detect_machine.sh" +source "${HOMEgfs_}/ush/detect_machine.sh" # Check the MACHINE_ID and set up the environment accordingly case "${MACHINE_ID}" in hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 ) @@ -39,7 +39,7 @@ esac # Source the platform-specific configuration file # This file contains platform-specific variables such as GITLAB_URL, GITLAB_CI_BUILDS_DIR, # and GITLAB_RUNNER_DIR which are required for runner registration and execution -source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" +source "${HOMEgfs_}/dev/ci/platforms/config.${MACHINE_ID}" # Change to the GitLab runner directory defined in the platform config cd "${GITLAB_RUNNER_DIR}" || exit 1 diff --git a/dev/ci/scripts/utils/launch_java_agent.sh b/dev/ci/scripts/utils/launch_java_agent.sh index 3ae686b7f65..8727533bf6a 100755 --- a/dev/ci/scripts/utils/launch_java_agent.sh +++ b/dev/ci/scripts/utils/launch_java_agent.sh @@ -65,16 +65,16 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -# Set the HOMEGFS_ variable to the root directory of the global workflow +# Set the HOMEgfs_ variable to the root directory of the global workflow SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -HOMEGFS_="$("${SCRIPT_DIR}/find_homegfs.py")" +HOMEgfs_="$("${SCRIPT_DIR}/find_homegfs.py")" host=$(hostname) ######################################################################### # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${HOMEGFS_}/ush/detect_machine.sh" +source "${HOMEgfs_}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 ) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; @@ -88,14 +88,14 @@ esac LOG=launched_agent-$(date +%Y%m%d%M).log rm -f "${LOG}" -HOMEgfs="${HOMEGFS_}" source "${HOMEGFS_}/ush/module-setup.sh" -module use "${HOMEGFS_}/modulefiles" +HOMEgfs="${HOMEgfs_}" source "${HOMEgfs_}/ush/module-setup.sh" +module use "${HOMEgfs_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" if [[ ${MACHINE_ID} == "noaacloud" ]]; then - source "${HOMEGFS_}/dev/ci/platforms/config.${PW_CSP}" + source "${HOMEgfs_}/dev/ci/platforms/config.${PW_CSP}" else - source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs_}/dev/ci/platforms/config.${MACHINE_ID}" fi JAVA_HOME="${JENKINS_AGENT_LAUNCH_DIR}/JAVA/jdk-17.0.10" From 48cfe3bcd9c131340204fb6bb0f61e31c20c27ed Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 22 Apr 2025 16:51:33 -0400 Subject: [PATCH 63/76] moved .gitlab-ci.yml to root default --- dev/ci/.gitlab-ci.yml => .gitlab-ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev/ci/.gitlab-ci.yml => .gitlab-ci.yml (100%) diff --git a/dev/ci/.gitlab-ci.yml b/.gitlab-ci.yml similarity index 100% rename from dev/ci/.gitlab-ci.yml rename to .gitlab-ci.yml From 2dd01176a2dde8997814fb6f063a1342e20d4dd5 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Tue, 8 Apr 2025 19:11:32 -0400 Subject: [PATCH 64/76] added NOTE about the supported ctests neeeding updates to there data sets --- dev/ci/scripts/utils/gitlab/docs/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/ci/scripts/utils/gitlab/docs/README.md b/dev/ci/scripts/utils/gitlab/docs/README.md index d4ac3c98706..70941433382 100644 --- a/dev/ci/scripts/utils/gitlab/docs/README.md +++ b/dev/ci/scripts/utils/gitlab/docs/README.md @@ -37,13 +37,15 @@ The CTests pipeline when triggered via GitLab API from GitHub: - Skips the standard experiment creation - Runs specialized CTest test cases on each machine -### Currently Supported CTests +### Currently Supported Functional CTests The pipeline currently supports the following tests: - `C48_ATM_gfs_fcst_seg0` - `C48_S2SW_gfs_fcst_seg0` - `C48_S2SW_gfs_atmos_prod` +NOTE: The input/output data needes to be updated on all these + Additional tests will be added in the future from within the ctest framework in `dev/ctests`. ## Shared Build Stage From 6363dd9a3ce19edc66bc484d24463827442f0d3e Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Wed, 9 Apr 2025 04:46:07 -0400 Subject: [PATCH 65/76] a misspelling of the include host supporting pipeline got in the repo and is rectified in this push --- dev/ci/{gitlat-ci-hosts.yml => gitlab-ci-hosts.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev/ci/{gitlat-ci-hosts.yml => gitlab-ci-hosts.yml} (100%) diff --git a/dev/ci/gitlat-ci-hosts.yml b/dev/ci/gitlab-ci-hosts.yml similarity index 100% rename from dev/ci/gitlat-ci-hosts.yml rename to dev/ci/gitlab-ci-hosts.yml From 469e6e67d807487d5310c3165f37f1877fd6139f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 23 Apr 2025 21:22:37 -0400 Subject: [PATCH 66/76] did some re-naming of variables to relax the confution between GitLab predifined vairbles the workflows (got a little verbose at times that we can pare down latter --- .gitlab-ci.yml | 16 ++++++++++------ dev/ci/platforms/config.hera | 19 +++++++++++-------- .../utils/gitlab/launch_gitlab_runner.sh | 6 +++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10335ec2f11..572353d8f22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,13 +30,17 @@ stages: variables: BUILD: '' - CI_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} - GIT_CLONE_PATH: '${CI_RUN_PATH}/global-workflow' + # Using GW prefix to avoid confusion with GitLab predefined variables + GW_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} + # Overriding GitLab's predefined GIT_CLONE_PATH variable to specify the exact location + # where the repository should be cloned. This works with custom_build_dir-enabled in the runner config. + GIT_CLONE_PATH: '${GW_RUN_PATH}/global-workflow' HOMEGFS: ${GIT_CLONE_PATH} - RUNTESTS_DIR: ${CI_RUN_PATH}/RUNTESTS - GIT_DEPTH: 1 - RUNNER_SCRIPT_TIMEOUT: 6h - RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS + GIT_DEPTH: 1 # GitLab predefined variable that limits git history to improve clone performance + # Using GW prefix to avoid confusion with GitLab runner variables + GW_SCRIPT_TIMEOUT: 6h + GW_AFTER_SCRIPT_TIMEOUT: 6h # Controls pipeline behavior: CTests (true) or PR cases (false) GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} diff --git a/dev/ci/platforms/config.hera b/dev/ci/platforms/config.hera index ede02d9a995..2c66e927170 100644 --- a/dev/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -8,7 +8,7 @@ ######################################################################### # Main CI root directory - Base directory for all CI/CD operations -export GFS_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD +export GW_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD # ICSDIR root directory - Contains initial condition data # Used by create_experiment.py for setting up test cases @@ -20,11 +20,11 @@ export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR ######################################################################### # JENKINS launch directory for agent - Where Jenkins agents are launched from -export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent +export JENKINS_AGENT_LAUNCH_DIR=${GW_CI_ROOT}/Jenkins/agent # JENKINS internal working directories for CI jobs (not for users use) # Where Jenkins stores temporary files during CI job execution -export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace +export JENKINS_WORK_DIR=${GW_CI_ROOT}/Jenkins/workspace # NOTE: JENKINS custom_workspace directory where CI jobs are run # and is defined in $HOMEgfs/dev/ci/Jenkinsfile as custom_workspace @@ -41,19 +41,22 @@ export GITLAB_URL=https://vlab.noaa.gov/gitlab-licensed export GITLAB_RUNNER_NAME="RDHPCS Hera" # Directory for GitLab builds -# Used in launch_gitlab_runner.sh for location of builds -export GITLAB_CI_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB +# Used in launch_gitlab_runner.sh for the --builds-dir parameter +# This works with --custom_build_dir-enabled=true to allow GIT_CLONE_PATH override +# from .gitlab-ci.yml to specify exact clone locations within this directory +export GW_BUILDS_DIR=${GW_CI_ROOT}/BUILDS/GITLAB # Directory for GitLab runner used by launch_gitlab_runner.sh -export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" +# This is where runner state/config files are stored (--working-directory parameter) +export GW_RUNNER_DIR="${GW_CI_ROOT}/GitLab/Runner" # CTest functional test directories for pre stagged input data -export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR +export STAGED_TESTS_DIR=${GW_CI_ROOT}/STAGED_TESTS_DIR ######################################################################### # CI CRON system configuration ######################################################################### -export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI +export GW_BASH_CI_ROOT=${GW_CI_ROOT}/GFS_BASH_CI export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index f175fb6e623..25ee9051362 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -92,7 +92,7 @@ if [[ "${1}" == "register" ]]; then # --builds-dir: Directory where builds will be stored (from config.MACHINE_ID) # --custom_build_dir-enabled: Enable custom build directories # --request-concurrency: Number of concurrent requests that can be handled - ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GITLAB_CI_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 + ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GW_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 # Set the concurrent job limit in the GitLab runner config file sed -i 's/concurrent.*/concurrent = 24/' ~/.gitlab-runner/config.toml @@ -105,8 +105,8 @@ fi if [[ "${1}" == "run" ]]; then # --working-directory: Directory where the runner is launched and keeps its working files (from config.$MACHINE_ID) - # do not confuse this with GitLabs CI_BUILDS_DIR which is designate by GFS_CI_BUILDS_DIR and is where the builds are stored - COMMAND="nohup ./gitlab-runner run --working-directory ${GITLAB_RUNNER_DIR}" + # do not confuse this with GitLab's CI_BUILDS_DIR which is designated by GW_BUILDS_DIR and is where the builds are stored + COMMAND="nohup ./gitlab-runner run --working-directory ${GW_RUNNER_DIR}" echo -e "Running gitlab-runner with the command:\n${COMMAND}\nsee log ${GITLAB_LOG}" echo -e "Running gitlab-runner with the command:${COMMAND}" >& "${GITLAB_LOG}" ${COMMAND} >> "${GITLAB_LOG}" 2>&1 & From 28544e56711c026894f2dc1178093ff4fab05d2b Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 23 Apr 2025 21:57:07 -0400 Subject: [PATCH 67/76] changed HOMEGFS to GW_HOMEgfs --- .gitlab-ci.yml | 4 ++-- dev/ci/gitlab-ci-cases.yml | 8 ++++---- dev/ci/gitlab-ci-ctests.yml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 572353d8f22..ad3a83e70c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ variables: # Overriding GitLab's predefined GIT_CLONE_PATH variable to specify the exact location # where the repository should be cloned. This works with custom_build_dir-enabled in the runner config. GIT_CLONE_PATH: '${GW_RUN_PATH}/global-workflow' - HOMEGFS: ${GIT_CLONE_PATH} + GW_HOMEgfs: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS GIT_DEPTH: 1 # GitLab predefined variable that limits git history to improve clone performance # Using GW prefix to avoid confusion with GitLab runner variables @@ -63,7 +63,7 @@ include: stage: build script: - | - echo "Using build directory ${HOMEGFS}" + echo "Using build directory ${GW_HOMEgfs}" dev/ci/scripts/utils/ci_utils.sh build sorc/link_workflow.sh mkdir -p ${RUNTESTS_DIR} diff --git a/dev/ci/gitlab-ci-cases.yml b/dev/ci/gitlab-ci-cases.yml index 4ee8676c0a4..d830f5f0541 100644 --- a/dev/ci/gitlab-ci-cases.yml +++ b/dev/ci/gitlab-ci-cases.yml @@ -22,7 +22,7 @@ stage: create_experiments script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml + - ${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${GW_HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml needs: - build-${machine} @@ -31,8 +31,8 @@ extends: .base_config stage: run_tests script: - - echo "Using build directory $HOMEGFS (dated $BUILD_DATE)" - - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + - echo "Using build directory $GW_HOMEgfs (dated $BUILD_DATE)" + - pslot=$(${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow needs: - setup_experiments-${machine} diff --git a/dev/ci/gitlab-ci-ctests.yml b/dev/ci/gitlab-ci-ctests.yml index 56ba02ed83b..38a17e3cfab 100644 --- a/dev/ci/gitlab-ci-ctests.yml +++ b/dev/ci/gitlab-ci-ctests.yml @@ -28,11 +28,11 @@ script: | set -e # Fail the job if any command fails echo "Setting up for ctests workflow" - source ${HOMEGFS}/dev/ci/platforms/config.${machine} - cd ${HOMEGFS}/dev/ctests + source ${GW_HOMEgfs}/dev/ci/platforms/config.${machine} + cd ${GW_HOMEgfs}/dev/ctests mkdir -p build cd build - cmake -S ${HOMEGFS} + cmake -S ${GW_HOMEgfs} ctest -N num=$(ctest -N | grep "Total Tests" | awk "{print \$3}") [ "$num" -gt 0 ] || { echo "No tests found"; exit 1; } @@ -46,8 +46,8 @@ stage: run_tests script: | set -e # Fail the job if any command fails - echo "Running ${CTEST} tests in ${HOMEGFS}/dev/ctests" - cd ${HOMEGFS}/dev/ctests/build + echo "Running ${CTEST} tests in ${GW_HOMEgfs}/dev/ctests" + cd ${GW_HOMEgfs}/dev/ctests/build ctest -L ${CTEST} --output-on-failure needs: - create_ctests-${machine} From 12b495630ddcebbfe20656d8c207bbd8f16022e4 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 23 Apr 2025 22:38:42 -0400 Subject: [PATCH 68/76] updated to corrrect RUNNER vars for script times and update config vars to laucher --- .gitlab-ci.yml | 5 ++--- dev/ci/platforms/config.gaeac6 | 2 +- dev/ci/platforms/config.hera | 10 +++++----- dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad3a83e70c0..35bb079b08a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,9 +38,8 @@ variables: GW_HOMEgfs: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS GIT_DEPTH: 1 # GitLab predefined variable that limits git history to improve clone performance - # Using GW prefix to avoid confusion with GitLab runner variables - GW_SCRIPT_TIMEOUT: 6h - GW_AFTER_SCRIPT_TIMEOUT: 6h + RUNNER_SCRIPT_TIMEOUT: 6h + RUNNER_AFTER_SCRIPT_TIMEOUT: 6h # Controls pipeline behavior: CTests (true) or PR cases (false) GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"} PR_NUMBER: ${PR_NUMBER:-"0"} diff --git a/dev/ci/platforms/config.gaeac6 b/dev/ci/platforms/config.gaeac6 index f32c7e5627e..08883e36782 100644 --- a/dev/ci/platforms/config.gaeac6 +++ b/dev/ci/platforms/config.gaeac6 @@ -41,7 +41,7 @@ export GITLAB_RUNNER_NAME="RDHPCS Gaea C6" # Directory for GitLab builds # Used in launch_gitlab_runner.sh for location of builds -export GITLAB_CI_BUILDS_DIR=/gpfs/f6/drsa-precip3/world-shared/global/CI/GITLAB +export GITLAB_BUILDS_DIR=/gpfs/f6/drsa-precip3/world-shared/global/CI/GITLAB # Directory for GitLab runner used by launch_gitlab_runner.sh export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" diff --git a/dev/ci/platforms/config.hera b/dev/ci/platforms/config.hera index 2c66e927170..339a11900e1 100644 --- a/dev/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -8,7 +8,7 @@ ######################################################################### # Main CI root directory - Base directory for all CI/CD operations -export GW_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD +export GFS_CI_ROOT=/scratch1/NCEPDEV/global/glopara/GFS_CI_CD # ICSDIR root directory - Contains initial condition data # Used by create_experiment.py for setting up test cases @@ -20,11 +20,11 @@ export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR ######################################################################### # JENKINS launch directory for agent - Where Jenkins agents are launched from -export JENKINS_AGENT_LAUNCH_DIR=${GW_CI_ROOT}/Jenkins/agent +export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent # JENKINS internal working directories for CI jobs (not for users use) # Where Jenkins stores temporary files during CI job execution -export JENKINS_WORK_DIR=${GW_CI_ROOT}/Jenkins/workspace +export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace # NOTE: JENKINS custom_workspace directory where CI jobs are run # and is defined in $HOMEgfs/dev/ci/Jenkinsfile as custom_workspace @@ -44,10 +44,10 @@ export GITLAB_RUNNER_NAME="RDHPCS Hera" # Used in launch_gitlab_runner.sh for the --builds-dir parameter # This works with --custom_build_dir-enabled=true to allow GIT_CLONE_PATH override # from .gitlab-ci.yml to specify exact clone locations within this directory -export GW_BUILDS_DIR=${GW_CI_ROOT}/BUILDS/GITLAB +export GITLAB_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB # Directory for GitLab runner used by launch_gitlab_runner.sh # This is where runner state/config files are stored (--working-directory parameter) -export GW_RUNNER_DIR="${GW_CI_ROOT}/GitLab/Runner" +export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" # CTest functional test directories for pre stagged input data diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index 25ee9051362..3883d83cdda 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -92,7 +92,7 @@ if [[ "${1}" == "register" ]]; then # --builds-dir: Directory where builds will be stored (from config.MACHINE_ID) # --custom_build_dir-enabled: Enable custom build directories # --request-concurrency: Number of concurrent requests that can be handled - ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GW_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 + ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GITLAB_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 # Set the concurrent job limit in the GitLab runner config file sed -i 's/concurrent.*/concurrent = 24/' ~/.gitlab-runner/config.toml From 4ac928553adb50d9b8e443ad5243935132f43787 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 23 Apr 2025 22:42:34 -0400 Subject: [PATCH 69/76] missed one update name in echo part of gitlab launcher for RUNNER dir --- dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh index 3883d83cdda..5a4224b5793 100755 --- a/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/gitlab/launch_gitlab_runner.sh @@ -106,7 +106,7 @@ fi if [[ "${1}" == "run" ]]; then # --working-directory: Directory where the runner is launched and keeps its working files (from config.$MACHINE_ID) # do not confuse this with GitLab's CI_BUILDS_DIR which is designated by GW_BUILDS_DIR and is where the builds are stored - COMMAND="nohup ./gitlab-runner run --working-directory ${GW_RUNNER_DIR}" + COMMAND="nohup ./gitlab-runner run --working-directory ${GITLAB_RUNNER_DIR}" echo -e "Running gitlab-runner with the command:\n${COMMAND}\nsee log ${GITLAB_LOG}" echo -e "Running gitlab-runner with the command:${COMMAND}" >& "${GITLAB_LOG}" ${COMMAND} >> "${GITLAB_LOG}" 2>&1 & From ab8d12b1075da8c8392b3684d2191e19bbaaca00 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Wed, 9 Apr 2025 08:12:29 -0400 Subject: [PATCH 70/76] debugging submodule checkout --- .gitlab-ci.yml | 6 +++++- dev/ci/platforms/config.hera | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35bb079b08a..e1172186f10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,7 @@ variables: GIT_CLONE_PATH: '${GW_RUN_PATH}/global-workflow' GW_HOMEgfs: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS - GIT_DEPTH: 1 # GitLab predefined variable that limits git history to improve clone performance + GIT_DEPTH: 10 # Increased to ensure submodule commits are available RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h # Controls pipeline behavior: CTests (true) or PR cases (false) @@ -59,10 +59,14 @@ include: variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive + GIT_SSL_NO_VERIFY: "true" # Address potential certificate verification issues stage: build script: - | echo "Using build directory ${GW_HOMEgfs}" + # Add diagnostic output for submodules before build + echo "Checking submodule status:" + git submodule status || echo "Continuing despite submodule status check issues" dev/ci/scripts/utils/ci_utils.sh build sorc/link_workflow.sh mkdir -p ${RUNTESTS_DIR} diff --git a/dev/ci/platforms/config.hera b/dev/ci/platforms/config.hera index 339a11900e1..c457f875b15 100644 --- a/dev/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -51,12 +51,12 @@ export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" # CTest functional test directories for pre stagged input data -export STAGED_TESTS_DIR=${GW_CI_ROOT}/STAGED_TESTS_DIR +export CTESTS_STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR ######################################################################### # CI CRON system configuration ######################################################################### -export GW_BASH_CI_ROOT=${GW_CI_ROOT}/GFS_BASH_CI +export GW_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI export max_concurrent_cases=5 export max_concurrent_pr=4 From 5752f177942935dee36af9d179378a5a9db4814f Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Wed, 9 Apr 2025 09:58:08 -0400 Subject: [PATCH 71/76] adding some debug outpouts to build and try to proporgating error to job in GitLab --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1172186f10..3e0375c69da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,10 +63,22 @@ include: stage: build script: - | + set -e # Fail the job if any command fails + echo "Setting up build environment for ${machine}" echo "Using build directory ${GW_HOMEgfs}" # Add diagnostic output for submodules before build echo "Checking submodule status:" git submodule status || echo "Continuing despite submodule status check issues" + + # Run build script and explicitly capture exit code dev/ci/scripts/utils/ci_utils.sh build + build_status=$? + + # Fail immediately if build fails + if [ $build_status -ne 0 ]; then + echo "Build failed with exit code $build_status" + exit $build_status + fi + sorc/link_workflow.sh mkdir -p ${RUNTESTS_DIR} From 56327808555ac4558194ae6152bcf8688fb3837d Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Wed, 9 Apr 2025 11:24:30 -0400 Subject: [PATCH 72/76] fixed bug with renaming GW_RUN_PATH in cases pipline --- dev/ci/gitlab-ci-cases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ci/gitlab-ci-cases.yml b/dev/ci/gitlab-ci-cases.yml index d830f5f0541..232a3f96e2b 100644 --- a/dev/ci/gitlab-ci-cases.yml +++ b/dev/ci/gitlab-ci-cases.yml @@ -33,6 +33,6 @@ script: - echo "Using build directory $GW_HOMEgfs (dated $BUILD_DATE)" - pslot=$(${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${CI_RUN_PATH} ${pslot} global-workflow + - ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${GW_RUN_PATH} ${pslot} global-workflow needs: - setup_experiments-${machine} From 28431aeae8d3dd4e55cdce0bb28e975c60a11927 Mon Sep 17 00:00:00 2001 From: Terry McGUinness Date: Wed, 9 Apr 2025 12:18:13 -0400 Subject: [PATCH 73/76] made script blocks more readble and added explicit exit to catch error and prop to jobs in GitLab --- .gitlab-ci.yml | 10 +++------- dev/ci/gitlab-ci-cases.yml | 14 +++++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e0375c69da..a08bbd63ddb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ stages: - finalize variables: - BUILD: '' + BUILD: '' # Placeholder for control over build directory path # Using GW prefix to avoid confusion with GitLab predefined variables GW_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA} # Overriding GitLab's predefined GIT_CLONE_PATH variable to specify the exact location @@ -37,7 +37,7 @@ variables: GIT_CLONE_PATH: '${GW_RUN_PATH}/global-workflow' GW_HOMEgfs: ${GIT_CLONE_PATH} RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS - GIT_DEPTH: 10 # Increased to ensure submodule commits are available + GIT_DEPTH: 10 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h # Controls pipeline behavior: CTests (true) or PR cases (false) @@ -66,15 +66,11 @@ include: set -e # Fail the job if any command fails echo "Setting up build environment for ${machine}" echo "Using build directory ${GW_HOMEgfs}" - # Add diagnostic output for submodules before build - echo "Checking submodule status:" - git submodule status || echo "Continuing despite submodule status check issues" + git submodule status - # Run build script and explicitly capture exit code dev/ci/scripts/utils/ci_utils.sh build build_status=$? - # Fail immediately if build fails if [ $build_status -ne 0 ]; then echo "Build failed with exit code $build_status" exit $build_status diff --git a/dev/ci/gitlab-ci-cases.yml b/dev/ci/gitlab-ci-cases.yml index 232a3f96e2b..773960871f4 100644 --- a/dev/ci/gitlab-ci-cases.yml +++ b/dev/ci/gitlab-ci-cases.yml @@ -21,8 +21,10 @@ extends: .base_config stage: create_experiments script: - - export RUNTESTS=${RUNTESTS_DIR} - - ${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${GW_HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml + - | + export RUNTESTS=${RUNTESTS_DIR} + ${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${GW_HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml + exit $? needs: - build-${machine} @@ -31,8 +33,10 @@ extends: .base_config stage: run_tests script: - - echo "Using build directory $GW_HOMEgfs (dated $BUILD_DATE)" - - pslot=$(${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${GW_RUN_PATH} ${pslot} global-workflow + - | + echo "Using build directory $GW_HOMEgfs (dated $BUILD_DATE)" + pslot=$(${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${GW_RUN_PATH} ${pslot} global-workflow + exit $? needs: - setup_experiments-${machine} From 7803099a3ed6ee9e2fd0d80c0a77c8a77776a332 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 25 Apr 2025 10:59:27 -0400 Subject: [PATCH 74/76] fixed bug with find_homegfs.py with using data before it was defined and added unit tests for it and for parse_yaml, added a couple of featchers to support bash --- .github/workflows/ci_unit_tests.yaml | 8 +- .../unittests/test_data/test_config.yaml | 21 ++++ dev/ci/scripts/unittests/test_find_homegfs.py | 102 ++++++++++++++++++ dev/ci/scripts/unittests/test_parse_yaml.py | 102 ++++++++++++++++++ dev/ci/scripts/utils/parse_yaml.py | 18 +++- 5 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 dev/ci/scripts/unittests/test_data/test_config.yaml create mode 100644 dev/ci/scripts/unittests/test_find_homegfs.py create mode 100644 dev/ci/scripts/unittests/test_parse_yaml.py diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index 83fbb2c9a40..f5305dc7ca3 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -26,8 +26,7 @@ jobs: sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl python -m pip install --upgrade pip python -m pip install -r global-workflow/dev/workflow/requirements.txt - pip install pytest - pip install wget + pip install pytest pytest-cov pyyaml jinja2 wget - name: Cache Rocoto id: cache-rocoto @@ -60,6 +59,11 @@ jobs: cd global-workflow/sorc git submodule update --init -j 2 wxflow ufs_model.fd ./link_workflow.sh + + # Create test data directory for unit tests + mkdir -p ../dev/ci/scripts/unittests/test_data + echo "Creating test directories and files for CI tests" + cd ../dev/ci/scripts/unittests pytest -v --junitxml test-results.xml diff --git a/dev/ci/scripts/unittests/test_data/test_config.yaml b/dev/ci/scripts/unittests/test_data/test_config.yaml new file mode 100644 index 00000000000..797538bf44c --- /dev/null +++ b/dev/ci/scripts/unittests/test_data/test_config.yaml @@ -0,0 +1,21 @@ +# Test configuration file for parse_yaml.py unit tests +top_level: simple_value +nested: + key1: value1 + key2: value2 + deeper: + key3: value3 +numbers: + integer: 42 + float: 3.14 +list_data: + - item1 + - item2 + - item3 +complex: + nested_list: + - name: first + value: 1 + - name: second + value: 2 +template_value: "{{ HOMEgfs }}/some/path" diff --git a/dev/ci/scripts/unittests/test_find_homegfs.py b/dev/ci/scripts/unittests/test_find_homegfs.py new file mode 100644 index 00000000000..b1dc3cc66a9 --- /dev/null +++ b/dev/ci/scripts/unittests/test_find_homegfs.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import unittest +import os +import sys +import tempfile +import shutil +from pathlib import Path + +# Add parent directory to sys.path to import find_homegfs module +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +UTILS_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', 'utils')) +sys.path.insert(0, UTILS_DIR) + +# Import find_homegfs directly +from find_homegfs import find_homegfs + + +class TestFindHOMEgfs(unittest.TestCase): + """Tests for the find_homegfs.py script""" + + def setUp(self): + # Create a temporary directory structure for testing + self.test_dir = tempfile.mkdtemp() + + # Create a fake repo structure with .github directory + self.fake_repo_path = os.path.join(self.test_dir, "fake_repo") + os.makedirs(os.path.join(self.fake_repo_path, ".github")) + + # Create a nested directory structure for testing + self.nested_dir = os.path.join(self.fake_repo_path, "dir1", "dir2", "dir3") + os.makedirs(self.nested_dir) + + def tearDown(self): + # Clean up the temporary directory + shutil.rmtree(self.test_dir) + + def test_find_homegfs_current_dir(self): + """Test find_homegfs when starting from the repo root""" + result = find_homegfs(self.fake_repo_path) + self.assertEqual(str(result), str(Path(self.fake_repo_path))) + + def test_find_homegfs_nested_dir(self): + """Test find_homegfs when starting from a nested directory""" + result = find_homegfs(self.nested_dir) + self.assertEqual(str(result), str(Path(self.fake_repo_path))) + + def test_find_homegfs_none_start_path(self): + """Test find_homegfs with None start_path (should use cwd)""" + # Save the current directory + original_dir = os.getcwd() + + try: + # Change to the fake repo directory + os.chdir(self.fake_repo_path) + result = find_homegfs(None) + self.assertEqual(str(result), str(Path(self.fake_repo_path))) + finally: + # Restore the original directory + os.chdir(original_dir) + + def test_find_homegfs_not_found(self): + """Test find_homegfs when .github directory doesn't exist""" + # Create a directory outside the fake repo + outside_dir = os.path.join(self.test_dir, "outside") + os.makedirs(outside_dir) + + # Patch os.path.dirname to ensure we don't traverse beyond our test directory + real_dirname = os.path.dirname + + def mock_dirname(path): + result = real_dirname(path) + # If we're about to go above our test directory, return the same path + # to simulate reaching the filesystem root + if result == self.test_dir or os.path.dirname(result) == self.test_dir: + return path + return result + + original_dirname = os.path.dirname + os.path.dirname = mock_dirname + + try: + with self.assertRaises(ValueError): + find_homegfs(outside_dir) + finally: + # Restore original function + os.path.dirname = original_dirname + + def test_with_string_path(self): + """Test find_homegfs with string path""" + result = find_homegfs(str(self.fake_repo_path)) + self.assertEqual(str(result), str(Path(self.fake_repo_path))) + + def test_with_path_object(self): + """Test find_homegfs with Path object""" + result = find_homegfs(Path(self.fake_repo_path)) + self.assertEqual(str(result), str(Path(self.fake_repo_path))) + + +if __name__ == '__main__': + print("Starting TestFindHOMEgfs tests...") + unittest.main(verbosity=2) diff --git a/dev/ci/scripts/unittests/test_parse_yaml.py b/dev/ci/scripts/unittests/test_parse_yaml.py new file mode 100644 index 00000000000..48a29ca3475 --- /dev/null +++ b/dev/ci/scripts/unittests/test_parse_yaml.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import unittest +import os +import sys +import subprocess +from pathlib import Path + +# Add parent directory to sys.path to import utils modules +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +UTILS_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', 'utils')) +sys.path.insert(0, UTILS_DIR) + +# Path to the test data directory +TEST_DATA_DIR = os.path.join(SCRIPT_DIR, 'test_data') +TEST_CONFIG = os.path.join(TEST_DATA_DIR, 'test_config.yaml') +SCRIPT_PATH = os.path.join(SCRIPT_DIR, '..', 'utils', 'parse_yaml.py') + + +class TestParseYAML(unittest.TestCase): + """Tests for the parse_yaml.py script""" + + @classmethod + def setUpClass(cls): + # Ensure test_data directory exists + os.makedirs(TEST_DATA_DIR, exist_ok=True) + + # Create test yaml file if it doesn't exist + if not os.path.exists(TEST_CONFIG): + with open(TEST_CONFIG, 'w') as f: + f.write('''# Test configuration file for parse_yaml.py unit tests +top_level: simple_value +nested: + key1: value1 + key2: value2 + deeper: + key3: value3 +numbers: + integer: 42 + float: 3.14 +list_data: + - item1 + - item2 + - item3 +complex: + nested_list: + - name: first + value: 1 + - name: second + value: 2 +template_value: "/path/to/homegfs/some/path"''') + + def test_cli_basic(self): + """Test the command-line interface with basic options""" + # Test retrieving a simple value + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'top_level'] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") + self.assertEqual(result.stdout.strip(), 'simple_value') + + # Test retrieving a nested value + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'nested.key2'] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") + self.assertEqual(result.stdout.strip(), 'value2') + + def test_cli_default_value(self): + """Test the --default option""" + # Test default value for non-existent key + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', + '-d', 'default_value'] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") + self.assertEqual(result.stdout.strip(), 'default_value') + + def test_cli_fail_on_missing(self): + """Test the --fail-on-missing option""" + # Test that non-existent key with --fail-on-missing fails + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', + '--fail-on-missing'] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 1, "Expected command to fail with code 1") + self.assertIn("not found", result.stderr) + + # JSON formatting test has been removed + + def test_cli_string_option(self): + """Test the --string option for list output""" + # Test string output for a list + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'list_data', + '--string'] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") + + # Each list item should be on a separate line + lines = result.stdout.strip().split('\n') + self.assertEqual(lines, ['item1', 'item2', 'item3']) + + +if __name__ == '__main__': + print("Starting TestParseYAML tests...") + unittest.main(verbosity=2) diff --git a/dev/ci/scripts/utils/parse_yaml.py b/dev/ci/scripts/utils/parse_yaml.py index 98fa4c66f10..ed7879155cc 100755 --- a/dev/ci/scripts/utils/parse_yaml.py +++ b/dev/ci/scripts/utils/parse_yaml.py @@ -26,6 +26,8 @@ def parse_args(): parser.add_argument('-y', '--yaml', help='full path to yaml file to parse', type=Path, required=True) parser.add_argument('-k', '--key', help='key to return value of', type=str, required=True) parser.add_argument('-s', '--string', help='output results as strings', action="store_true", required=False) + parser.add_argument('-d', '--default', help='default value to return if key is not found', type=str, required=False) + parser.add_argument('-f', '--fail-on-missing', help='exit with code 1 if key is not found', action="store_true", required=False) return parser.parse_args() @@ -42,8 +44,7 @@ def yq(yamlfile, key): """ HOMEgfs = find_homegfs() - data.update({'HOMEgfs': HOMEgfs}) - ydict = parse_j2yaml(path=yamlfile, data=data) + ydict = parse_j2yaml(path=yamlfile, data={'HOMEgfs': HOMEgfs}) if key == 'all': return ydict list_keys = key.split('.') @@ -61,6 +62,19 @@ def yq(yamlfile, key): args = parse_args() values = yq(args.yaml, args.key) + + # Handle missing values + if values is None: + if hasattr(args, 'fail_on_missing') and args.fail_on_missing: + print(f"Error: Key '{args.key}' not found in {args.yaml}", file=sys.stderr) + sys.exit(1) + elif hasattr(args, 'default') and args.default is not None: + values = args.default + else: + # For shell script usage, an empty output is often more useful than "None" + sys.exit(0) + + # Output formatting if args.string and isinstance(values, list): for value in values: print(value) From ad9df9e77da5b2191d1085ac46a618a7ba5e5c4f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 25 Apr 2025 11:03:08 -0400 Subject: [PATCH 75/76] pynorms inforcement by elmenating spaceds on empty lines --- dev/ci/scripts/utils/parse_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ci/scripts/utils/parse_yaml.py b/dev/ci/scripts/utils/parse_yaml.py index ed7879155cc..96969d89ad1 100755 --- a/dev/ci/scripts/utils/parse_yaml.py +++ b/dev/ci/scripts/utils/parse_yaml.py @@ -62,7 +62,7 @@ def yq(yamlfile, key): args = parse_args() values = yq(args.yaml, args.key) - + # Handle missing values if values is None: if hasattr(args, 'fail_on_missing') and args.fail_on_missing: @@ -73,7 +73,7 @@ def yq(yamlfile, key): else: # For shell script usage, an empty output is often more useful than "None" sys.exit(0) - + # Output formatting if args.string and isinstance(values, list): for value in values: From 7ec104601a0023f3d0401a84d97510196337bfd5 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 25 Apr 2025 11:30:18 -0400 Subject: [PATCH 76/76] fixed pynorms spaces --- dev/ci/scripts/unittests/test_find_homegfs.py | 22 +++++++------------ dev/ci/scripts/unittests/test_parse_yaml.py | 8 +++---- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/dev/ci/scripts/unittests/test_find_homegfs.py b/dev/ci/scripts/unittests/test_find_homegfs.py index b1dc3cc66a9..6ae75e490e8 100644 --- a/dev/ci/scripts/unittests/test_find_homegfs.py +++ b/dev/ci/scripts/unittests/test_find_homegfs.py @@ -6,15 +6,13 @@ import tempfile import shutil from pathlib import Path +from find_homegfs import find_homegfs # Add parent directory to sys.path to import find_homegfs module SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) UTILS_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', 'utils')) sys.path.insert(0, UTILS_DIR) -# Import find_homegfs directly -from find_homegfs import find_homegfs - class TestFindHOMEgfs(unittest.TestCase): """Tests for the find_homegfs.py script""" @@ -22,11 +20,9 @@ class TestFindHOMEgfs(unittest.TestCase): def setUp(self): # Create a temporary directory structure for testing self.test_dir = tempfile.mkdtemp() - # Create a fake repo structure with .github directory self.fake_repo_path = os.path.join(self.test_dir, "fake_repo") os.makedirs(os.path.join(self.fake_repo_path, ".github")) - # Create a nested directory structure for testing self.nested_dir = os.path.join(self.fake_repo_path, "dir1", "dir2", "dir3") os.makedirs(self.nested_dir) @@ -44,12 +40,11 @@ def test_find_homegfs_nested_dir(self): """Test find_homegfs when starting from a nested directory""" result = find_homegfs(self.nested_dir) self.assertEqual(str(result), str(Path(self.fake_repo_path))) - + def test_find_homegfs_none_start_path(self): """Test find_homegfs with None start_path (should use cwd)""" # Save the current directory original_dir = os.getcwd() - try: # Change to the fake repo directory os.chdir(self.fake_repo_path) @@ -58,16 +53,16 @@ def test_find_homegfs_none_start_path(self): finally: # Restore the original directory os.chdir(original_dir) - + def test_find_homegfs_not_found(self): """Test find_homegfs when .github directory doesn't exist""" # Create a directory outside the fake repo outside_dir = os.path.join(self.test_dir, "outside") os.makedirs(outside_dir) - + # Patch os.path.dirname to ensure we don't traverse beyond our test directory real_dirname = os.path.dirname - + def mock_dirname(path): result = real_dirname(path) # If we're about to go above our test directory, return the same path @@ -75,22 +70,21 @@ def mock_dirname(path): if result == self.test_dir or os.path.dirname(result) == self.test_dir: return path return result - + original_dirname = os.path.dirname os.path.dirname = mock_dirname - try: with self.assertRaises(ValueError): find_homegfs(outside_dir) finally: # Restore original function os.path.dirname = original_dirname - + def test_with_string_path(self): """Test find_homegfs with string path""" result = find_homegfs(str(self.fake_repo_path)) self.assertEqual(str(result), str(Path(self.fake_repo_path))) - + def test_with_path_object(self): """Test find_homegfs with Path object""" result = find_homegfs(Path(self.fake_repo_path)) diff --git a/dev/ci/scripts/unittests/test_parse_yaml.py b/dev/ci/scripts/unittests/test_parse_yaml.py index 48a29ca3475..ca704e568ca 100644 --- a/dev/ci/scripts/unittests/test_parse_yaml.py +++ b/dev/ci/scripts/unittests/test_parse_yaml.py @@ -24,7 +24,6 @@ class TestParseYAML(unittest.TestCase): def setUpClass(cls): # Ensure test_data directory exists os.makedirs(TEST_DATA_DIR, exist_ok=True) - # Create test yaml file if it doesn't exist if not os.path.exists(TEST_CONFIG): with open(TEST_CONFIG, 'w') as f: @@ -67,7 +66,7 @@ def test_cli_basic(self): def test_cli_default_value(self): """Test the --default option""" # Test default value for non-existent key - cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', '-d', 'default_value'] result = subprocess.run(cmd, capture_output=True, text=True) self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") @@ -76,7 +75,7 @@ def test_cli_default_value(self): def test_cli_fail_on_missing(self): """Test the --fail-on-missing option""" # Test that non-existent key with --fail-on-missing fails - cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'missing.key', '--fail-on-missing'] result = subprocess.run(cmd, capture_output=True, text=True) self.assertEqual(result.returncode, 1, "Expected command to fail with code 1") @@ -87,11 +86,10 @@ def test_cli_fail_on_missing(self): def test_cli_string_option(self): """Test the --string option for list output""" # Test string output for a list - cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'list_data', + cmd = [sys.executable, SCRIPT_PATH, '-y', TEST_CONFIG, '-k', 'list_data', '--string'] result = subprocess.run(cmd, capture_output=True, text=True) self.assertEqual(result.returncode, 0, f"Command failed: {result.stderr}") - # Each list item should be on a separate line lines = result.stdout.strip().split('\n') self.assertEqual(lines, ['item1', 'item2', 'item3'])