GitLab CI Framework for schedule PR cases and ctests on multi hosts

.github/workflows/ci_unit_tests.yaml

@@ -26,8 +26,7 @@ jobs:
         sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl
         python -m pip install --upgrade pip
         python -m pip install -r global-workflow/dev/workflow/requirements.txt
-        pip install pytest
-        pip install wget
+        pip install pytest pytest-cov pyyaml jinja2 wget
 
     - name: Cache Rocoto
       id: cache-rocoto
@@ -60,14 +59,19 @@ jobs:
         cd global-workflow/sorc
         git submodule update --init -j 2 wxflow ufs_model.fd
         ./link_workflow.sh
-        cd ../dev/ci/scripts/tests
+
+        # Create test data directory for unit tests
+        mkdir -p ../dev/ci/scripts/unittests/test_data
+        echo "Creating test directories and files for CI tests"
+
+        cd ../dev/ci/scripts/unittests
         pytest -v --junitxml test-results.xml
 
 
     - name: Publish Test Results
       if: always()
       uses: EnricoMi/publish-unit-test-result-action@v2
       with:
-        files: global-workflow/dev/ci/scripts/tests/test-results.xml
+        files: global-workflow/dev/ci/scripts/unittests/test-results.xml
         job_summary: true
         comment_mode: off

.github/workflows/pw_aws_ci.yaml

@@ -157,7 +157,7 @@ jobs:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./dev/ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
+          ./dev/ci/scripts/run_check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
 
   clean-up:
     needs: run-experiments

.gitlab-ci.yml

@@ -0,0 +1,80 @@
+# ==========================================================================
+# Main GitLab CI Configuration for the global-workflow Project
+# ==========================================================================
+#
+# This is the primary CI/CD configuration file that orchestrates the GitLab CI
+# testing framework for the global-workflow project. It defines pipeline stages
+# and includes specialized configuration files for different testing aspects.
+#
+# The pipeline architecture supports:
+#   1. Multiple host environments (hera, gaeac6, etc.)
+#   2. Two primary testing modes:
+#      - PR validation via standard test cases
+#      - CTest-based tests triggered via GitHub API
+#
+# Included files:
+#   - .gitlab-ci-ctests.yml: Handles CMake/CTest-based testing framework
+#   - .gitlab-ci-cases.yml: Defines templates for standard experiment cases
+#   - .gitlab-ci-hosts.yml: Contains host-specific configurations and test matrices
+#                          that can be extended to support additional hosts
+#
+# The host configuration in .gitlab-ci-hosts.yml is designed to be easily 
+# extended with new computing platforms and allows per-host specification 
+# of which test cases to run.
+
+stages:
+  - build
+  - create_experiments
+  - run_tests
+  - finalize
+
+variables:
+  BUILD: '' # Placeholder for control over build directory path
+  # Using GW prefix to avoid confusion with GitLab predefined variables
+  GW_RUN_PATH: ${CI_BUILDS_DIR}/${BUILD}/${CI_COMMIT_SHORT_SHA}
+  # Overriding GitLab's predefined GIT_CLONE_PATH variable to specify the exact location
+  # where the repository should be cloned. This works with custom_build_dir-enabled in the runner config.
+  GIT_CLONE_PATH: '${GW_RUN_PATH}/global-workflow'
+  GW_HOMEgfs: ${GIT_CLONE_PATH}
+  RUNTESTS_DIR: ${GW_RUN_PATH}/RUNTESTS
+  GIT_DEPTH: 10
+  RUNNER_SCRIPT_TIMEOUT: 6h
+  RUNNER_AFTER_SCRIPT_TIMEOUT: 6h
+  # Controls pipeline behavior: CTests (true) or PR cases (false)
+  GITHUB_API_TRIGGER: ${GITHUB_API_TRIGGER:-"false"}
+  PR_NUMBER: ${PR_NUMBER:-"0"}
+
+# Include specialized pipeline configuration files
+include:
+  - local: 'dev/ci/gitlab-ci-ctests.yml'  # CTest framework configuration
+  - local: 'dev/ci/gitlab-ci-cases.yml'   # Standard test case templates
+  - local: 'dev/ci/gitlab-ci-hosts.yml'   # Host-specific configurations
+
+.base_config:
+  variables:
+    GIT_STRATEGY: none
+
+# Common build template for all modalities
+.build_template:
+  variables:
+    GIT_STRATEGY: clone
+    GIT_SUBMODULE_STRATEGY: recursive
+    GIT_SSL_NO_VERIFY: "true"  # Address potential certificate verification issues
+  stage: build
+  script:
+    - |
+      set -e  # Fail the job if any command fails
+      echo "Setting up build environment for ${machine}"
+      echo "Using build directory ${GW_HOMEgfs}"
+      git submodule status
+
+      dev/ci/scripts/utils/ci_utils.sh build
+      build_status=$?
+
+      if [ $build_status -ne 0 ]; then
+        echo "Build failed with exit code $build_status"
+        exit $build_status
+      fi
+
+      sorc/link_workflow.sh
+      mkdir -p ${RUNTESTS_DIR}

CMakeLists.txt

@@ -24,4 +24,4 @@ endif()
 # add_subdirectory(sorc)
 
 # Setup tests
-add_subdirectory(ctests)
+add_subdirectory(dev/ctests)

dev/ci/Jenkinsfile

@@ -129,7 +129,7 @@ pipeline {
                             def error_logs_message = ""
                             dir("${HOMEgfs}/sorc") {
                                 try {
-                                    sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables
+                                    sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh build") // build the global-workflow executables
                                 } catch (Exception error_build) {
                                     echo "Failed to build global-workflow: ${error_build.getMessage()}"
                                     if ( fileExists("logs/error.logs") ) {
@@ -203,7 +203,7 @@ pipeline {
                                         try {
                                               error_output = sh(script: """
                                                 source ${HOMEgfs_dev}/ush/gw_setup.sh
-                                                ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs_dev}/ci/cases/pr/${caseName}.yaml
+                                                ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs_dev}/ci/cases/pr/${caseName}.yaml
                                               """, returnStdout: true).trim()
                                         } catch (Exception error_create) {
                                             sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body '${caseName} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n```\n${error_output}```' """)
@@ -215,22 +215,22 @@ pipeline {
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
-                                                def pslot = sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
+                                                def pslot = sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
                                                     sh(script: """
                                                       source ${HOMEgfs_dev}/ush/gw_setup.sh
-                                                      ${HOMEgfs_dev}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
+                                                      ${HOMEgfs_dev}/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
                                                     """)
                                                     sh(script: """
                                                       source ${HOMEgfs_dev}/ush/gw_setup.sh
-                                                      ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
+                                                      ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
                                                     """)
                                                 } catch (Exception error_experment) {
                                                     sh(script: """
                                                       source ${HOMEgfs_dev}/ush/gw_setup.sh
-                                                      ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}
+                                                      ${HOMEgfs_dev}/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot}
                                                     """)
                                                     ws(CUSTOM_WORKSPACE) {
                                                         def error_logs = ""
@@ -300,9 +300,9 @@ pipeline {
                         done
                     """, returnStatus: true)
                     sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
-                    if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log")) {
-                        sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}<br>Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true)
-                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log  """, returnStatus: true)
+                    if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log")) {
+                        sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}<br>Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log""", returnStatus: true)
+                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/run_check_ci.log  """, returnStatus: true)
                     }
                     if (STATUS == 'Passed') {
                         try {

dev/ci/Jenkinsfile4AWS

@@ -134,7 +134,7 @@ pipeline {
                             def error_logs_message = ""
                             dir("${HOMEgfs}/sorc") {
                                 try {
-                                    // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables
+                                    // sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh build") // build the global-workflow executables
                                     sh(script: './build_compute.sh -A ${USER} gfs gefs sfs')  // build the global-workflow executables
                                 } catch (Exception error_build) {
                                     echo "Failed to build global-workflow: ${error_build.getMessage()}"
@@ -205,10 +205,10 @@ pipeline {
                                 script {
                                         env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
                                         try {
-                                              error_output = sh(script: """
+                                              error_output = sh(script: \"\"\"
                                                 source ${HOMEgfs}/workflow/gw_setup.sh
-                                                ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml
-                                              """, returnStdout: true).trim()
+                                                ${HOMEgfs}/ci/scripts/utils/ci_utils.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml
+                                              \"\"\", returnStdout: true).trim()
                                         } catch (Exception error_create) {
                                             sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
                                             error("Case ${caseName} failed to create experiment directory")
@@ -219,22 +219,22 @@ pipeline {
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
-                                                def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
+                                                def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
                                                     sh(script: """
                                                       source ${HOMEgfs}/workflow/gw_setup.sh
-                                                      ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
+                                                      ${HOMEgfs}/ci/scripts/run_check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
                                                     """)
                                                     sh(script: """
                                                       source ${HOMEgfs}/workflow/gw_setup.sh
-                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
+                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
                                                     """)
                                                 } catch (Exception error_experment) {
                                                     sh(script: """
                                                       source ${HOMEgfs}/workflow/gw_setup.sh
-                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}
+                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils.sh cancel_batch_jobs ${pslot}
                                                     """)
                                                     ws(CUSTOM_WORKSPACE) {
                                                         def error_logs = ""

dev/ci/gitlab-ci-cases.yml

@@ -0,0 +1,42 @@
+# ==========================================================================
+# Templates for Standard Test Cases
+# ==========================================================================
+#
+# This file defines templates for setting up and running standard experiment 
+# test cases for the global-workflow project. These cases represent more 
+# comprehensive, end-to-end testing scenarios than the CTesting framework.
+#
+# These templates are used primarily for PR testing when GITHUB_API_TRIGGER=false.
+# The actual test case matrix for each host is defined in .gitlab-ci-hosts.yml.
+#
+# Key templates:
+# - .setup_template: Creates experiment directories based on case YAML files
+# - .run_tests_template: Executes and validates the experiment workflows
+#
+# Test cases are defined as YAML files in the dev/ci/cases/pr directory and
+# are referenced by name in the host configuration file's test matrix.
+
+# Template for experiment setup jobs
+.setup_template:
+  extends: .base_config
+  stage: create_experiments
+  script:
+    - |
+      export RUNTESTS=${RUNTESTS_DIR}
+      ${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh create_experiment ${GW_HOMEgfs}/dev/ci/cases/pr/${caseName}.yaml
+      exit $?
+  needs:
+    - build-${machine}
+
+# Template for test execution jobs
+.run_tests_template:
+  extends: .base_config
+  stage: run_tests
+  script:
+    - |
+      echo "Using build directory $GW_HOMEgfs (dated $BUILD_DATE)"
+      pslot=$(${GW_HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh get_pslot ${RUNTESTS_DIR} ${caseName})
+      ${GW_HOMEgfs}/dev/ci/scripts/run_check_ci.sh ${GW_RUN_PATH} ${pslot} global-workflow
+      exit $?
+  needs:
+    - setup_experiments-${machine}