Scheduled Nightly CI/CD Pipeline Script in GitLab on Gaea C6

ci/.gitlab-ci.yml

@@ -0,0 +1,70 @@
+stages:
+  - build
+  - create_experiments
+  - run_tests
+
+# Global variables
+variables:
+  BUILD: 'TODAY'
+  GIT_CLONE_PATH: '$CI_BUILDS_DIR/${BUILD}/global-workflow'
+  HOMEGFS: $GIT_CLONE_PATH
+  RUNTESTS_DIR: $CI_BUILDS_DIR/${BUILD}/RUNTESTS
+  GIT_DEPTH: 1
+  RUNNER_SCRIPT_TIMEOUT: 6h
+  RUNNER_AFTER_SCRIPT_TIMEOUT: 6h
+
+
+# Build stage for the global workflow on compute nodes
+build:
+  variables:
+    GIT_STRATEGY: clone
+    GIT_SUBMODULE_STRATEGY: recursive
+  stage: build
+  script:
+    - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})"
+    - ci/scripts/utils/ci_utils_wrapper.sh build_compute
+    - sorc/link_workflow.sh
+    - mkdir -p ${RUNTESTS_DIR}
+  # TODO - Add more machines to the list and make
+  # the setup and run_tests stages 2D matrices
+  parallel:
+    matrix:
+      - MACHINE: ["gaeac6"]
+  tags:
+    - ${MACHINE}
+
+
+# Create experiments stage from a fixed list of cases in $HOMEGFS/ci/cases/pr
+# TODO: Find a way to dynamically generate the list of cases (maybe using Jinja2)
+setup_experiments:
+  variables:
+    GIT_STRATEGY: none
+  stage: create_experiments
+  script:
+    - export RUNTESTS=${RUNTESTS_DIR}
+    - ${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml
+  parallel:
+    matrix:
+      - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"]
+  tags:
+    - gaeac6
+  dependencies:
+    - build
+
+# Running the list of experiments created in the previous stage
+# using the run-check_ci.sh script from $HOMEgfs/ci/scripts directory
+run_tests:
+  variables:
+    GIT_STRATEGY: none
+  stage: run_tests
+  script:
+    - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})"
+    - pslot=$(${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName})
+    - ${HOMEGFS}/ci/scripts/run-check_ci.sh ${CI_BUILDS_DIR}/${BUILD} ${pslot} global-workflow
+  parallel:
+    matrix:
+       - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"]
+  tags:
+    - gaeac6
+  dependencies:
+    - setup_experiments

ci/platforms/config.gaeac6

@@ -1,24 +1,64 @@
 #!/usr/bin/bash
 
-# Main CI root directory
-export GFS_CI_ROOT=/ncrc/proj/nggps_emc/${USER}/GFS_CI_CD
-# ICSDIR root directory used on the create_experment.py command line
+#########################################################################
+# config.gaeac6 - Platform-specific configuration for Gaea C6
+#
+# This file contains environment variables used by CI/CD scripts,
+# directories, and settings specific to the Gaea C6 platform.
+#########################################################################
+
+# Main CI root directory - Base directory for all CI/CD operations
+export GFS_CI_ROOT="/gpfs/f6/drsa-precip3/proj-shared/${USER}/GFS_CI_CD"
+
+# ICSDIR root directory - Contains initial condition data
+# Used by create_experiment.py for setting up test cases
 export ICSDIR_ROOT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR
 
-# JENKINS launch directory for agent
-export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent
+#########################################################################
+# Jenkins configuration settings
+#########################################################################
+
+# JENKINS launch directory for agent - Where Jenkins agents are launched from
+export JENKINS_AGENT_LAUNCH_DIR="${GFS_CI_ROOT}/Jenkins/agent"
+
 # JENKINS internal working directories for CI jobs (not for users use)
-export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace
+# Where Jenkins stores temporary files during CI job execution
+export JENKINS_WORK_DIR="${GFS_CI_ROOT}/Jenkins/workspace"
+
 # NOTE: JENKINS custom_workspace directory where CI jobs are run
 # /gpfs/f6/drsa-precip3/proj-shared/global/CI
-# is defined in the Jenkinsfile
+# and is defined in $HOMEgfs/ci/Jenkinsfile
+
+
+#########################################################################
+# GitLab CI configuration
+# These variables are referenced directly by launch_gitlab_runner.sh
+#########################################################################
 
-# CTest functional test directories for pre stagged input data
-export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
+# Used in the 'register' step of launch_gitlab_runner.sh for --url parameter
+export GITLAB_URL=https://vlab.noaa.gov/gitlab-licensed
 
-# CI BASH test directories
-export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI
+# Directory for GitLab builds
+# Used in launch_gitlab_runner.sh for location of builds
+export GITLAB_CI_BUILDS_DIR=/gpfs/f6/drsa-precip3/scratch/role.glopara/GFS_CI_ROOT/GITLAB/CI
 
+# Directory for GitLab runner used by launch_gitlab_runner.sh
+export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner"
+
+
+#########################################################################
+# CI CRON system configuration
+#########################################################################
+export GFS_BASH_CI_ROOT="${GFS_CI_ROOT}/GFS_BASH_CI"
+export max_concurrent_cases=5 # number of concurrent cases that can run simultaneously
+export max_concurrent_pr=4    # number of concurrent pull requests
+
+
+#########################################################################
+# CTest functional test directories for pre-staged input data
+#########################################################################
+export STAGED_TESTS_DIR="${GFS_CI_ROOT}/STAGED_TESTS_DIR"
+
+# HPC account information - For job submission
 export HPC_ACCOUNT=drsa-precip3
-export max_concurrent_cases=5
-export max_concurrent_pr=4
+

ci/scripts/utils/launch_gitlab_runner.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+set -e
+
+#########################################################################
+# launch_gitlab_runner.sh - Script to manage GitLab runners for CI/CD
+# 
+# This script handles three main operations for GitLab runners:
+# 1. register - Registers a new GitLab runner with the GitLab server
+# 2. run - Starts a GitLab runner in the background
+# 3. unregister - Removes a GitLab runner from the GitLab server
+#
+# Usage: ./launch_gitlab_runner.sh [register|run|unregister] [token]
+#########################################################################
+
+# Set the HOMEGFS_ variable to the root directory of the global workflow
+HOMEGFS_="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+# Get the hostname of the current machine
+host="$(hostname)"
+
+#########################################################################
+#  Set up runtime environment variables for accounts on supported machines
+#########################################################################
+
+# Source the detect_machine.sh script to determine the MACHINE_ID
+source "${HOMEGFS_}/ush/detect_machine.sh"
+# Check the MACHINE_ID and set up the environment accordingly
+case "${MACHINE_ID}" in
+  hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 )
+    echo "Launching GitLab Runner on ${MACHINE_ID}";;
+  noaacloud )
+    echo "Launching GitLab Runner on ${PW_CSP}";;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1;;
+esac
+
+# Source the platform-specific configuration file
+# This file contains platform-specific variables such as GITLAB_URL, GITLAB_CI_BUILDS_DIR, 
+# and GITLAB_RUNNER_DIR which are required for runner registration and execution
+source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"
+
+# Change to the GitLab runner directory defined in the platform config
+cd "${GITLAB_RUNNER_DIR}" || exit 1
+
+# Set the log file name with the current date and time
+DATE=$(date +%Y%m%d%M) || true
+GITLAB_LOG="${PWD}/launched_gitlab_runner-${DATE}.log"
+rm -f "${GITLAB_LOG}"
+
+#########################################################################
+# GitLab Token Handling
+# The token is used to authenticate the runner with the GitLab server
+#########################################################################
+
+# Get the GitLab runner token from:
+# 1. The second command-line argument
+# 2. The GITLAB_RUNNER_TOKEN environment variable
+# 3. A gitlab_token file in the current directory
+GITLAB_RUNNER_TOKEN="${2:-${GITLAB_RUNNER_TOKEN}}"
+if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then
+  if [[ -f gitlab_token ]]; then	
+   source gitlab_token
+  fi 
+fi
+if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then
+  echo "ERROR: GITLAB_RUNNER_TOKEN not set"
+  exit 1
+fi  
+
+# Download the GitLab runner binary if it does not exist
+if [[ ! -f gitlab-runner ]]; then
+  curl -L --output "${PWD}/gitlab-runner" https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64
+  chmod +x ./gitlab-runner
+fi
+
+#########################################################################
+# REGISTER argument handling
+# Registers a new GitLab runner with the GitLab server
+#########################################################################
+
+if [[ "${1}" == "register" ]]; then
+
+  echo "Registering GitLab Runner ${MACHINE_ID} on host ${host} at ${DATE}" >> "${GITLAB_LOG}"
+  echo "with runner name: ${GITLAB_RUNNER_NAME}" >> "${GITLAB_LOG}"
+  # Register the GitLab runner with the following parameters:
+  # -n: Run in non-interactive mode
+  # -t: Registration token from GitLab
+  # --url: URL of the GitLab server (from config.MACHINE_ID)
+  # --executor: Type of executor (shell in this case)
+  # --builds-dir: Directory where builds will be stored (from config.MACHINE_ID)
+  # --custom_build_dir-enabled: Enable custom build directories
+  # --request-concurrency: Number of concurrent requests that can be handled
+  ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GITLAB_CI_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24
+
+  # Set the concurrent job limit in the GitLab runner config file
+  sed -i 's/concurrent.*/concurrent = 24/' ~/.gitlab-runner/config.toml
+  exit 0
+fi
+
+#########################################################################
+# RUN: Starts a GitLab runner in the background
+#########################################################################
+
+if [[ "${1}" == "run" ]]; then
+  # --working-directory: Directory where the runner is launched and keeps its working files (from config.$MACHINE_ID)
+  # do not confuse this with GitLabs CI_BUILDS_DIR which is designate by GFS_CI_BUILDS_DIR and is where the builds are stored
+  COMMAND="nohup ./gitlab-runner run --working-directory ${GITLAB_RUNNER_DIR}"
+  echo -e "Running gitlab-runner with the command:\n${COMMAND}\nsee log ${GITLAB_LOG}"
+  echo -e "Running gitlab-runner with the command:${COMMAND}" >& "${GITLAB_LOG}"
+  ${COMMAND} >> "${GITLAB_LOG}" 2>&1 &
+  cat "${GITLAB_LOG}"
+  exit 0
+fi
+
+#########################################################################
+# UNREGISTER: Removes a GitLab runner from the GitLab server
+#########################################################################
+
+if [[ "${1}" == "unregister" ]]; then
+  # Unregister the GitLab runner by name
+  ./gitlab-runner unregister --name "${GITLAB_RUNNER_NAME}"
+fi