diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml new file mode 100644 index 00000000000..1882234bb87 --- /dev/null +++ b/ci/.gitlab-ci.yml @@ -0,0 +1,70 @@ +stages: + - build + - create_experiments + - run_tests + +# Global variables +variables: + BUILD: 'TODAY' + GIT_CLONE_PATH: '$CI_BUILDS_DIR/${BUILD}/global-workflow' + HOMEGFS: $GIT_CLONE_PATH + RUNTESTS_DIR: $CI_BUILDS_DIR/${BUILD}/RUNTESTS + GIT_DEPTH: 1 + RUNNER_SCRIPT_TIMEOUT: 6h + RUNNER_AFTER_SCRIPT_TIMEOUT: 6h + + +# Build stage for the global workflow on compute nodes +build: + variables: + GIT_STRATEGY: clone + GIT_SUBMODULE_STRATEGY: recursive + stage: build + script: + - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" + - ci/scripts/utils/ci_utils_wrapper.sh build_compute + - sorc/link_workflow.sh + - mkdir -p ${RUNTESTS_DIR} + # TODO - Add more machines to the list and make + # the setup and run_tests stages 2D matrices + parallel: + matrix: + - MACHINE: ["gaeac6"] + tags: + - ${MACHINE} + + +# Create experiments stage from a fixed list of cases in $HOMEGFS/ci/cases/pr +# TODO: Find a way to dynamically generate the list of cases (maybe using Jinja2) +setup_experiments: + variables: + GIT_STRATEGY: none + stage: create_experiments + script: + - export RUNTESTS=${RUNTESTS_DIR} + - ${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + parallel: + matrix: + - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] + tags: + - gaeac6 + dependencies: + - build + +# Running the list of experiments created in the previous stage +# using the run-check_ci.sh script from $HOMEgfs/ci/scripts directory +run_tests: + variables: + GIT_STRATEGY: none + stage: run_tests + script: + - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" + - pslot=$(${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - ${HOMEGFS}/ci/scripts/run-check_ci.sh ${CI_BUILDS_DIR}/${BUILD} ${pslot} global-workflow + parallel: + matrix: + - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] + tags: + - gaeac6 + dependencies: + - setup_experiments diff --git a/ci/platforms/config.gaeac6 b/ci/platforms/config.gaeac6 index c8e124040bf..6958f544089 100644 --- a/ci/platforms/config.gaeac6 +++ b/ci/platforms/config.gaeac6 @@ -1,24 +1,64 @@ #!/usr/bin/bash -# Main CI root directory -export GFS_CI_ROOT=/ncrc/proj/nggps_emc/${USER}/GFS_CI_CD -# ICSDIR root directory used on the create_experment.py command line +######################################################################### +# config.gaeac6 - Platform-specific configuration for Gaea C6 +# +# This file contains environment variables used by CI/CD scripts, +# directories, and settings specific to the Gaea C6 platform. +######################################################################### + +# Main CI root directory - Base directory for all CI/CD operations +export GFS_CI_ROOT="/gpfs/f6/drsa-precip3/proj-shared/${USER}/GFS_CI_CD" + +# ICSDIR root directory - Contains initial condition data +# Used by create_experiment.py for setting up test cases export ICSDIR_ROOT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR -# JENKINS launch directory for agent -export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent +######################################################################### +# Jenkins configuration settings +######################################################################### + +# JENKINS launch directory for agent - Where Jenkins agents are launched from +export JENKINS_AGENT_LAUNCH_DIR="${GFS_CI_ROOT}/Jenkins/agent" + # JENKINS internal working directories for CI jobs (not for users use) -export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace +# Where Jenkins stores temporary files during CI job execution +export JENKINS_WORK_DIR="${GFS_CI_ROOT}/Jenkins/workspace" + # NOTE: JENKINS custom_workspace directory where CI jobs are run # /gpfs/f6/drsa-precip3/proj-shared/global/CI -# is defined in the Jenkinsfile +# and is defined in $HOMEgfs/ci/Jenkinsfile + + +######################################################################### +# GitLab CI configuration +# These variables are referenced directly by launch_gitlab_runner.sh +######################################################################### -# CTest functional test directories for pre stagged input data -export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR +# Used in the 'register' step of launch_gitlab_runner.sh for --url parameter +export GITLAB_URL=https://vlab.noaa.gov/gitlab-licensed -# CI BASH test directories -export GFS_BASH_CI_ROOT=${GFS_CI_ROOT}/GFS_BASH_CI +# Directory for GitLab builds +# Used in launch_gitlab_runner.sh for location of builds +export GITLAB_CI_BUILDS_DIR=/gpfs/f6/drsa-precip3/scratch/role.glopara/GFS_CI_ROOT/GITLAB/CI +# Directory for GitLab runner used by launch_gitlab_runner.sh +export GITLAB_RUNNER_DIR="${GFS_CI_ROOT}/GitLab/Runner" + + +######################################################################### +# CI CRON system configuration +######################################################################### +export GFS_BASH_CI_ROOT="${GFS_CI_ROOT}/GFS_BASH_CI" +export max_concurrent_cases=5 # number of concurrent cases that can run simultaneously +export max_concurrent_pr=4 # number of concurrent pull requests + + +######################################################################### +# CTest functional test directories for pre-staged input data +######################################################################### +export STAGED_TESTS_DIR="${GFS_CI_ROOT}/STAGED_TESTS_DIR" + +# HPC account information - For job submission export HPC_ACCOUNT=drsa-precip3 -export max_concurrent_cases=5 -export max_concurrent_pr=4 + diff --git a/ci/scripts/utils/launch_gitlab_runner.sh b/ci/scripts/utils/launch_gitlab_runner.sh new file mode 100755 index 00000000000..653c4ccf771 --- /dev/null +++ b/ci/scripts/utils/launch_gitlab_runner.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -e + +######################################################################### +# launch_gitlab_runner.sh - Script to manage GitLab runners for CI/CD +# +# This script handles three main operations for GitLab runners: +# 1. register - Registers a new GitLab runner with the GitLab server +# 2. run - Starts a GitLab runner in the background +# 3. unregister - Removes a GitLab runner from the GitLab server +# +# Usage: ./launch_gitlab_runner.sh [register|run|unregister] [token] +######################################################################### + +# Set the HOMEGFS_ variable to the root directory of the global workflow +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +# Get the hostname of the current machine +host="$(hostname)" + +######################################################################### +# Set up runtime environment variables for accounts on supported machines +######################################################################### + +# Source the detect_machine.sh script to determine the MACHINE_ID +source "${HOMEGFS_}/ush/detect_machine.sh" +# Check the MACHINE_ID and set up the environment accordingly +case "${MACHINE_ID}" in + hera | orion | hercules | wcoss2 | gaeac5 | gaeac6 ) + echo "Launching GitLab Runner on ${MACHINE_ID}";; + noaacloud ) + echo "Launching GitLab Runner on ${PW_CSP}";; + *) + echo "Unsupported platform. Exiting with error." + exit 1;; +esac + +# Source the platform-specific configuration file +# This file contains platform-specific variables such as GITLAB_URL, GITLAB_CI_BUILDS_DIR, +# and GITLAB_RUNNER_DIR which are required for runner registration and execution +source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" + +# Change to the GitLab runner directory defined in the platform config +cd "${GITLAB_RUNNER_DIR}" || exit 1 + +# Set the log file name with the current date and time +DATE=$(date +%Y%m%d%M) || true +GITLAB_LOG="${PWD}/launched_gitlab_runner-${DATE}.log" +rm -f "${GITLAB_LOG}" + +######################################################################### +# GitLab Token Handling +# The token is used to authenticate the runner with the GitLab server +######################################################################### + +# Get the GitLab runner token from: +# 1. The second command-line argument +# 2. The GITLAB_RUNNER_TOKEN environment variable +# 3. A gitlab_token file in the current directory +GITLAB_RUNNER_TOKEN="${2:-${GITLAB_RUNNER_TOKEN}}" +if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then + if [[ -f gitlab_token ]]; then + source gitlab_token + fi +fi +if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then + echo "ERROR: GITLAB_RUNNER_TOKEN not set" + exit 1 +fi + +# Download the GitLab runner binary if it does not exist +if [[ ! -f gitlab-runner ]]; then + curl -L --output "${PWD}/gitlab-runner" https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64 + chmod +x ./gitlab-runner +fi + +######################################################################### +# REGISTER argument handling +# Registers a new GitLab runner with the GitLab server +######################################################################### + +if [[ "${1}" == "register" ]]; then + + echo "Registering GitLab Runner ${MACHINE_ID} on host ${host} at ${DATE}" >> "${GITLAB_LOG}" + echo "with runner name: ${GITLAB_RUNNER_NAME}" >> "${GITLAB_LOG}" + # Register the GitLab runner with the following parameters: + # -n: Run in non-interactive mode + # -t: Registration token from GitLab + # --url: URL of the GitLab server (from config.MACHINE_ID) + # --executor: Type of executor (shell in this case) + # --builds-dir: Directory where builds will be stored (from config.MACHINE_ID) + # --custom_build_dir-enabled: Enable custom build directories + # --request-concurrency: Number of concurrent requests that can be handled + ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GITLAB_CI_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 + + # Set the concurrent job limit in the GitLab runner config file + sed -i 's/concurrent.*/concurrent = 24/' ~/.gitlab-runner/config.toml + exit 0 +fi + +######################################################################### +# RUN: Starts a GitLab runner in the background +######################################################################### + +if [[ "${1}" == "run" ]]; then + # --working-directory: Directory where the runner is launched and keeps its working files (from config.$MACHINE_ID) + # do not confuse this with GitLabs CI_BUILDS_DIR which is designate by GFS_CI_BUILDS_DIR and is where the builds are stored + COMMAND="nohup ./gitlab-runner run --working-directory ${GITLAB_RUNNER_DIR}" + echo -e "Running gitlab-runner with the command:\n${COMMAND}\nsee log ${GITLAB_LOG}" + echo -e "Running gitlab-runner with the command:${COMMAND}" >& "${GITLAB_LOG}" + ${COMMAND} >> "${GITLAB_LOG}" 2>&1 & + cat "${GITLAB_LOG}" + exit 0 +fi + +######################################################################### +# UNREGISTER: Removes a GitLab runner from the GitLab server +######################################################################### + +if [[ "${1}" == "unregister" ]]; then + # Unregister the GitLab runner by name + ./gitlab-runner unregister --name "${GITLAB_RUNNER_NAME}" +fi