diff --git a/.gitignore b/.gitignore index b6e47617d..81cee202d 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# bin directory +bin/ diff --git a/modulefiles/GDAS/hera.lua b/modulefiles/GDAS/hera.lua new file mode 100644 index 000000000..0ae09e4ad --- /dev/null +++ b/modulefiles/GDAS/hera.lua @@ -0,0 +1,24 @@ +-- NOAA RDHPCS Hera Modulefile for UFS-DA +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local jedi_opt = '/scratch1/NCEPDEV/jcsda/jedipara/opt/modules' +setenv('JEDI_OPT', jedi_opt) +local jedi_core = pathJoin(jedi_opt, 'modulefiles/core') +prepend_path("MODULEPATH", jedi_core) + +load('jedi/intel-impi/2020.2') + +local mpiexec = '/apps/slurm/default/bin/srun' +local mpinproc = '-n' +setenv('MPIEXEC_EXEC', mpiexec) +setenv('MPIEXEC_NPROC', mpinproc) + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: UFS-DA") +whatis("Description: Load JEDI-Stack for UFS-DA") diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua new file mode 100644 index 000000000..34e329334 --- /dev/null +++ b/modulefiles/GDAS/orion.lua @@ -0,0 +1,29 @@ +-- NOAA HPC Orion Modulefile for UFS-DA +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local jedi_opt = '/work/noaa/da/jedipara/opt/modules' +setenv('JEDI_OPT', jedi_opt) +local jedi_core = pathJoin(jedi_opt, 'modulefiles/core') +prepend_path("MODULEPATH", jedi_core) + +load('jedi/intel-impi') + +local mpiexec = '/opt/slurm/bin/srun' +local mpinproc = '-n' +setenv('MPIEXEC_EXEC', mpiexec) +setenv('MPIEXEC_NPROC', mpinproc) + +-- add R2D2 and SOLO to PYTHONPATH +prepend_path("PYTHONPATH", "/work2/noaa/da/cmartin/UFSDA/python/local/lib/python3.9/site-packages") +-- add R2D2 to path +prepend_path("PATH", "/work2/noaa/da/cmartin/UFSDA/python/local/bin") + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: UFS-DA") +whatis("Description: Load JEDI-Stack for UFS-DA") diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 000000000..1bdb5e747 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,24 @@ +atlas/ +crtm/ +eckit/ +fckit/ +femps/ +fms/ +fv3/ +fv3-jedi/ +fv3-jedi-data/ +fv3-jedi-lm/ +gsw/ +ioda/ +ioda-data/ +iodaconv/ +jedicmake/ +mom6/ +oops/ +saber/ +saber-data/ +soca/ +test-data-release/ +ufo/ +ufo-data/ +build/ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..3ffa2edac --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,130 @@ +# # ------------------------------------------------------------------------- # +# # ufs-da-bundle for building all the components needed for the UFS-DA system # +# # ------------------------------------------------------------------------- # + + +# Preamble +# -------- + +# Check for minimim cmake requirement +cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) + +# Create project +project( ufsda-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran ) + +# Include ecbuild_bundle macro +include( ecbuild_bundle ) + +# Default build mode, change with --build=[release]|debug|bit +set( ECBUILD_DEFAULT_BUILD_TYPE Release ) + +# Enable MPI +set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) + + +# Initialize bundle +# ----------------- +ecbuild_bundle_initialize() + + +# Add packages to the bundle +# -------------------------- + +# jedi-cmake +ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" BRANCH develop UPDATE ) + +# ECMWF libraries +option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF +option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF +option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + +# turn off optional OOPS toy models +option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) +option( ENABLE_QG_MODEL "Build QG toy model" OFF ) + +ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) +ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) + +# External (required) observation operators +ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) + +# Core JEDI repositories +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT ufo GIT "https://github.com/noaa-emc/ufo.git" BRANCH feature/ufsda UPDATE ) + +# FMS and FV3 dynamical core +ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE ) +ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE ) + +# fv3-jedi and associated repositories +ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop UPDATE ) + +# SOCA associated repositories +ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop ) +ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE ) +ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop ) + +# Build IODA converters +option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF) +if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop UPDATE ) +endif() + +# ioda, ufo, fv3-jedi, and saber test data +#--------------------------------- + +# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch. +# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case, +# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned. +# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored +# in a local directory, ioda-data repo will not be cloned + +find_branch_name(REPO_DIR_NAME ioda) +# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored +# in a local directory, ioda-data repo will not be cloned +if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) + + # If IODA's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME ioda-data + BRANCH ${GIT_BRANCH_FUNC} ) +endif() + +# same procedure for ufo-data +find_branch_name(REPO_DIR_NAME ufo) +if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) + + # If UFO's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME ufo-data + BRANCH ${GIT_BRANCH_FUNC} ) +endif() + +# same procedure for fv3-jedi-data +find_branch_name(REPO_DIR_NAME fv3-jedi) +if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) + + # If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME fv3-jedi-data + BRANCH ${GIT_BRANCH_FUNC} ) +endif() + +# same procedure for saber-data +find_branch_name(REPO_DIR_NAME saber) +if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop UPDATE ) + + # If saber's current branch is available in saber-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME saber-data + BRANCH ${GIT_BRANCH_FUNC} ) +endif() + +# Finalize bundle +# --------------- +ecbuild_bundle_finalize() diff --git a/ush/build_GDASApp.sh b/ush/build_GDASApp.sh new file mode 100755 index 000000000..e7eba4314 --- /dev/null +++ b/ush/build_GDASApp.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# build_GDASApp.sh +# 1 - determine if on supported host, load modules if so +# 2 - run ecbuild +# 3 - build +# 4 - optional, run unit tests + +set -eux + +dir_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" + +dir_modules=$dir_root/modulefiles + +if [ ${BUILD_TARGET:-} = "orion" -o ${BUILD_TARGET:-} = "hera" ]; then + set +x + source $MODULESHOME/init/sh + module purge + module use $dir_modules + module load GDAS/${BUILD_TARGET} + module list + set -x +fi + +# remove/create directories +[ -d $dir_root/bin ] || mkdir -p $dir_root/bin + +rm -rf $dir_root/src/build +mkdir -p $dir_root/src/build +cd $dir_root/src/build + +# run ecbuild +ecbuild -DMPIEXEC_EXECUTABLE=$MPIEXEC_EXEC -DMPIEXEC_NUMPROC_FLAG=$MPIEXEC_NPROC ../ + +# run make +make -j ${BUILD_JOBS:-8} VERBOSE=${BUILD_VERBOSE:-} + +# link executables to exec dir +ln -sf $dir_root/src/build/bin/fv3jedi* $dir_root/bin/. +ln -sf $dir_root/src/build/bin/soca* $dir_root/bin/. + +exit 0