Skip to content

Consolidated build for fv3-jedi and soca #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mark-a-potts wants to merge 16 commits into from
Closed

Consolidated build for fv3-jedi and soca #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
eb24a03
updates to get things building/linking
mark-a-potts Jan 25, 2022
142975a
using two different versions of fv3-jedi depending on UFS_APP
mark-a-potts Jan 26, 2022
2c2a90f
small change to ioda for expat linking
mark-a-potts Jan 27, 2022
6e4c053
fix for building in container
mark-a-potts Jan 27, 2022
81895ee
Merge branch 'feature/soca' of https://github.com/JCSDA/ufs-jedi-bund…
mark-a-potts Jan 27, 2022
eb61ff0
update for godas
mark-a-potts Mar 18, 2022
25ade24
Updating with consolidated versions for atm/godas/s2s builds
mark-a-potts Jun 24, 2022
aa86ef2
Updated atlas version and README
mark-a-potts Jun 30, 2022
a5851a4
updated readme
mark-a-potts Jun 30, 2022
c92d43c
updated atlas version
mark-a-potts Jun 30, 2022
379dd4b
Merge branch 'feature/soca' of https://github.com/jcsda/ufs-jedi-bund…
mark-a-potts Jun 30, 2022
e9dbfe0
updated README for cheyenne
mark-a-potts Jun 30, 2022
0cd1352
updated README for cheyenne
mark-a-potts Jun 30, 2022
ea2387b
Changed to hashes
mark-a-potts Jun 30, 2022
31e2a88
Switched version of FMS. Might be able to remove it as an external now
mark-a-potts Jul 29, 2022
285c325
Merge branch 'develop' into feature/soca
climbfuji Aug 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 27 additions & 24 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI")

# Initialize
ecbuild_bundle_initialize()
ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE )
ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.3.0 )
Copy link
Collaborator

@climbfuji climbfuji Aug 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this needed?

Copy link
Collaborator

@climbfuji climbfuji Aug 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see you are using tags in many other places instead of branches as well, do you really want this in "develop"?

include( jedicmake/cmake/Functions/git_functions.cmake )

# ECMWF libs
Expand All @@ -42,14 +42,15 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29.
# External (required) observation operators
# ------------------------------
option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON
ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi UPDATE )
ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi )

# Core JEDI repositories
# ----------------------
ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc )
ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 1.0.0 )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 8bee2e68efad0734a95565b14249bf0b7dd4e6f2 )
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG b38794726d0f3fcfc27b3bff80d590bb451110f0 )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 67ab97f61893782fc826c0be05b7ef7678ba7dba )

# Options for building with certain models
# ----------------------------------------
Expand All @@ -63,9 +64,9 @@ ecbuild_info("Building with UFS application: ${UFS_APP}")
set(FV3_FORECAST_MODEL "UFS")
# fv3-jedi linear model
# ---------------------
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs UPDATE )
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs )

include_directories(${DEPEND_LIB_ROOT}/include)
include_directories(${DEPEND_LIB_ROOT}/include_r8)
link_directories(${DEPEND_LIB_ROOT}/lib)

include( ExternalProject )
Expand All @@ -74,20 +75,20 @@ ExternalProject_Add(ufs-weather-model
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model
GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git
GIT_SUBMODULES_RECURSE TRUE
GIT_TAG develop
GIT_TAG feature/consolidate
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model
INSTALL_DIR ${DEPEND_LIB_ROOT}
CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS}
CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT}
CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT}
Copy link
Collaborator

@climbfuji climbfuji Aug 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain why the additional flags -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" are needed?

INSTALL_COMMAND make install
BUILD_ALWAYS FALSE
)

ExternalProject_Add(FMS
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms
GIT_REPOSITORY https://github.com/jcsda/FMS.git
GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git
GIT_TAG 2022.02
GIT_SUBMODULES_RECURSE TRUE
GIT_TAG dev/jcsda
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms
INSTALL_DIR ${DEPEND_LIB_ROOT}
CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT}
Expand All @@ -114,16 +115,18 @@ set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/l
if(UFS_APP MATCHES "^(S2S)$")
# fv3-jedi and associated repositories
# ------------------------------------
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE )
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs )
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs )
add_dependencies(soca ufs-weather-model)
elseif(UFS_APP MATCHES "^(NG-GODAS)$")
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs )
add_dependencies(soca ufs-weather-model)
elseif(UFS_APP MATCHES "^(ATM)$")
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE )
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs )
endif()


Expand All @@ -140,7 +143,7 @@ find_branch_name(REPO_DIR_NAME ioda)
# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
# in a local directory, ioda-data repo will not be cloned
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca )
endif()

# If IODA's current branch is available in ioda-data repo, that branch will be checked out
Expand All @@ -150,7 +153,7 @@ branch_checkout (REPO_DIR_NAME ioda-data
# same procedure for ufo-data
find_branch_name(REPO_DIR_NAME ufo)
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop )
endif()

# If UFO's current branch is available in ufo-data repo, that branch will be checked out
Expand All @@ -160,7 +163,7 @@ branch_checkout (REPO_DIR_NAME ufo-data
# same procedure for fv3-jedi-data
find_branch_name(REPO_DIR_NAME fv3-jedi)
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs )
endif()

# If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out
Expand All @@ -170,7 +173,7 @@ branch_checkout (REPO_DIR_NAME fv3-jedi-data
# same procedure for saber-data
find_branch_name(REPO_DIR_NAME saber)
if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop )
endif()

# If saber's current branch is available in saber-data repo, that branch will be checked out
Expand Down
41 changes: 41 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,43 @@
# ufs-jedi-bundle
Bundle for interfacing UFS models with JEDI interfaces

This bundle requires the following modules be loaded--
compiler gnu 9.0+ or intel 18+
MPI
esmf-8.2+ hdf5 netcdf eckit fckit eigen boost atlas-0.27+ pio mkl/openblas
parallel-netcdf cmake-3.20+ ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite

The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only,
with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS
configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build.

The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line.

To build on Hera, load the following modules--
module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
module load hpc hpc-gnu hpc-mpich
module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost-headers atlas pio ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio


create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired.

ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS ..

While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run.

After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on
Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast--

srun --mpi=pmi2 -n 8 ../../bin/soca_forecast.x testinput/forecast_ufs.yml


The code will build on Cheyenne, but there is some sort of mpi issue when trying to run. Here are the steps to build--

module use /glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack
module load gnu/10.1.0 mpt/2.22 hpc hpc-gnu hpc-mpt/2.22
module load hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 git

git lfs install

ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS ..