diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e100fe..a69980b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") # Initialize ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) +ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.3.0 ) include( jedicmake/cmake/Functions/git_functions.cmake ) # ECMWF libs @@ -42,14 +42,15 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29. # External (required) observation operators # ------------------------------ option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON -ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi UPDATE ) +ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi ) # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 1.0.0 ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 8bee2e68efad0734a95565b14249bf0b7dd4e6f2 ) +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG b38794726d0f3fcfc27b3bff80d590bb451110f0 ) +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 67ab97f61893782fc826c0be05b7ef7678ba7dba ) # Options for building with certain models # ---------------------------------------- @@ -63,9 +64,9 @@ ecbuild_info("Building with UFS application: ${UFS_APP}") set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs ) -include_directories(${DEPEND_LIB_ROOT}/include) +include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) include( ExternalProject ) @@ -74,20 +75,20 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git GIT_SUBMODULES_RECURSE TRUE - GIT_TAG develop + GIT_TAG feature/consolidate SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) ExternalProject_Add(FMS PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms - GIT_REPOSITORY https://github.com/jcsda/FMS.git + GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git + GIT_TAG 2022.02 GIT_SUBMODULES_RECURSE TRUE - GIT_TAG dev/jcsda SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} @@ -114,16 +115,18 @@ set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/l if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) endif() @@ -140,7 +143,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -150,7 +153,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop ) endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -160,7 +163,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out @@ -170,7 +173,7 @@ branch_checkout (REPO_DIR_NAME fv3-jedi-data # same procedure for saber-data find_branch_name(REPO_DIR_NAME saber) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop ) endif() # If saber's current branch is available in saber-data repo, that branch will be checked out diff --git a/README.md b/README.md index 8735c52..ac8bc1b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,43 @@ # ufs-jedi-bundle Bundle for interfacing UFS models with JEDI interfaces + +This bundle requires the following modules be loaded-- +compiler gnu 9.0+ or intel 18+ +MPI +esmf-8.2+ hdf5 netcdf eckit fckit eigen boost atlas-0.27+ pio mkl/openblas +parallel-netcdf cmake-3.20+ ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite + +The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, +with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS +configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. + +The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. + +To build on Hera, load the following modules-- +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc hpc-gnu hpc-mpich +module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost-headers atlas pio ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio + + +create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired. + +ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. + +While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. + +After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on +Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast-- + +srun --mpi=pmi2 -n 8 ../../bin/soca_forecast.x testinput/forecast_ufs.yml + + +The code will build on Cheyenne, but there is some sort of mpi issue when trying to run. Here are the steps to build-- + +module use /glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack +module load gnu/10.1.0 mpt/2.22 hpc hpc-gnu hpc-mpt/2.22 +module load hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 git + +git lfs install + +ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. +