From eb24a031a23bc48d8b5db30d94ae3a106aa6e919 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Tue, 25 Jan 2022 17:29:43 -0500 Subject: [PATCH 01/43] updates to get things building/linking --- CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bcb2b2..e463c81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,11 +74,11 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git GIT_SUBMODULES_RECURSE TRUE - GIT_TAG develop + GIT_TAG feature/soca SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) @@ -117,15 +117,16 @@ if(UFS_APP MATCHES "^(S2S)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) elseif(UFS_APP MATCHES "^(NG-GODAS)$") ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) endif() +add_dependencies(soca ufs-weather-model) # ioda, ufo, fv3-jedi and saber test data #---------------------------------------- From 142975a8d356b51a7e393e98c11901e969889d77 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Wed, 26 Jan 2022 13:34:30 -0500 Subject: [PATCH 02/43] using two different versions of fv3-jedi depending on UFS_APP --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e463c81..d9a5939 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,7 +65,7 @@ set(FV3_FORECAST_MODEL "UFS") # --------------------- ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs UPDATE ) -include_directories(${DEPEND_LIB_ROOT}/include) +include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) include( ExternalProject ) @@ -115,7 +115,7 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs-s2s UPDATE ) ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) elseif(UFS_APP MATCHES "^(NG-GODAS)$") @@ -161,7 +161,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs UPDATE ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out From 2c2a90f212030d2e1d93633dd1b2f8a17976a8fd Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 27 Jan 2022 16:54:37 +0000 Subject: [PATCH 03/43] small change to ioda for expat linking --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9a5939..d2308fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca UPDATE ) endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out From 6e4c053179481d1dd39f69f4d2a1839f86d648de Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Thu, 27 Jan 2022 17:10:33 -0500 Subject: [PATCH 04/43] fix for building in container --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9a5939..d3b20ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ ExternalProject_Add(ufs-weather-model SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) From eb61ff0f264b47e83410d705eb1aa74c2baf5653 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Fri, 18 Mar 2022 13:12:43 -0400 Subject: [PATCH 05/43] update for godas --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d416769..b0636cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,7 +74,7 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/soca + GIT_TAG feature/godas SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} From 25ade2465402e09ca311ea61af08e80fd88c605b Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Fri, 24 Jun 2022 19:24:28 -0400 Subject: [PATCH 06/43] Updating with consolidated versions for atm/godas/s2s builds --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0636cb..46f840f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BR # Core JEDI repositories # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) @@ -74,11 +75,11 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/godas + GIT_TAG feature/consolidate SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) @@ -115,18 +116,19 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs-s2s UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) endif() -add_dependencies(soca ufs-weather-model) # ioda, ufo, fv3-jedi and saber test data #---------------------------------------- From aa86ef20887f995ffa71dc2372ac11a66070c3cf Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 30 Jun 2022 12:27:41 +0000 Subject: [PATCH 07/43] Updated atlas version and README --- CMakeLists.txt | 2 +- README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46f840f..2904917 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless u ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.27.0 ) # External (required) observation operators # ------------------------------ diff --git a/README.md b/README.md index 8735c52..aa611fe 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,20 @@ # ufs-jedi-bundle Bundle for interfacing UFS models with JEDI interfaces + +This bundle requires the following modules be loaded-- +compiler gnu 9.0+ or intel 18+ +MPI +esmf-8.2+ hdf5 netcdf eckit fckit eigen boost atlas-0.27+ pio mkl/openblas +parallel-netcdf cmake-3.20+ ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite + +The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, +with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS +configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. + +The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. + +The following ecbuild line will configure the NG-GODAS application on Hera-- + +ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. + + From a5851a4226aa387d0f5981d6177f83e9d8c9fddd Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 30 Jun 2022 12:38:05 +0000 Subject: [PATCH 08/43] updated readme --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aa611fe..9bbd88f 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,21 @@ configured with S2S coupling. Only the first two configurations have test cases The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. -The following ecbuild line will configure the NG-GODAS application on Hera-- +To build on Hera, load the following modules-- +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc hpc-gnu hpc-mpich +module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost atlas pio intel-mkl parallel-netcdf ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio expat + +create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired. ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. +While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. + +After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on +Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast-- + +srun --mpi=pmi2 -n 8 /scratch1/NCEPDEV/da/Mark.Potts/jedi/ufs-jedi-bundle/build-gnu/bin/soca_forecast.x testinput/forecast_ufs.yml + + From c92d43c66d0ff0406cc06a8da0d128e375beb39a Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Thu, 30 Jun 2022 14:02:25 -0600 Subject: [PATCH 09/43] updated atlas version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46f840f..2904917 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless u ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.27.0 ) # External (required) observation operators # ------------------------------ From e9dbfe03bebd96f0620639807233c9a6ab0da4e3 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 30 Jun 2022 21:38:36 +0000 Subject: [PATCH 10/43] updated README for cheyenne --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9bbd88f..9faf4cd 100644 --- a/README.md +++ b/README.md @@ -16,18 +16,26 @@ The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S) To build on Hera, load the following modules-- module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc hpc-gnu hpc-mpich -module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost atlas pio intel-mkl parallel-netcdf ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio expat +module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost-headers atlas pio ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio + create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired. -ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. +ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast-- -srun --mpi=pmi2 -n 8 /scratch1/NCEPDEV/da/Mark.Potts/jedi/ufs-jedi-bundle/build-gnu/bin/soca_forecast.x testinput/forecast_ufs.yml +srun --mpi=pmi2 -n 8 ../../bin/soca_forecast.x testinput/forecast_ufs.yml + + +The code will build on Cheyenne, but there is some sort of mpi issue when trying to run. Here are the steps to build-- +module use /glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack +module load gnu/10.1.0 mpt/2.22 hpc hpc-gnu hpc-mpt/2.22 +hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 +ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. From 0cd1352856bde7eae26b29d7996ccdac14521666 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 30 Jun 2022 21:39:30 +0000 Subject: [PATCH 11/43] updated README for cheyenne --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9faf4cd..ac8bc1b 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ The code will build on Cheyenne, but there is some sort of mpi issue when trying module use /glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack module load gnu/10.1.0 mpt/2.22 hpc hpc-gnu hpc-mpt/2.22 -hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 +module load hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 git + +git lfs install ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. From ea2387bdd302f645caef855e570c2f4d34eb939e Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 30 Jun 2022 22:29:49 +0000 Subject: [PATCH 12/43] Changed to hashes --- CMakeLists.txt | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2904917..deb5cbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") # Initialize ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.3.0 ) include( jedicmake/cmake/Functions/git_functions.cmake ) # ECMWF libs @@ -42,15 +42,15 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.27. # External (required) observation operators # ------------------------------ option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON -ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi UPDATE ) +ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi ) # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 1.0.0 ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 8bee2e68efad0734a95565b14249bf0b7dd4e6f2 ) +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG b38794726d0f3fcfc27b3bff80d590bb451110f0 ) +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 67ab97f61893782fc826c0be05b7ef7678ba7dba ) # Options for building with certain models # ---------------------------------------- @@ -64,7 +64,7 @@ ecbuild_info("Building with UFS application: ${UFS_APP}") set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs ) include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -115,18 +115,18 @@ set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/l if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) endif() @@ -143,7 +143,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca UPDATE ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -153,7 +153,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop ) endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -163,7 +163,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out @@ -173,7 +173,7 @@ branch_checkout (REPO_DIR_NAME fv3-jedi-data # same procedure for saber-data find_branch_name(REPO_DIR_NAME saber) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop ) endif() # If saber's current branch is available in saber-data repo, that branch will be checked out From 31e2a880650cca4da4575053fba8ea59b2348fe0 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Fri, 29 Jul 2022 13:05:34 -0400 Subject: [PATCH 13/43] Switched version of FMS. Might be able to remove it as an external now --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index deb5cbb..a55047a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,9 +86,9 @@ ExternalProject_Add(ufs-weather-model ExternalProject_Add(FMS PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms - GIT_REPOSITORY https://github.com/jcsda/FMS.git + GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git + GIT_TAG 2022.02 GIT_SUBMODULES_RECURSE TRUE - GIT_TAG dev/jcsda SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} From 18be977f60750fc20d7dc0beb6af782fc7956006 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 18 Oct 2022 11:24:00 -0600 Subject: [PATCH 14/43] Add vader and ufs-weather-model to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index de20ac4..53ea98e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ ioda/ oops/ ufo/ ewok/ +vader/ # FMS fms/ @@ -41,6 +42,7 @@ fv3/ femps/ fv3-jedi-lm/ fv3-jedi/ +ufs-weather-model/ # Observation operators crtm/ From dd3c425cc02ce8dd6ced6857dc90828e05d14e19 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 18 Oct 2022 11:24:14 -0600 Subject: [PATCH 15/43] Update CMakeLists.txt --- CMakeLists.txt | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a69980b..c3e8117 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,14 @@ # (C) Copyright 2021-2021 NOAA # ########## -# ufs-jedi-bundle +# ufs-bundle # ########## cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) -project( ufs-jedi-bundle VERSION 1.1.0 LANGUAGES C CXX Fortran ) +project( ufs-bundle VERSION 1.1.0 LANGUAGES C CXX Fortran ) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -24,10 +24,13 @@ set( ECBUILD_DEFAULT_BUILD_TYPE Release ) # Enable MPI set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") -# Initialize -ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.3.0 ) -include( jedicmake/cmake/Functions/git_functions.cmake ) +# Use external jedi-cmake or build in bundle +if(DEFINED ENV{jedi_cmake_ROOT}) + include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake ) +else() + ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) + include( jedicmake/cmake/Functions/git_functions.cmake ) +endif() # ECMWF libs # ---------- @@ -41,9 +44,12 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29. # External (required) observation operators # ------------------------------ -option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON +option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "ON") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi ) +# External FMS +find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) + # Core JEDI repositories # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) @@ -79,33 +85,33 @@ ExternalProject_Add(ufs-weather-model SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) -ExternalProject_Add(FMS - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms - GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git - GIT_TAG 2022.02 - GIT_SUBMODULES_RECURSE TRUE - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms - INSTALL_DIR ${DEPEND_LIB_ROOT} - CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} - INSTALL_COMMAND make install - BUILD_ALWAYS TRUE - ) - -add_dependencies(ufs-weather-model FMS ) +#ExternalProject_Add(FMS +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms +# GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git +# GIT_TAG 2022.02 +# GIT_SUBMODULES_RECURSE TRUE +# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms +# INSTALL_DIR ${DEPEND_LIB_ROOT} +# CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} +# INSTALL_COMMAND make install +# BUILD_ALWAYS TRUE +# ) + +#add_dependencies(ufs-weather-model FMS ) add_library( ufs IMPORTED STATIC) -add_library( FMS::fms_r4 IMPORTED STATIC) -add_library( FMS::fms_r8 IMPORTED STATIC) +#add_library( FMS::fms_r4 IMPORTED STATIC) +#add_library( FMS::fms_r8 IMPORTED STATIC) add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) add_library( fv3atm IMPORTED STATIC) add_library( ccppphys IMPORTED STATIC) -set_target_properties(FMS::fms_r4 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r4.a) -set_target_properties(FMS::fms_r8 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r8.a) +#set_target_properties(FMS::fms_r4 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r4.a) +#set_target_properties(FMS::fms_r8 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r8.a) set_target_properties(ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_framework.a) set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libstochastic_physics.a) From e1a5aee0809a17a05fd3d2df7a4c841a2614d028 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 18 Oct 2022 11:47:40 -0600 Subject: [PATCH 16/43] Update CMakeLists.txt --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3e8117..e329ffd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,7 @@ ecbuild_info("Building with UFS application: ${UFS_APP}") set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -79,9 +79,9 @@ include( ExternalProject ) ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model - GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git + GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/consolidate + GIT_TAG feature/consolidate_dom SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} @@ -122,7 +122,7 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) add_dependencies(soca ufs-weather-model) From 3a81f4af3f360ad07654808efdc8aa819d443682 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 18 Oct 2022 13:17:20 -0600 Subject: [PATCH 17/43] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e329ffd..55c3ac6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,7 +132,7 @@ elseif(UFS_APP MATCHES "^(NG-GODAS)$") add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) endif() From 089ad9c21742da4d3b2bb915117d6340ab0e8fff Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 19 Oct 2022 14:17:54 -0600 Subject: [PATCH 18/43] More updates to CMakeLists.txt --- CMakeLists.txt | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55c3ac6..dad7f02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,11 +42,6 @@ ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.18. ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.5 ) ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29.0 ) -# External (required) observation operators -# ------------------------------ -option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "ON") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON -ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi ) - # External FMS find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) @@ -54,7 +49,7 @@ find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 1.0.0 ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 8bee2e68efad0734a95565b14249bf0b7dd4e6f2 ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG b38794726d0f3fcfc27b3bff80d590bb451110f0 ) ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 67ab97f61893782fc826c0be05b7ef7678ba7dba ) @@ -85,33 +80,16 @@ ExternalProject_Add(ufs-weather-model SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DMPI_Fortran_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) -#ExternalProject_Add(FMS -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms -# GIT_REPOSITORY https://github.com/NOAA-GFDL/FMS.git -# GIT_TAG 2022.02 -# GIT_SUBMODULES_RECURSE TRUE -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms -# INSTALL_DIR ${DEPEND_LIB_ROOT} -# CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} -# INSTALL_COMMAND make install -# BUILD_ALWAYS TRUE -# ) - -#add_dependencies(ufs-weather-model FMS ) add_library( ufs IMPORTED STATIC) -#add_library( FMS::fms_r4 IMPORTED STATIC) -#add_library( FMS::fms_r8 IMPORTED STATIC) add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) add_library( fv3atm IMPORTED STATIC) add_library( ccppphys IMPORTED STATIC) -#set_target_properties(FMS::fms_r4 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r4.a) -#set_target_properties(FMS::fms_r8 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r8.a) set_target_properties(ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_framework.a) set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libstochastic_physics.a) From a75ca115d163d416ef9d25d05e8083da87fdb966 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 19 Oct 2022 16:45:34 -0600 Subject: [PATCH 19/43] Work in progress: README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index ac8bc1b..b43103d 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,15 @@ create a build directory under ufs-jedi-bundle and cd to it. Then run the follow ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. +# NOTE: WITH SPACK-STACK, I DIDN'T NEED ANY OF THIS ... just "ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild" + While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast-- +NO, FIX AND USE -DMPIEXEC_EXECUTABLE ETC. THEN SALLOC + srun --mpi=pmi2 -n 8 ../../bin/soca_forecast.x testinput/forecast_ufs.yml From 4efaf347383e60212ae41ab4b506828782c14f92 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 20 Oct 2022 08:57:28 -0600 Subject: [PATCH 20/43] Update soca branch in CMakeLists.txt --- .gitignore | 1 + CMakeLists.txt | 4 ++-- README.md | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 53ea98e..9454213 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ ioda/ oops/ ufo/ ewok/ +soca/ vader/ # FMS diff --git a/CMakeLists.txt b/CMakeLists.txt index dad7f02..5dd2601 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,11 +102,11 @@ if(UFS_APP MATCHES "^(S2S)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) diff --git a/README.md b/README.md index b43103d..cd9c600 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2 # NOTE: WITH SPACK-STACK, I DIDN'T NEED ANY OF THIS ... just "ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild" +ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild + +ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild + While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on From 28f8b378ae54a8bb335aa5ab20c22a531efc8360 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 20 Oct 2022 15:05:57 -0600 Subject: [PATCH 21/43] Use MPI compiler wrappers for external project ufs-weather-model --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dd2601..eb38fae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,7 @@ ExternalProject_Add(ufs-weather-model SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) From 50bb3e793088a506f536aeba931b2f3af0225588 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 20 Oct 2022 15:50:22 -0600 Subject: [PATCH 22/43] Attempt to fix dependency for gsw in CMakeLists.txt --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb38fae..148753b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,10 @@ set(FV3_FORECAST_MODEL "UFS") # --------------------- ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) +if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) +endif() + include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -85,6 +89,7 @@ ExternalProject_Add(ufs-weather-model BUILD_ALWAYS FALSE ) +add_dependencies(ufs-weather-model gsw) add_library( ufs IMPORTED STATIC) add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) @@ -101,11 +106,11 @@ if(UFS_APP MATCHES "^(S2S)$") # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") From b0b1ee0b68702152a8553ff91b7190fa9250a376 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 21 Oct 2022 12:21:28 -0600 Subject: [PATCH 23/43] Update README.md --- README.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd9c600..7b12801 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,75 @@ +# ufs-jedi-bundle + +Bundle for interfacing UFS models with JEDI interfaces + +This bundle requires the following spack-stack modules be loaded (not all are in skylab-2.0.0 environment): + +heinzell@JCSDA-L-18146:~/work/ufs-bundle/ufs-bundle/build-debug-ng-godas [brew-x86_64]> module li + +Currently Loaded Modules: + 1) stack-apple-clang/13.1.6 19) base-env/1.0.0 37) hdf/4.2.15 55) py-pycparser/2.20 73) py-pandas/1.4.0 91) py-botocore/1.13.44 109) py-pyshp/2.1.0 127) g2/3.4.5 + 2) pmix/4.1.2 20) boost/1.78.0 38) jedi-cmake/1.4.0 56) py-cffi/1.15.0 74) py-pybind11/2.8.1 92) py-s3transfer/0.2.1 110) py-shapely/1.8.0 128) g2tmpl/1.10.0 + 3) zlib/1.2.12 21) bufr/11.7.1 39) libpng/1.6.37 57) py-findlibs/0.0.2 75) py-pycodestyle/2.8.0 93) py-boto3/1.10.44 111) py-cartopy/0.20.2 129) ip/3.3.3 + 4) openmpi/4.1.4 22) git-lfs/3.1.4 40) libxt/1.1.5 58) py-setuptools/59.4.0 76) py-pyhdf/0.10.4 94) brunsli/0.1 112) py-markupsafe/2.0.1 130) gftl-shared/1.5.0 + 5) stack-openmpi/4.1.4 23) ecbuild/3.6.5 41) libxmu/1.1.2 59) py-numpy/1.22.3 77) py-pyyaml/6.0 95) geos/3.9.1 113) py-jinja2/3.0.3 131) yafyaml/0.5.1 + 6) stack-python/3.9.13 24) openjpeg/2.3.1 42) pigz/2.7 60) py-eccodes/1.4.2 78) llvm-openmp/14.0.6 96) json-c/0.16 114) py-ruamel-yaml/0.17.16 132) mapl/2.22.0-esmf-8.3.0b09-esmf-8.3.0 + 7) cmake/3.23.1 25) eccodes/2.27.0 43) zstd/1.5.2 61) py-f90nml/1.4.2 79) py-gast/0.5.3 97) lerc/3.0 115) py-ruamel-yaml-clib/0.2.4 133) ufs-pyenv/1.0.0 + 8) curl/7.83.0 26) eigen/3.4.0 44) tar/1.34 62) py-mpi4py/3.1.2 80) py-beniget/0.4.1 98) sqlite/3.38.5 116) jedi-ewok-env/1.0.0 134) w3nco/2.4.1 + 9) git/2.36.0 27) openblas/0.3.19 45) gettext/0.21 63) py-h5py/3.6.0 81) py-ply/3.11 99) proj/8.1.0 117) antlr/2.7.7 135) ufs-weather-model-env/1.0.0 + 10) pkg-config/0.29.2 28) eckit/1.19.0 46) libxpm/3.5.12 64) py-cftime/1.0.3.4 82) py-pythran/0.11.0 100) gdal/3.5.1 118) gsl/2.7.1 136) crtm/v2.4_jedi + 11) hdf5/1.12.1 29) fftw/3.3.10 47) libxaw/1.0.13 65) py-netcdf4/1.5.3 83) py-scipy/1.8.0 101) py-cycler/0.11.0 119) nco/5.0.6 137) w3emc/2.9.1 + 12) libbacktrace/2020-02-19 30) fckit/0.9.5 48) udunits/2.2.28 66) py-bottleneck/1.3.2 84) py-xarray/2022.3.0 102) py-fonttools/4.31.2 120) soca-env/1.0.0 138) nemsio/2.5.2 + 13) parallel-netcdf/1.12.2 31) fiat/1.0.0 49) ncview/2.1.8 67) py-pyparsing/3.0.6 85) jedi-base-env/1.0.0 103) py-kiwisolver/1.3.2 121) sp/2.3.3 + 14) netcdf-c/4.8.1 32) ectrans/1.0.0 50) netcdf-cxx4/4.3.1 68) py-packaging/21.3 86) jedi-fv3-env/1.0.0 104) py-pillow/9.2.0 122) sigio/2.3.2 + 15) nccmp/1.9.0.1 33) atlas/0.30.0 51) json/3.10.5 69) py-numexpr/2.8.3 87) ecflow/5.8.3 105) py-matplotlib/3.5.3 123) fms/2022.01 + 16) netcdf-fortran/4.5.4 34) gsibec/1.0.5 52) json-schema-validator/2.1.0 70) py-six/1.16.0 88) py-docutils/0.18.1 106) py-certifi/2021.10.8 124) bacio/2.4.1 + 17) parallelio/2.5.7 35) gsl-lite/0.37.0 53) odc/1.4.5 71) py-python-dateutil/2.8.2 89) py-jmespath/0.10.0 107) py-cython/0.29.30 125) esmf/8.3.0b09 + 18) wget/1.21.3 36) libjpeg/2.1.0 54) py-attrs/21.4.0 72) py-pytz/2021.3 90) py-urllib3/1.25.9 108) py-pyproj/3.1.0 126) jasper/2.0.32 + +Basically, do this: + +module use /Users/heinzell/prod/spack-stack-v1/envs/skylab-2.0.0-plus-ufs/install/modulefiles/Core +module load stack-apple-clang/13.1.6 +module load stack-openmpi/4.1.4 +module load stack-python/3.9.13 +module load jedi-fv3-env/1.0.0 +module load jedi-ewok-env/1.0.0 +module load soca-env/1.0.0 +# Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM +module load ufs-weather-model-env/1.0.0 +# Need to reload JEDI CRTM +module load crtm/v2.4_jedi +module load sigio/2.3.2 +module load w3emc/2.9.1 +module load nemsio/2.5.2 + +The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, +with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS +configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. + +The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. Create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired): + +ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild +ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild +# Note - not yet tested on macOS with spack-stack +#ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild + +While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. + +After configuration, run "make -j 8" to build. + +The ctests for ATM are called fv3jedi_test_tier1_model_ufs and fv3jedi_test_tier1_forecast_ufs. +The ctest for NG-GODAS is called test_soca_forecast_ufs. + +Note for later for running on Hera: need to add -DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" to cmake/ecbuild command, then run ctest -R get_ on login node, then acquire compute node, for example via salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu, then run the actual ctests on login node. + + + + + + +#### OLD STUFF FROM MARK FOR BUILDING ON HERA WITH OLD SOFTWARE STACK + # ufs-jedi-bundle Bundle for interfacing UFS models with JEDI interfaces @@ -18,12 +90,12 @@ module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc hpc-gnu hpc-mpich module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost-headers atlas pio ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio - create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired. ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. -# NOTE: WITH SPACK-STACK, I DIDN'T NEED ANY OF THIS ... just "ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild" + +NOTE: WITH SPACK-STACK, I DIDN'T NEED ANY OF THIS ... just do "ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild" ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild From 2b7866f46fd5120b01616e3d04271f40c7691554 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 26 Oct 2022 14:12:39 -0600 Subject: [PATCH 24/43] Add missing imported library statements to CMakeLists.txt, update README.md --- CMakeLists.txt | 6 +++++- README.md | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 148753b..957ed19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,11 +91,15 @@ ExternalProject_Add(ufs-weather-model add_dependencies(ufs-weather-model gsw) add_library( ufs IMPORTED STATIC) +add_library( cdeps IMPORTED STATIC) +add_library( cmeps IMPORTED STATIC) add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) add_library( fv3atm IMPORTED STATIC) add_library( ccppphys IMPORTED STATIC) -set_target_properties(ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) +set_target_properties( ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) +set_target_properties( cdeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcdeps.a) +set_target_properties( cmeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcmeps.a) set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_framework.a) set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libstochastic_physics.a) set_target_properties(fv3atm PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfv3atm.a) diff --git a/README.md b/README.md index 7b12801..6115196 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild # Note - not yet tested on macOS with spack-stack #ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild +On macOS, it may be required to pass -DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind" to the ecbuild command. + While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. After configuration, run "make -j 8" to build. From 25164955f436feabe1322f9d3b7d8914419d6614 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 26 Oct 2022 20:57:04 -0600 Subject: [PATCH 25/43] Catch unknown UFS_APP in top-level CMakeLists.txt --- CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 957ed19..352ced5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,9 @@ ExternalProject_Add(ufs-weather-model BUILD_ALWAYS FALSE ) -add_dependencies(ufs-weather-model gsw) +if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") + add_dependencies(ufs-weather-model gsw) +endif() add_library( ufs IMPORTED STATIC) add_library( cdeps IMPORTED STATIC) add_library( cmeps IMPORTED STATIC) @@ -110,16 +112,16 @@ if(UFS_APP MATCHES "^(S2S)$") # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) - #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - #ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) +else() + message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() From 48d5a7dbc3eed5d0bee338fec3fba73fabc64f4e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 1 Nov 2022 17:28:24 -0600 Subject: [PATCH 26/43] Point to branch feature/ufs_dom of fv3-jedi-data --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 352ced5..3134f66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,7 +158,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out From 73d419258fe48039a4360c39516fd30e5601c294 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 1 Nov 2022 17:28:39 -0600 Subject: [PATCH 27/43] WIP on README.md --- README.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6115196..c3af65e 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,39 @@ module load jedi-ewok-env/1.0.0 module load soca-env/1.0.0 # Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM module load ufs-weather-model-env/1.0.0 -# Need to reload JEDI CRTM +# Need to reload JEDI CRTM - do NOT reload FMS, we need fms/2022.01 module load crtm/v2.4_jedi module load sigio/2.3.2 -module load w3emc/2.9.1 +module load w3emc/2.9.1 # # So far not on macOS: 2.9.2 module load nemsio/2.5.2 +# So far not on macOS: module load fms/2022.02 + + + +# ORION +module purge +module use /work/noaa/da/role-da/spack-stack/modulefiles +module load miniconda/3.9.7 +module load ecflow/5.8.4 + +module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-plus-ufs-intel-2022.0.2/install/modulefiles/Core +module load stack-intel/2022.0.2 +module load stack-intel-oneapi-mpi/2021.5.1 +module load stack-python/3.9.7 + +module load jedi-fv3-env/1.0.0 +module load jedi-ewok-env/1.0.0 +module load soca-env/1.0.0 +# Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM +module load ufs-weather-model-env/1.0.0 +# Need to reload JEDI CRTM - do NOT reload FMS, we need fms/2022.01 +module load crtm/v2.4_jedi +module load sigio/2.3.2 +module load w3emc/2.9.2 +module load nemsio/2.5.2 +module load fms/2022.02 + +cd /work2/noaa/da/dheinzel/ufs-bundle/build-debug-atm The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS @@ -56,6 +84,7 @@ ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild On macOS, it may be required to pass -DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind" to the ecbuild command. + While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. After configuration, run "make -j 8" to build. From 266c9a30d882422232984ee088dd540d04518492 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 20 Nov 2022 20:37:15 -0700 Subject: [PATCH 28/43] Update CMakeLists.txt, now s2s compiles --- CMakeLists.txt | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3134f66..5baccfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,10 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29. # External FMS find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) +if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) +endif() + # Core JEDI repositories # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) @@ -67,10 +71,6 @@ set(FV3_FORECAST_MODEL "UFS") # --------------------- ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) -if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) -endif() - include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -80,11 +80,12 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/consolidate_dom + GIT_TAG feature/consolidate_dom_update_20221114 SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model + UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) @@ -99,6 +100,7 @@ add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) add_library( fv3atm IMPORTED STATIC) add_library( ccppphys IMPORTED STATIC) +add_library( mom6 IMPORTED STATIC) set_target_properties( ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) set_target_properties( cdeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcdeps.a) set_target_properties( cmeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcmeps.a) @@ -106,12 +108,13 @@ set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/l set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libstochastic_physics.a) set_target_properties(fv3atm PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfv3atm.a) set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_physics.a) +set_target_properties(mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libmom6.a) if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) # feature/ufs_dom ) ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") @@ -119,7 +122,7 @@ elseif(UFS_APP MATCHES "^(NG-GODAS)$") add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) # feature/ufs_dom ) else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() From 8d5c55185e6bcf42fd69cc73a4b3cef6210a017d Mon Sep 17 00:00:00 2001 From: Francois Hebert Date: Mon, 21 Nov 2022 15:36:29 +0000 Subject: [PATCH 29/43] Update JEDI repo versions in bundle CMakeLists --- CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5baccfd..0321bcc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,7 @@ option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless u ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.18.2 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.5 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29.0 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.30.0 ) # External FMS find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) @@ -51,11 +51,11 @@ endif() # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 8ef84459463663fcdef38cdb94e67dc915ea80bc ) -ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 1.0.0 ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG c308945cbc6db4d6add0dac3581bbc4d457e8785 ) # develop on nov 15 +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3363fe28ba65a61227ec1d64504d2641730f51a4 ) # develop on nov 15 ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG b38794726d0f3fcfc27b3bff80d590bb451110f0 ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 67ab97f61893782fc826c0be05b7ef7678ba7dba ) +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 876e7874fc917376500822157cda82c15ede02f8 ) # develop on nov 15 +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG f95dd253a850debd1f1b391ec63a47f99c341efa ) # develop on nov 15 # Options for building with certain models # ---------------------------------------- @@ -115,10 +115,10 @@ if(UFS_APP MATCHES "^(S2S)$") # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) # feature/ufs_dom ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # was NOT updated on nov 15th add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # was NOT updated on nov 15th add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) @@ -141,7 +141,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) # was NOT updated on nov 15th endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out From 7bf14ba518476831d3959c584955d88385c40f9e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Nov 2022 09:10:55 -0700 Subject: [PATCH 30/43] Update README.md --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c3af65e..7c15e53 100644 --- a/README.md +++ b/README.md @@ -37,14 +37,12 @@ module load jedi-ewok-env/1.0.0 module load soca-env/1.0.0 # Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM module load ufs-weather-model-env/1.0.0 -# Need to reload JEDI CRTM - do NOT reload FMS, we need fms/2022.01 +# Need to reload JEDI CRTM - and fms/2022.02 +fpic module load crtm/v2.4_jedi +module laod fms/2022.02 module load sigio/2.3.2 -module load w3emc/2.9.1 # # So far not on macOS: 2.9.2 +module load w3emc/2.9.2 module load nemsio/2.5.2 -# So far not on macOS: module load fms/2022.02 - - # ORION module purge @@ -62,12 +60,12 @@ module load jedi-ewok-env/1.0.0 module load soca-env/1.0.0 # Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM module load ufs-weather-model-env/1.0.0 -# Need to reload JEDI CRTM - do NOT reload FMS, we need fms/2022.01 +# Need to reload JEDI CRTM - and fms/2022.02 +fpic module load crtm/v2.4_jedi +module laod fms/2022.02 module load sigio/2.3.2 module load w3emc/2.9.2 module load nemsio/2.5.2 -module load fms/2022.02 cd /work2/noaa/da/dheinzel/ufs-bundle/build-debug-atm From ebdfa59a87c2ab1d1a3692ecc1424a204c7fe837 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Nov 2022 10:10:08 -0700 Subject: [PATCH 31/43] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5baccfd..229a144 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,7 +161,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out From f6d7324f8c3d51177e6354efda000c97cd7d4fdc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Nov 2022 10:58:15 -0700 Subject: [PATCH 32/43] Update CMakeLists.txt --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5801e9b..b6ee0ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") if(DEFINED ENV{jedi_cmake_ROOT}) include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake ) else() - ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) + ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.4.0 UPDATE RECURSIVE ) include( jedicmake/cmake/Functions/git_functions.cmake ) endif() @@ -46,14 +46,14 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.30. find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG ac1d9aee7042a745a6f88dc93e179882e0cd81ee ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on nov 15 endif() # Core JEDI repositories # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG c308945cbc6db4d6add0dac3581bbc4d457e8785 ) # develop on nov 15 ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3363fe28ba65a61227ec1d64504d2641730f51a4 ) # develop on nov 15 -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) # updated from develop on nov 15 ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 876e7874fc917376500822157cda82c15ede02f8 ) # develop on nov 15 ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG f95dd253a850debd1f1b391ec63a47f99c341efa ) # develop on nov 15 @@ -69,7 +69,7 @@ ecbuild_info("Building with UFS application: ${UFS_APP}") set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -80,7 +80,7 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/consolidate_dom_update_20221114 + GIT_TAG feature/consolidate_dom_update_20221114 # updated from develop on nov 14 SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} @@ -114,7 +114,7 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) # feature/ufs_dom ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # was NOT updated on nov 15th add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") @@ -122,7 +122,7 @@ elseif(UFS_APP MATCHES "^(NG-GODAS)$") add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) # feature/ufs_dom ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() @@ -141,7 +141,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) # was NOT updated on nov 15th + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) # updated from develop on nov 15 endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -151,7 +151,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" TAG 998ac3b375248b850ca38ea05749b37a411dac4d ) # develop on nov 15 endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -161,7 +161,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom_temp_without_francois_updates_20221120 ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs ) # updated from develop on nov 15 endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out @@ -171,7 +171,7 @@ branch_checkout (REPO_DIR_NAME fv3-jedi-data # same procedure for saber-data find_branch_name(REPO_DIR_NAME saber) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop ) + ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" TAG 16da933fe4434f73fa51c000db787061d25dcedf ) # develop on nov 15 endif() # If saber's current branch is available in saber-data repo, that branch will be checked out From a517472134b309016be39925f18dd61cedc0e020 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Nov 2022 18:24:46 -0700 Subject: [PATCH 33/43] Update CMakeLists.txt --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6ee0ee..3c8542f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,7 +69,7 @@ ecbuild_info("Building with UFS application: ${UFS_APP}") set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22 include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -141,7 +141,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH feature/soca ) # updated from develop on nov 15 + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" TAG 2d9fb746b26a306711800960f471cb2ddc56c15c ) # develop on nov 22 endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -161,7 +161,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs ) # updated from develop on nov 15 + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out From a675dc8558c1f5a11129b4d81dc6e68dc86625d9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 17 Jan 2023 20:23:50 -0700 Subject: [PATCH 34/43] Update ufs-weather-model branch name --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c8542f..1a898b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,7 @@ ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model GIT_SUBMODULES_RECURSE TRUE - GIT_TAG feature/consolidate_dom_update_20221114 # updated from develop on nov 14 + GIT_TAG feature/consolidate_dom_update_20230117 # updated from develop on Jan 17 SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} From 8092f797e0897602736996e7e76b40f66821c794 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 18 Jan 2023 12:14:43 -0700 Subject: [PATCH 35/43] macOS OpenMP flags for ufs-weather-model --- CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a898b4..227b13e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,17 @@ link_directories(${DEPEND_LIB_ROOT}/lib) include( ExternalProject ) +# Needed to get correct OpenMP link libraries on macOS +set(UFS_CMAKE_EXE_LINKER_FLAGS "-L${CMAKE_CURRENT_BINARY_DIR}/lib") +find_package(OpenMP REQUIRED) +if(APPLE) + set(UFS_CMAKE_EXE_LINKER_FLAGS "${UFS_CMAKE_EXE_LINKER_FLAGS} ${OpenMP_libomp_LIBRARY} ${OpenMP_libomp_LIBRARY}") +endif() +message (INFO "OpenMP_gomp_LIBRARY: ${OpenMP_gomp_LIBRARY}") +message (INFO "OpenMP_libomp_LIBRARY: ${OpenMP_libomp_LIBRARY}") +message (INFO "OpenMP_EXE_LINKER_FLAGS: ${OpenMP_EXE_LINKER_FLAGS}") +message (INFO "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") + ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model @@ -84,8 +95,7 @@ ExternalProject_Add(ufs-weather-model SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} - CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}/lib -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) From e695f93573366097e1d11ab75c4f6594752bc252 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Jan 2023 06:45:42 -0700 Subject: [PATCH 36/43] Fix macOS OpenMP linker issue --- CMakeLists.txt | 8 +++----- README.md | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 227b13e..db1b946 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ set( ECBUILD_DEFAULT_BUILD_TYPE Release ) # Enable MPI set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") +set( ENABLE_OMP ON CACHE BOOL "Compile with OpenMP") # Use external jedi-cmake or build in bundle if(DEFINED ENV{jedi_cmake_ROOT}) @@ -65,8 +66,8 @@ if(NOT UFS_APP MATCHES "^(ATM|NG-GODAS|S2S)$") ecbuild_error("Valid options for -DUFS_APP are ATM|NG-GODAS|S2S") endif() ecbuild_info("Building with UFS application: ${UFS_APP}") - set(FV3_FORECAST_MODEL "UFS") + # fv3-jedi linear model # --------------------- ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22 @@ -82,10 +83,7 @@ find_package(OpenMP REQUIRED) if(APPLE) set(UFS_CMAKE_EXE_LINKER_FLAGS "${UFS_CMAKE_EXE_LINKER_FLAGS} ${OpenMP_libomp_LIBRARY} ${OpenMP_libomp_LIBRARY}") endif() -message (INFO "OpenMP_gomp_LIBRARY: ${OpenMP_gomp_LIBRARY}") -message (INFO "OpenMP_libomp_LIBRARY: ${OpenMP_libomp_LIBRARY}") -message (INFO "OpenMP_EXE_LINKER_FLAGS: ${OpenMP_EXE_LINKER_FLAGS}") -message (INFO "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") +message (SATUS "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model diff --git a/README.md b/README.md index 7c15e53..04618ac 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,7 @@ ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild # Note - not yet tested on macOS with spack-stack #ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild -On macOS, it may be required to pass -DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind" to the ecbuild command. - +On macOS, it may be required to pass -DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind" to the ecbuild command. Further, if using the native Apple clang compiler with llvm-openmp installed via homebrew or spack, it may be necessary to add -DCMAKE_SHARED_LINKER_FLAGS="/path/to/llvm-openmp-x.y.z/lib/libomp.dylib" to the ecbuild command. While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. From 842d20e03cdb9a4c7c867ced3c1cb7402866e8eb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Jan 2023 06:45:57 -0700 Subject: [PATCH 37/43] Temporarily add PRs.txt TMP_README_UFS_STANDALONE.sh --- PRs.txt | 11 +++++++++++ TMP_README_UFS_STANDALONE.sh | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 PRs.txt create mode 100755 TMP_README_UFS_STANDALONE.sh diff --git a/PRs.txt b/PRs.txt new file mode 100644 index 0000000..39add6a --- /dev/null +++ b/PRs.txt @@ -0,0 +1,11 @@ +https://github.com/JCSDA-internal/fv3-jedi-linearmodel/pull/16 +https://github.com/JCSDA-internal/fv3-jedi/pull/697 +https://github.com/JCSDA-internal/soca/pull/824 +https://github.com/JCSDA-internal/fv3-jedi-data/pull/52 +https://github.com/JCSDA-internal/saber/pull/385 + +ufs-weather-model PRs: +https://github.com/NOAA-EMC/AQM/pull/40 +https://github.com/NOAA-EMC/MOM6/pull/107 +FV3 PR missing! +https://github.com/ufs-community/ufs-weather-model/pull/1504 diff --git a/TMP_README_UFS_STANDALONE.sh b/TMP_README_UFS_STANDALONE.sh new file mode 100755 index 0000000..bc2abc1 --- /dev/null +++ b/TMP_README_UFS_STANDALONE.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# cd /mnt/experiments-efs/the-real-dom/ufs/ +# source setup.sh +# git clone -b feature/consolidate_dom_update_20221114 --recursive https://github.com/climbfuji/ufs-weather-model ufs-weather-model-feature-consolidate_dom_update_20221114 +# mkdir build-atm +# cd build-atm +# cmake -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16 -DMPI=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF ../ufs-weather-model-feature-consolidate_dom_update_20221114 2>&1 | tee log.cmake +# make VERBOSE=1 -j4 2>&1 | tee log.make + +cd /mnt/experiments-efs/the-real-dom/ufs/ +mkdir -p test/ufs_c48_coldstart +cd test/ufs_c48_coldstart +ln -sf /mnt/experiments-efs/the-real-dom/ufs/static/atm_c48/* . +mkdir INPUT +cd INPUT +ln -sf /mnt/experiments-efs/the-real-dom/ufs/ic/atm_c48/2021072800/2021072800/* . +cd .. +mkdir RESTART +ln -sf /mnt/experiments-efs/the-real-dom/ufs/build-atm/ufs_model . \ No newline at end of file From d83842e4e559f4bd83c651bdd33bcc0a32863cb1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Jan 2023 11:11:40 -0700 Subject: [PATCH 38/43] Update PRs.txt --- PRs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PRs.txt b/PRs.txt index 39add6a..8bb6261 100644 --- a/PRs.txt +++ b/PRs.txt @@ -7,5 +7,6 @@ https://github.com/JCSDA-internal/saber/pull/385 ufs-weather-model PRs: https://github.com/NOAA-EMC/AQM/pull/40 https://github.com/NOAA-EMC/MOM6/pull/107 -FV3 PR missing! +https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere/pull/231 +https://github.com/NOAA-EMC/fv3atm/pull/620 https://github.com/ufs-community/ufs-weather-model/pull/1504 From c78d0dd0bee08b4326f49ad51cfc503e6da3fb4c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 21 Jan 2023 16:29:48 -0700 Subject: [PATCH 39/43] Bug fix in CMakeLists.txt, update README.md --- CMakeLists.txt | 4 +- README.md | 151 ++++++++++++++++++++++--------------------------- 2 files changed, 70 insertions(+), 85 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index db1b946..954d05c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,7 +44,7 @@ ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.5 ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.30.0 ) # External FMS -find_package(FMS 2022.01 REQUIRED COMPONENTS R4 R8) +find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on nov 15 @@ -83,7 +83,7 @@ find_package(OpenMP REQUIRED) if(APPLE) set(UFS_CMAKE_EXE_LINKER_FLAGS "${UFS_CMAKE_EXE_LINKER_FLAGS} ${OpenMP_libomp_LIBRARY} ${OpenMP_libomp_LIBRARY}") endif() -message (SATUS "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") +message (STATUS "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model diff --git a/README.md b/README.md index 04618ac..bf569c7 100644 --- a/README.md +++ b/README.md @@ -2,102 +2,87 @@ Bundle for interfacing UFS models with JEDI interfaces -This bundle requires the following spack-stack modules be loaded (not all are in skylab-2.0.0 environment): +## Required thirdparty libraries -heinzell@JCSDA-L-18146:~/work/ufs-bundle/ufs-bundle/build-debug-ng-godas [brew-x86_64]> module li +This bundle requires the following spack-stack modules be loaded (all except `fms@2022.04` are in the `skylab-3.0.0` environment): +``` +> module li Currently Loaded Modules: - 1) stack-apple-clang/13.1.6 19) base-env/1.0.0 37) hdf/4.2.15 55) py-pycparser/2.20 73) py-pandas/1.4.0 91) py-botocore/1.13.44 109) py-pyshp/2.1.0 127) g2/3.4.5 - 2) pmix/4.1.2 20) boost/1.78.0 38) jedi-cmake/1.4.0 56) py-cffi/1.15.0 74) py-pybind11/2.8.1 92) py-s3transfer/0.2.1 110) py-shapely/1.8.0 128) g2tmpl/1.10.0 - 3) zlib/1.2.12 21) bufr/11.7.1 39) libpng/1.6.37 57) py-findlibs/0.0.2 75) py-pycodestyle/2.8.0 93) py-boto3/1.10.44 111) py-cartopy/0.20.2 129) ip/3.3.3 - 4) openmpi/4.1.4 22) git-lfs/3.1.4 40) libxt/1.1.5 58) py-setuptools/59.4.0 76) py-pyhdf/0.10.4 94) brunsli/0.1 112) py-markupsafe/2.0.1 130) gftl-shared/1.5.0 - 5) stack-openmpi/4.1.4 23) ecbuild/3.6.5 41) libxmu/1.1.2 59) py-numpy/1.22.3 77) py-pyyaml/6.0 95) geos/3.9.1 113) py-jinja2/3.0.3 131) yafyaml/0.5.1 - 6) stack-python/3.9.13 24) openjpeg/2.3.1 42) pigz/2.7 60) py-eccodes/1.4.2 78) llvm-openmp/14.0.6 96) json-c/0.16 114) py-ruamel-yaml/0.17.16 132) mapl/2.22.0-esmf-8.3.0b09-esmf-8.3.0 - 7) cmake/3.23.1 25) eccodes/2.27.0 43) zstd/1.5.2 61) py-f90nml/1.4.2 79) py-gast/0.5.3 97) lerc/3.0 115) py-ruamel-yaml-clib/0.2.4 133) ufs-pyenv/1.0.0 - 8) curl/7.83.0 26) eigen/3.4.0 44) tar/1.34 62) py-mpi4py/3.1.2 80) py-beniget/0.4.1 98) sqlite/3.38.5 116) jedi-ewok-env/1.0.0 134) w3nco/2.4.1 - 9) git/2.36.0 27) openblas/0.3.19 45) gettext/0.21 63) py-h5py/3.6.0 81) py-ply/3.11 99) proj/8.1.0 117) antlr/2.7.7 135) ufs-weather-model-env/1.0.0 - 10) pkg-config/0.29.2 28) eckit/1.19.0 46) libxpm/3.5.12 64) py-cftime/1.0.3.4 82) py-pythran/0.11.0 100) gdal/3.5.1 118) gsl/2.7.1 136) crtm/v2.4_jedi - 11) hdf5/1.12.1 29) fftw/3.3.10 47) libxaw/1.0.13 65) py-netcdf4/1.5.3 83) py-scipy/1.8.0 101) py-cycler/0.11.0 119) nco/5.0.6 137) w3emc/2.9.1 - 12) libbacktrace/2020-02-19 30) fckit/0.9.5 48) udunits/2.2.28 66) py-bottleneck/1.3.2 84) py-xarray/2022.3.0 102) py-fonttools/4.31.2 120) soca-env/1.0.0 138) nemsio/2.5.2 - 13) parallel-netcdf/1.12.2 31) fiat/1.0.0 49) ncview/2.1.8 67) py-pyparsing/3.0.6 85) jedi-base-env/1.0.0 103) py-kiwisolver/1.3.2 121) sp/2.3.3 - 14) netcdf-c/4.8.1 32) ectrans/1.0.0 50) netcdf-cxx4/4.3.1 68) py-packaging/21.3 86) jedi-fv3-env/1.0.0 104) py-pillow/9.2.0 122) sigio/2.3.2 - 15) nccmp/1.9.0.1 33) atlas/0.30.0 51) json/3.10.5 69) py-numexpr/2.8.3 87) ecflow/5.8.3 105) py-matplotlib/3.5.3 123) fms/2022.01 - 16) netcdf-fortran/4.5.4 34) gsibec/1.0.5 52) json-schema-validator/2.1.0 70) py-six/1.16.0 88) py-docutils/0.18.1 106) py-certifi/2021.10.8 124) bacio/2.4.1 - 17) parallelio/2.5.7 35) gsl-lite/0.37.0 53) odc/1.4.5 71) py-python-dateutil/2.8.2 89) py-jmespath/0.10.0 107) py-cython/0.29.30 125) esmf/8.3.0b09 - 18) wget/1.21.3 36) libjpeg/2.1.0 54) py-attrs/21.4.0 72) py-pytz/2021.3 90) py-urllib3/1.25.9 108) py-pyproj/3.1.0 126) jasper/2.0.32 - -Basically, do this: - -module use /Users/heinzell/prod/spack-stack-v1/envs/skylab-2.0.0-plus-ufs/install/modulefiles/Core + 1) stack-apple-clang/13.1.6 18) libpng/1.6.37 35) openjpeg/2.3.1 52) zstd/1.5.2 69) py-eccodes/1.4.2 86) py-pyyaml/6.0 + 2) pmix/4.1.2 19) g2/3.4.5 36) eccodes/2.27.0 53) tar/1.34 70) py-f90nml/1.4.3 87) py-gast/0.5.3 + 3) zlib/1.2.13 20) g2tmpl/1.10.0 37) eigen/3.4.0 54) gettext/0.21.1 71) py-mpi4py/3.1.4 88) py-beniget/0.4.1 + 4) openmpi/4.1.4 21) sp/2.3.3 38) openblas/0.3.19 55) libxpm/3.5.12 72) py-h5py/3.6.0 89) py-ply/3.11 + 5) stack-openmpi/4.1.4 22) ip/3.3.3 39) eckit/1.20.2 56) libxaw/1.0.13 73) py-cftime/1.0.3.4 90) py-pythran/0.11.0 + 6) stack-python/3.9.13 23) cmake/3.23.1 40) fftw/3.3.10 57) udunits/2.2.28 74) py-netcdf4/1.5.3 91) py-scipy/1.8.0 + 7) bacio/2.4.1 24) git/2.36.0 41) fckit/0.9.5 58) ncview/2.1.8 75) py-bottleneck/1.3.5 92) py-xarray/2022.3.0 + 8) curl/7.83.0 25) libbacktrace/2020-02-19 42) fiat/1.0.0 59) netcdf-cxx4/4.3.1 76) py-pyparsing/3.0.9 93) jedi-base-env/1.0.0 + 9) pkg-config/0.29.2 26) nccmp/1.9.0.1 43) ectrans/1.1.0 60) json/3.10.5 77) py-packaging/21.3 94) gftl-shared/1.5.0 + 10) hdf5/1.12.1 27) parallelio/2.5.7 44) atlas/0.31.1 61) json-schema-validator/2.1.0 78) py-numexpr/2.8.3 95) yafyaml/0.5.1 + 11) parallel-netcdf/1.12.2 28) wget/1.21.3 45) gsibec/1.0.6 62) odc/1.4.5 79) py-six/1.16.0 96) mapl/2.22.0-esmf-8.3.0b09-esmf-8.3.0 + 12) netcdf-c/4.8.1 29) base-env/1.0.0 46) gsl-lite/0.37.0 63) py-attrs/22.1.0 80) py-python-dateutil/2.8.2 97) w3nco/2.4.1 + 13) netcdf-fortran/4.5.4 30) boost/1.78.0 47) hdf/4.2.15 64) py-pycparser/2.21 81) py-pytz/2022.2.1 98) nemsio/2.5.2 + 14) esmf/8.3.0b09 31) bufr/11.7.1 48) jedi-cmake/1.4.0 65) py-cffi/1.15.0 82) py-pandas/1.4.0 99) sigio/2.3.2 + 15) llvm-openmp/14.0.6 32) git-lfs/3.1.4 49) libxt/1.1.5 66) py-findlibs/0.0.2 83) py-pybind11/2.8.1 100) w3emc/2.9.2 + 16) libjpeg/2.1.0 33) crtm/v2.4-jedi.2 50) libxmu/1.1.2 67) py-setuptools/59.4.0 84) py-pycodestyle/2.8.0 101) jedi-ufs-env/1.0.0 + 17) jasper/2.0.32 34) ecbuild/3.6.5 51) pigz/2.7 68) py-numpy/1.22.3 85) py-pyhdf/0.10.4 102) fms/2022.04 +``` +On a pre-configured platform (including an AWS single-node/parallelcluster instance based on a spack-stack AMI), load the spack-stack modules for `skylab-3.0.0` as described in https://spack-stack.readthedocs.io/en/1.2.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.6.0, and run `module av` to see if `fms@2022.04` has already been installed. If yes, do +``` +module load fms@2022.04 +``` +to swap the currently loaded `fms@2022.02` with `fms@2022.04`. + +On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.2.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.6.0 to build the `skylab-3.0.0`, then run (use the appropriate path, compiler/mpi/python versions for your system): +``` +module use /Users/heinzell/prod/spack-stack-v1/envs/skylab-3.0.0/install/modulefiles/Core module load stack-apple-clang/13.1.6 module load stack-openmpi/4.1.4 module load stack-python/3.9.13 -module load jedi-fv3-env/1.0.0 -module load jedi-ewok-env/1.0.0 -module load soca-env/1.0.0 -# Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM -module load ufs-weather-model-env/1.0.0 -# Need to reload JEDI CRTM - and fms/2022.02 +fpic -module load crtm/v2.4_jedi -module laod fms/2022.02 -module load sigio/2.3.2 -module load w3emc/2.9.2 -module load nemsio/2.5.2 - -# ORION -module purge -module use /work/noaa/da/role-da/spack-stack/modulefiles -module load miniconda/3.9.7 -module load ecflow/5.8.4 - -module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-plus-ufs-intel-2022.0.2/install/modulefiles/Core -module load stack-intel/2022.0.2 -module load stack-intel-oneapi-mpi/2021.5.1 -module load stack-python/3.9.7 - -module load jedi-fv3-env/1.0.0 -module load jedi-ewok-env/1.0.0 -module load soca-env/1.0.0 -# Loading ufs-weather-model-env replaces the JEDI version of FMS and CRTM -module load ufs-weather-model-env/1.0.0 -# Need to reload JEDI CRTM - and fms/2022.02 +fpic -module load crtm/v2.4_jedi -module laod fms/2022.02 -module load sigio/2.3.2 -module load w3emc/2.9.2 -module load nemsio/2.5.2 - -cd /work2/noaa/da/dheinzel/ufs-bundle/build-debug-atm +module load jedi-ufs-env/1.0.0 -The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, +# Loading fms@2022.04 replaces the default fms@2022.02 +module load fms@2022.04 +``` + +## Building ufs-bundle + +The bundle can be built in three different configuragions: with fv3-jedi and the UFS configured for atmosphere-only, with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. -The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. Create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired): - +The parameter for specifying the configuration type is `-DAPP=[ATM|NG-GODAS|S2S]` on the ecbuild configuration line. Create a build directory under/outside `ufs-bundle` and `cd` to it. Then run one of the following `ecbuild` commands: +``` ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild # Note - not yet tested on macOS with spack-stack #ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild - -On macOS, it may be required to pass -DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind" to the ecbuild command. Further, if using the native Apple clang compiler with llvm-openmp installed via homebrew or spack, it may be necessary to add -DCMAKE_SHARED_LINKER_FLAGS="/path/to/llvm-openmp-x.y.z/lib/libomp.dylib" to the ecbuild command. - -While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. - -After configuration, run "make -j 8" to build. - -The ctests for ATM are called fv3jedi_test_tier1_model_ufs and fv3jedi_test_tier1_forecast_ufs. -The ctest for NG-GODAS is called test_soca_forecast_ufs. - -Note for later for running on Hera: need to add -DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" to cmake/ecbuild command, then run ctest -R get_ on login node, then acquire compute node, for example via salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu, then run the actual ctests on login node. - - - - - - -#### OLD STUFF FROM MARK FOR BUILDING ON HERA WITH OLD SOFTWARE STACK - +``` +On macOS, it may be required to pass `-DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind"` to the ecbuild command,. Further, if using the native Apple `clang` compiler with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="/path/to/llvm-openmp-x.y.z/lib/libomp.dylib"` to the ecbuild command. + +While building with soca (NG-GODAS or S2S), there will be a long pause during configuration when `ecbuild` is downloading the input files for the test to be run. + +After configuration, run `make -j 8` to build. + +The ctests for ATM are called `fv3jedi_test_tier1_model_ufs` and `fv3jedi_test_tier1_forecast_ufs`. +``` +ctest -R _ufs 2>&1 | tee log.ctest.ufs +Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm + Start 1061: fv3jedi_setup_ufs_rundir +1/3 Test #1061: fv3jedi_setup_ufs_rundir .......... Passed 0.31 sec + Start 1107: fv3jedi_test_tier1_model_ufs +2/3 Test #1107: fv3jedi_test_tier1_model_ufs ...... Passed 28.07 sec + Start 1111: fv3jedi_test_tier1_forecast_ufs +3/3 Test #1111: fv3jedi_test_tier1_forecast_ufs ... Passed 24.87 sec +``` +The ctest for NG-GODAS is called `test_soca_forecast_ufs`. Note that there is no test to download the data, this is currently hardcoded in the `soca` top-level `CMakeLists.txt` file. +``` +MISSING +``` +Note for later for running on Hera: need to add `-DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"` to `ecbuild` command, then run `ctest -R get_` on login node, then acquire a compute node, for example via `salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu -I`, then run the actual `ctests` on the compute node. + +# OLD STUFF FROM MARK FOR BUILDING ON HERA WITH OLD SOFTWARE STACK +``` # ufs-jedi-bundle Bundle for interfacing UFS models with JEDI interfaces @@ -148,4 +133,4 @@ module load hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetc git lfs install ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. - +``` From 35c899cee2192931230877b0164d67a469e8aedb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 23 Jan 2023 08:20:45 -0700 Subject: [PATCH 40/43] Add more info on ctests to README.md --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bf569c7..cc9bf89 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,23 @@ Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm ``` The ctest for NG-GODAS is called `test_soca_forecast_ufs`. Note that there is no test to download the data, this is currently hardcoded in the `soca` top-level `CMakeLists.txt` file. ``` -MISSING +ctest -R _ufs 2>&1 | tee log.ctest.ufs +Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-ng-godas + Start 1094: test_soca_forecast_ufs +1/1 Test #1094: test_soca_forecast_ufs ...........***Failed 11.51 sec + +0% tests passed, 1 tests failed out of 1 + +Label Time Summary: +mpi = 11.51 sec*proc (1 test) +script = 11.51 sec*proc (1 test) +soca = 11.51 sec*proc (1 test) + +Total Test time (real) = 13.28 sec + +The following tests FAILED: + 1094 - test_soca_forecast_ufs (Failed) +Errors while running CTest ``` Note for later for running on Hera: need to add `-DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"` to `ecbuild` command, then run `ctest -R get_` on login node, then acquire a compute node, for example via `salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu -I`, then run the actual `ctests` on the compute node. From 234f3cb88cacb5a34e17bf54088450f3c0229d10 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 27 Jan 2023 07:41:29 -0700 Subject: [PATCH 41/43] Cleanup to finalize ufs atm build and warmstart tests --- CMakeLists.txt | 4 +- PRs.txt | 12 ------ README.md | 80 ++++++++---------------------------- TMP_README_UFS_STANDALONE.sh | 20 --------- 4 files changed, 19 insertions(+), 97 deletions(-) delete mode 100644 PRs.txt delete mode 100755 TMP_README_UFS_STANDALONE.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 954d05c..fd29e27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,9 +39,9 @@ option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF -ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.18.2 ) +ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.20.2 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.5 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.30.0 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.31.1 ) # External FMS find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) diff --git a/PRs.txt b/PRs.txt deleted file mode 100644 index 8bb6261..0000000 --- a/PRs.txt +++ /dev/null @@ -1,12 +0,0 @@ -https://github.com/JCSDA-internal/fv3-jedi-linearmodel/pull/16 -https://github.com/JCSDA-internal/fv3-jedi/pull/697 -https://github.com/JCSDA-internal/soca/pull/824 -https://github.com/JCSDA-internal/fv3-jedi-data/pull/52 -https://github.com/JCSDA-internal/saber/pull/385 - -ufs-weather-model PRs: -https://github.com/NOAA-EMC/AQM/pull/40 -https://github.com/NOAA-EMC/MOM6/pull/107 -https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere/pull/231 -https://github.com/NOAA-EMC/fv3atm/pull/620 -https://github.com/ufs-community/ufs-weather-model/pull/1504 diff --git a/README.md b/README.md index cc9bf89..754796e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ufs-jedi-bundle +# ufs-bundle Bundle for interfacing UFS models with JEDI interfaces @@ -64,16 +64,24 @@ While building with soca (NG-GODAS or S2S), there will be a long pause during co After configuration, run `make -j 8` to build. -The ctests for ATM are called `fv3jedi_test_tier1_model_ufs` and `fv3jedi_test_tier1_forecast_ufs`. +The ctests for ATM all have `_ufs` in their names. After running `ctest -R get_`, run the following: ``` ctest -R _ufs 2>&1 | tee log.ctest.ufs -Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm - Start 1061: fv3jedi_setup_ufs_rundir -1/3 Test #1061: fv3jedi_setup_ufs_rundir .......... Passed 0.31 sec - Start 1107: fv3jedi_test_tier1_model_ufs -2/3 Test #1107: fv3jedi_test_tier1_model_ufs ...... Passed 28.07 sec - Start 1111: fv3jedi_test_tier1_forecast_ufs -3/3 Test #1111: fv3jedi_test_tier1_forecast_ufs ... Passed 24.87 sec +Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm-20230121-with-ufs-import + Start 1061: fv3jedi_setup_ufs_rundir_warmstart +1/6 Test #1061: fv3jedi_setup_ufs_rundir_warmstart .......... Passed 0.10 sec + Start 1062: fv3jedi_setup_ufs_rundir_coldstart +2/6 Test #1062: fv3jedi_setup_ufs_rundir_coldstart .......... Passed 0.09 sec + Start 1108: fv3jedi_test_tier1_model_ufs_warmstart +3/6 Test #1108: fv3jedi_test_tier1_model_ufs_warmstart ...... Passed 25.49 sec + Start 1109: fv3jedi_test_tier1_model_ufs_coldstart +4/6 Test #1109: fv3jedi_test_tier1_model_ufs_coldstart ......***Failed 7.82 sec + Start 1113: fv3jedi_test_tier1_forecast_ufs_warmstart +5/6 Test #1113: fv3jedi_test_tier1_forecast_ufs_warmstart ... Passed 25.04 sec + Start 1114: fv3jedi_test_tier1_forecast_ufs_coldstart +6/6 Test #1114: fv3jedi_test_tier1_forecast_ufs_coldstart ...***Failed 4.02 sec + +67% tests passed, 2 tests failed out of 6 ``` The ctest for NG-GODAS is called `test_soca_forecast_ufs`. Note that there is no test to download the data, this is currently hardcoded in the `soca` top-level `CMakeLists.txt` file. ``` @@ -96,57 +104,3 @@ The following tests FAILED: Errors while running CTest ``` Note for later for running on Hera: need to add `-DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"` to `ecbuild` command, then run `ctest -R get_` on login node, then acquire a compute node, for example via `salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu -I`, then run the actual `ctests` on the compute node. - -# OLD STUFF FROM MARK FOR BUILDING ON HERA WITH OLD SOFTWARE STACK -``` -# ufs-jedi-bundle -Bundle for interfacing UFS models with JEDI interfaces - -This bundle requires the following modules be loaded-- -compiler gnu 9.0+ or intel 18+ -MPI -esmf-8.2+ hdf5 netcdf eckit fckit eigen boost atlas-0.27+ pio mkl/openblas -parallel-netcdf cmake-3.20+ ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite - -The bundle can be built in three different configuragions--with fv3-jedi and the UFS configured for atmosphere-only, -with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS -configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. - -The parameter for specifying the configuration type is "-DAPP=(ATM/NG-GODAS/S2S)" on the ecbuild configuration line. - -To build on Hera, load the following modules-- -module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack -module load hpc hpc-gnu hpc-mpich -module load esmf/8.3.0b09 hdf5 netcdf eckit fckit cmake gsl-lite bacio sfcio sp eigen boost-headers atlas pio ecbuild udunits nemsio/2.5.2 w3emc/2.9.2 sigio - -create a build directory under ufs-jedi-bundle and cd to it. Then run the following ecbuild command (substitute "ATM" for "NG-GODAS" as desired. - -ecbuild -DLAPACK_LIBRARIES="/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/liblapack.a;/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a" -DBLAS_openblas_LIBRARY:FILEPATH=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules/gnu-9.2.0/lapack/3.8.0/lib/libblas.a -DMPI_Fortran_LINK_FLAGS="-lexpat" -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpifort -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. - - -NOTE: WITH SPACK-STACK, I DIDN'T NEED ANY OF THIS ... just do "ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild" - -ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild - -ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild - -While building with SOCA (NG-GODAS or S2S) there will be a long pause during configuration when ecbuild is downloading the input files for the test to be run. - -After configuration, run "make -j 8" to build. The ctest for NG-GODAS is called test_soca_forecast_ufs, but the ctests using mpich don't run out of the box on -Hera. You will need to run it manually from the build/soca/test directory. Cd to that directory and run the following to test the forecast-- - -NO, FIX AND USE -DMPIEXEC_EXECUTABLE ETC. THEN SALLOC - -srun --mpi=pmi2 -n 8 ../../bin/soca_forecast.x testinput/forecast_ufs.yml - - -The code will build on Cheyenne, but there is some sort of mpi issue when trying to run. Here are the steps to build-- - -module use /glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack -module load gnu/10.1.0 mpt/2.22 hpc hpc-gnu hpc-mpt/2.22 -module load hdf5/1.10.6 esmf/8.3.0b09 netcdf/4.7.4 eckit fckit eigen boost pnetcdf pio/2.5.3 ecbuild udunits bacio w3nco w3emc nemsio sigio sfcio sp expat gsl-lite openblas cmake/3.22.0 python/3.7.9 git - -git lfs install - -ecbuild -DMPI_Fortran_LINK_FLAGS="-lexpat" -DCMAKE_PREFIX_PATH=/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/core/gsl-lite/0.37.0/lib64/cmake -DBLAS_openblas_LIBRARY:FILEPATH=$NCAR_ROOT_OPENBLAS/lib/libopenblas.so -DBUNDLE_SKIP_ATLAS=OFF -DBUILD_TESTING=TRUE -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_Fortran_COMPILER=mpif90 -DMPI_CXX_LINK_FLAGS:STRING="-Wl,--copy-dt-needed-entries" -DCMAKE_BUILD_TYPE=DEBUG -DUFS_APP=NG-GODAS .. -``` diff --git a/TMP_README_UFS_STANDALONE.sh b/TMP_README_UFS_STANDALONE.sh deleted file mode 100755 index bc2abc1..0000000 --- a/TMP_README_UFS_STANDALONE.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# cd /mnt/experiments-efs/the-real-dom/ufs/ -# source setup.sh -# git clone -b feature/consolidate_dom_update_20221114 --recursive https://github.com/climbfuji/ufs-weather-model ufs-weather-model-feature-consolidate_dom_update_20221114 -# mkdir build-atm -# cd build-atm -# cmake -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16 -DMPI=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF ../ufs-weather-model-feature-consolidate_dom_update_20221114 2>&1 | tee log.cmake -# make VERBOSE=1 -j4 2>&1 | tee log.make - -cd /mnt/experiments-efs/the-real-dom/ufs/ -mkdir -p test/ufs_c48_coldstart -cd test/ufs_c48_coldstart -ln -sf /mnt/experiments-efs/the-real-dom/ufs/static/atm_c48/* . -mkdir INPUT -cd INPUT -ln -sf /mnt/experiments-efs/the-real-dom/ufs/ic/atm_c48/2021072800/2021072800/* . -cd .. -mkdir RESTART -ln -sf /mnt/experiments-efs/the-real-dom/ufs/build-atm/ufs_model . \ No newline at end of file From 75777165e9cc67090c1c09617c02cd2b5f5de817 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 27 Jan 2023 07:48:24 -0700 Subject: [PATCH 42/43] Bump eckit etc versions in CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index fd29e27..5674b53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ endif() # ---------------------- ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG c308945cbc6db4d6add0dac3581bbc4d457e8785 ) # develop on nov 15 ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3363fe28ba65a61227ec1d64504d2641730f51a4 ) # develop on nov 15 +# Note that the saber PR was already merged into develop, but we can't just update to the head of develop while keep the other JEDI repos fixed ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) # updated from develop on nov 15 ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 876e7874fc917376500822157cda82c15ede02f8 ) # develop on nov 15 ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG f95dd253a850debd1f1b391ec63a47f99c341efa ) # develop on nov 15 From f7023c9e829208ee66b2a3e1138408b66009d000 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 1 Feb 2023 08:44:44 -0700 Subject: [PATCH 43/43] Update jedi repos to develop as of Jan 28 2023 (#20) * Update jedi repos to develop as of Jan 28 2023 * Update branch for oops for fixing missing netcdf-c dependencies * Update CMakeLists.txt * Update oops tag in CMakeLists.txt * Change branches back to feature/ufs_dom --------- Co-authored-by: Dom Heinzeller --- CMakeLists.txt | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64aa5c3..52ca9a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,17 +47,16 @@ ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.31. find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on nov 15 + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on jan 28 endif() # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG c308945cbc6db4d6add0dac3581bbc4d457e8785 ) # develop on nov 15 -ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3363fe28ba65a61227ec1d64504d2641730f51a4 ) # develop on nov 15 -# Note that the saber PR was already merged into develop, but we can't just update to the head of develop while keep the other JEDI repos fixed -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH bugfix/add_missing_netcdf_dependency_to_build_system ) # updated from develop on nov 15 -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 876e7874fc917376500822157cda82c15ede02f8 ) # develop on nov 15 -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG f95dd253a850debd1f1b391ec63a47f99c341efa ) # develop on nov 15 +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 0932772f4b20b0defedfb12e31a0c17c18caac71 ) # develop on feb 1 +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3c8fc58ed67c013a955ca7d8f8d2346f2844ac96 ) # develop on jan 28 +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 1203356523b123654a8871846a55a06ed26b7247 ) # develop on jan 28 +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 39ab4dd1dfe737f71385af802c2b5c67c2ef8efc ) # develop on jan 28 +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG ab106a84c770414a3edf9dacabeadea5d70d9edf ) # develop on jan 28 # Options for building with certain models # ---------------------------------------- @@ -71,7 +70,7 @@ set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22 +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22, develop hasn't changed by jan 28 include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) @@ -123,15 +122,15 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # was NOT updated on nov 15th + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # was NOT updated on nov 15th + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() @@ -150,7 +149,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" TAG 2d9fb746b26a306711800960f471cb2ddc56c15c ) # develop on nov 22 + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" TAG 3b4c1ba80a7cb9ae132d74c715492eae36d9d87f ) # develop on jan 28 endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -160,7 +159,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" TAG 998ac3b375248b850ca38ea05749b37a411dac4d ) # develop on nov 15 + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" TAG 87702a24ac6349495e9182d5ee833b488fb1f2fb ) # develop on jan 28 endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -170,7 +169,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) # updated from develop on nov 15 + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) # updated from develop on jan 28 endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out