diff --git a/.gitignore b/.gitignore index de20ac4..9454213 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ ioda/ oops/ ufo/ ewok/ +soca/ +vader/ # FMS fms/ @@ -41,6 +43,7 @@ fv3/ femps/ fv3-jedi-lm/ fv3-jedi/ +ufs-weather-model/ # Observation operators crtm/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 05d45a6..52ca9a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,14 @@ # (C) Copyright 2021-2021 NOAA # ########## -# ufs-jedi-bundle +# ufs-bundle # ########## cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) -project( ufs-jedi-bundle VERSION 1.1.0 LANGUAGES C CXX Fortran ) +project( ufs-bundle VERSION 1.1.0 LANGUAGES C CXX Fortran ) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -23,11 +23,15 @@ set( ECBUILD_DEFAULT_BUILD_TYPE Release ) # Enable MPI set( ENABLE_MPI ON CACHE BOOL "Compile with MPI") - -# Initialize -ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) -include( jedicmake/cmake/Functions/git_functions.cmake ) +set( ENABLE_OMP ON CACHE BOOL "Compile with OpenMP") + +# Use external jedi-cmake or build in bundle +if(DEFINED ENV{jedi_cmake_ROOT}) + include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake ) +else() + ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda-internal/jedi-cmake.git" TAG 1.4.0 UPDATE RECURSIVE ) + include( jedicmake/cmake/Functions/git_functions.cmake ) +endif() # ECMWF libs # ---------- @@ -35,21 +39,24 @@ option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF -ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.18.2 ) +ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.20.2 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.5 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29.0 ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.31.1 ) -# External (required) observation operators -# ------------------------------ -option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Build crtm unless user passes -DBUNDLE_SKIP_CRTM=ON -ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" BRANCH release/crtm_jedi UPDATE ) +# External FMS +find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) + +if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on jan 28 +endif() # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 0932772f4b20b0defedfb12e31a0c17c18caac71 ) # develop on feb 1 +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG 3c8fc58ed67c013a955ca7d8f8d2346f2844ac96 ) # develop on jan 28 +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG 1203356523b123654a8871846a55a06ed26b7247 ) # develop on jan 28 +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG 39ab4dd1dfe737f71385af802c2b5c67c2ef8efc ) # develop on jan 28 +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG ab106a84c770414a3edf9dacabeadea5d70d9edf ) # develop on jan 28 # Options for building with certain models # ---------------------------------------- @@ -59,71 +66,73 @@ if(NOT UFS_APP MATCHES "^(ATM|NG-GODAS|S2S)$") ecbuild_error("Valid options for -DUFS_APP are ATM|NG-GODAS|S2S") endif() ecbuild_info("Building with UFS application: ${UFS_APP}") - set(FV3_FORECAST_MODEL "UFS") + # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22, develop hasn't changed by jan 28 -include_directories(${DEPEND_LIB_ROOT}/include) +include_directories(${DEPEND_LIB_ROOT}/include_r8) link_directories(${DEPEND_LIB_ROOT}/lib) include( ExternalProject ) +# Needed to get correct OpenMP link libraries on macOS +set(UFS_CMAKE_EXE_LINKER_FLAGS "-L${CMAKE_CURRENT_BINARY_DIR}/lib") +find_package(OpenMP REQUIRED) +if(APPLE) + set(UFS_CMAKE_EXE_LINKER_FLAGS "${UFS_CMAKE_EXE_LINKER_FLAGS} ${OpenMP_libomp_LIBRARY} ${OpenMP_libomp_LIBRARY}") +endif() +message (STATUS "UFS_CMAKE_EXE_LINKER_FLAGS: ${UFS_CMAKE_EXE_LINKER_FLAGS}") + ExternalProject_Add(ufs-weather-model PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model - GIT_REPOSITORY https://github.com/mark-a-potts/ufs-weather-model.git + GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model GIT_SUBMODULES_RECURSE TRUE - GIT_TAG develop + GIT_TAG feature/consolidate_dom_update_20230117 # updated from develop on Jan 17 SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model + UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} - CMAKE_ARGS ${UFS_WEATHER_MODEL_ARGS} - CMAKE_ARGS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DREPRO=on -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw -DREPRO=ON -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} INSTALL_COMMAND make install BUILD_ALWAYS FALSE ) -ExternalProject_Add(FMS - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/fms - GIT_REPOSITORY https://github.com/jcsda/FMS.git - GIT_SUBMODULES_RECURSE TRUE - GIT_TAG dev/jcsda - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fms - INSTALL_DIR ${DEPEND_LIB_ROOT} - CMAKE_ARGS -DGFS_PHYS=ON -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_Fortran_FLAGS="-fPIC" -D64BIT=ON -DOPENMP=ON -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} - INSTALL_COMMAND make install - BUILD_ALWAYS TRUE - ) - -add_dependencies(ufs-weather-model FMS ) +if(UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") + add_dependencies(ufs-weather-model gsw) +endif() add_library( ufs IMPORTED STATIC) -add_library( FMS::fms_r4 IMPORTED STATIC) -add_library( FMS::fms_r8 IMPORTED STATIC) +add_library( cdeps IMPORTED STATIC) +add_library( cmeps IMPORTED STATIC) add_library( ccpp IMPORTED STATIC) add_library( stochastic_physics IMPORTED STATIC) add_library( fv3atm IMPORTED STATIC) add_library( ccppphys IMPORTED STATIC) -set_target_properties(FMS::fms_r4 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r4.a) -set_target_properties(FMS::fms_r8 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfms_r8.a) -set_target_properties(ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) +add_library( mom6 IMPORTED STATIC) +set_target_properties( ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libufs.a) +set_target_properties( cdeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcdeps.a) +set_target_properties( cmeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libcmeps.a) set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_framework.a) set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libstochastic_physics.a) set_target_properties(fv3atm PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libfv3atm.a) set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libccpp_physics.a) +set_target_properties(mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/lib/libmom6.a) if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 + add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$") - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs UPDATE ) + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on jan28 +else() + message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() @@ -140,7 +149,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" TAG 3b4c1ba80a7cb9ae132d74c715492eae36d9d87f ) # develop on jan 28 endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -150,7 +159,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" TAG 87702a24ac6349495e9182d5ee833b488fb1f2fb ) # develop on jan 28 endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -160,7 +169,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) # updated from develop on jan 28 endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out diff --git a/README.md b/README.md index 8735c52..754796e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,106 @@ -# ufs-jedi-bundle +# ufs-bundle + Bundle for interfacing UFS models with JEDI interfaces + +## Required thirdparty libraries + +This bundle requires the following spack-stack modules be loaded (all except `fms@2022.04` are in the `skylab-3.0.0` environment): +``` +> module li + +Currently Loaded Modules: + 1) stack-apple-clang/13.1.6 18) libpng/1.6.37 35) openjpeg/2.3.1 52) zstd/1.5.2 69) py-eccodes/1.4.2 86) py-pyyaml/6.0 + 2) pmix/4.1.2 19) g2/3.4.5 36) eccodes/2.27.0 53) tar/1.34 70) py-f90nml/1.4.3 87) py-gast/0.5.3 + 3) zlib/1.2.13 20) g2tmpl/1.10.0 37) eigen/3.4.0 54) gettext/0.21.1 71) py-mpi4py/3.1.4 88) py-beniget/0.4.1 + 4) openmpi/4.1.4 21) sp/2.3.3 38) openblas/0.3.19 55) libxpm/3.5.12 72) py-h5py/3.6.0 89) py-ply/3.11 + 5) stack-openmpi/4.1.4 22) ip/3.3.3 39) eckit/1.20.2 56) libxaw/1.0.13 73) py-cftime/1.0.3.4 90) py-pythran/0.11.0 + 6) stack-python/3.9.13 23) cmake/3.23.1 40) fftw/3.3.10 57) udunits/2.2.28 74) py-netcdf4/1.5.3 91) py-scipy/1.8.0 + 7) bacio/2.4.1 24) git/2.36.0 41) fckit/0.9.5 58) ncview/2.1.8 75) py-bottleneck/1.3.5 92) py-xarray/2022.3.0 + 8) curl/7.83.0 25) libbacktrace/2020-02-19 42) fiat/1.0.0 59) netcdf-cxx4/4.3.1 76) py-pyparsing/3.0.9 93) jedi-base-env/1.0.0 + 9) pkg-config/0.29.2 26) nccmp/1.9.0.1 43) ectrans/1.1.0 60) json/3.10.5 77) py-packaging/21.3 94) gftl-shared/1.5.0 + 10) hdf5/1.12.1 27) parallelio/2.5.7 44) atlas/0.31.1 61) json-schema-validator/2.1.0 78) py-numexpr/2.8.3 95) yafyaml/0.5.1 + 11) parallel-netcdf/1.12.2 28) wget/1.21.3 45) gsibec/1.0.6 62) odc/1.4.5 79) py-six/1.16.0 96) mapl/2.22.0-esmf-8.3.0b09-esmf-8.3.0 + 12) netcdf-c/4.8.1 29) base-env/1.0.0 46) gsl-lite/0.37.0 63) py-attrs/22.1.0 80) py-python-dateutil/2.8.2 97) w3nco/2.4.1 + 13) netcdf-fortran/4.5.4 30) boost/1.78.0 47) hdf/4.2.15 64) py-pycparser/2.21 81) py-pytz/2022.2.1 98) nemsio/2.5.2 + 14) esmf/8.3.0b09 31) bufr/11.7.1 48) jedi-cmake/1.4.0 65) py-cffi/1.15.0 82) py-pandas/1.4.0 99) sigio/2.3.2 + 15) llvm-openmp/14.0.6 32) git-lfs/3.1.4 49) libxt/1.1.5 66) py-findlibs/0.0.2 83) py-pybind11/2.8.1 100) w3emc/2.9.2 + 16) libjpeg/2.1.0 33) crtm/v2.4-jedi.2 50) libxmu/1.1.2 67) py-setuptools/59.4.0 84) py-pycodestyle/2.8.0 101) jedi-ufs-env/1.0.0 + 17) jasper/2.0.32 34) ecbuild/3.6.5 51) pigz/2.7 68) py-numpy/1.22.3 85) py-pyhdf/0.10.4 102) fms/2022.04 +``` +On a pre-configured platform (including an AWS single-node/parallelcluster instance based on a spack-stack AMI), load the spack-stack modules for `skylab-3.0.0` as described in https://spack-stack.readthedocs.io/en/1.2.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.6.0, and run `module av` to see if `fms@2022.04` has already been installed. If yes, do +``` +module load fms@2022.04 +``` +to swap the currently loaded `fms@2022.02` with `fms@2022.04`. + +On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.2.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.6.0 to build the `skylab-3.0.0`, then run (use the appropriate path, compiler/mpi/python versions for your system): +``` +module use /Users/heinzell/prod/spack-stack-v1/envs/skylab-3.0.0/install/modulefiles/Core +module load stack-apple-clang/13.1.6 +module load stack-openmpi/4.1.4 +module load stack-python/3.9.13 +module load jedi-ufs-env/1.0.0 + +# Loading fms@2022.04 replaces the default fms@2022.02 +module load fms@2022.04 +``` + +## Building ufs-bundle + +The bundle can be built in three different configuragions: with fv3-jedi and the UFS configured for atmosphere-only, +with soca and the UFS configured with a data-atmosphere using NG-GODAS, or with fv3-jedi and soca and the UFS +configured with S2S coupling. Only the first two configurations have test cases and are known to work, but all will build. + +The parameter for specifying the configuration type is `-DAPP=[ATM|NG-GODAS|S2S]` on the ecbuild configuration line. Create a build directory under/outside `ufs-bundle` and `cd` to it. Then run one of the following `ecbuild` commands: +``` +ecbuild --build=debug -DUFS_APP=ATM .. 2>&1 | tee log.ecbuild +ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild +# Note - not yet tested on macOS with spack-stack +#ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild +``` +On macOS, it may be required to pass `-DCMAKE_EXE_LINKER_FLAGS="-Wl,-no_compact_unwind"` to the ecbuild command,. Further, if using the native Apple `clang` compiler with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="/path/to/llvm-openmp-x.y.z/lib/libomp.dylib"` to the ecbuild command. + +While building with soca (NG-GODAS or S2S), there will be a long pause during configuration when `ecbuild` is downloading the input files for the test to be run. + +After configuration, run `make -j 8` to build. + +The ctests for ATM all have `_ufs` in their names. After running `ctest -R get_`, run the following: +``` +ctest -R _ufs 2>&1 | tee log.ctest.ufs +Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm-20230121-with-ufs-import + Start 1061: fv3jedi_setup_ufs_rundir_warmstart +1/6 Test #1061: fv3jedi_setup_ufs_rundir_warmstart .......... Passed 0.10 sec + Start 1062: fv3jedi_setup_ufs_rundir_coldstart +2/6 Test #1062: fv3jedi_setup_ufs_rundir_coldstart .......... Passed 0.09 sec + Start 1108: fv3jedi_test_tier1_model_ufs_warmstart +3/6 Test #1108: fv3jedi_test_tier1_model_ufs_warmstart ...... Passed 25.49 sec + Start 1109: fv3jedi_test_tier1_model_ufs_coldstart +4/6 Test #1109: fv3jedi_test_tier1_model_ufs_coldstart ......***Failed 7.82 sec + Start 1113: fv3jedi_test_tier1_forecast_ufs_warmstart +5/6 Test #1113: fv3jedi_test_tier1_forecast_ufs_warmstart ... Passed 25.04 sec + Start 1114: fv3jedi_test_tier1_forecast_ufs_coldstart +6/6 Test #1114: fv3jedi_test_tier1_forecast_ufs_coldstart ...***Failed 4.02 sec + +67% tests passed, 2 tests failed out of 6 +``` +The ctest for NG-GODAS is called `test_soca_forecast_ufs`. Note that there is no test to download the data, this is currently hardcoded in the `soca` top-level `CMakeLists.txt` file. +``` +ctest -R _ufs 2>&1 | tee log.ctest.ufs +Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-ng-godas + Start 1094: test_soca_forecast_ufs +1/1 Test #1094: test_soca_forecast_ufs ...........***Failed 11.51 sec + +0% tests passed, 1 tests failed out of 1 + +Label Time Summary: +mpi = 11.51 sec*proc (1 test) +script = 11.51 sec*proc (1 test) +soca = 11.51 sec*proc (1 test) + +Total Test time (real) = 13.28 sec + +The following tests FAILED: + 1094 - test_soca_forecast_ufs (Failed) +Errors while running CTest +``` +Note for later for running on Hera: need to add `-DMPIEXEC_EXECUTABLE="/apps/slurm/default/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"` to `ecbuild` command, then run `ctest -R get_` on login node, then acquire a compute node, for example via `salloc -N 1 -n 24 --time=480 --qos=batch -A da-cpu -I`, then run the actual `ctests` on the compute node.