diff --git a/CMakeLists.txt b/CMakeLists.txt index de708dd..bb0519a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,32 +37,28 @@ endif() # ECMWF libs # ---------- -option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF -option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF -option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF - -ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.20.2 ) -ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.10.0 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.32.1 ) +# ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.23.0 ) +# ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.10.1 ) +# ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.33.0 ) # External FMS find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) # Core JEDI repositories # ---------------------- -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG 3318acc4b527e49fcc2e2d78549a06fc4ab43dd0 ) # develop on apr 18 -ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" TAG c5e0c1a8d4feeedacfef788f7670511e4db41d5d ) # develop on apr 18 -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" TAG fc8992d7c6634c384f2f61eb121743ab8ede0d6a ) # develop on apr 18 +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" TAG 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 ) # develop on apr 18 + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) endif() -#ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda-internal/crtm.git" TAG v2.4-jedi.2 ) +ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/CRTMv3.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" TAG fb651c65332692472ea490d72a85ed6a63e8650f ) # develop on apr 18 -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" TAG 3bb109889ab5a01938d189a243660ac52fc9a39b ) # develop on apr 18 +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) # Options for building with certain models # ---------------------------------------- @@ -76,7 +72,7 @@ set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22, develop hasn't changed by apr 18 +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom ) # updated from develop on nov 22, develop hasn't changed by june 13 message(INFO "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}") include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}) @@ -112,7 +108,7 @@ ExternalProject_Add(ufs-weather-model CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_coupled,FV3_GFS_v15p2 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DJEDI_DRIVER=ON -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw ${UFS_CMAKE_BUILD_TYPE_FLAG} -DOPENMP=OFF -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} # *DH INSTALL_COMMAND make install - BUILD_ALWAYS FALSE + BUILD_ALWAYS TRUE ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") @@ -161,15 +157,15 @@ if(UFS_APP MATCHES "^(S2S)$") # fv3-jedi and associated repositories # ------------------------------------ ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on apr 18 - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on apr 18 + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) # updated from develop on apr 18 + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom ) add_dependencies(soca ufs-weather-model) elseif(UFS_APP MATCHES "^(ATM)$" OR UFS_APP MATCHES "^(ATMAERO)$") ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) # updated from develop on april 18 + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/ufs_dom ) else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") endif() @@ -188,7 +184,7 @@ find_branch_name(REPO_DIR_NAME ioda) # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored # in a local directory, ioda-data repo will not be cloned if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" TAG 8f092c6f164cd2f9ff803f720b13916fa40b0c08 ) # develop on apr 18 + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) endif() # If IODA's current branch is available in ioda-data repo, that branch will be checked out @@ -198,7 +194,7 @@ branch_checkout (REPO_DIR_NAME ioda-data # same procedure for ufo-data find_branch_name(REPO_DIR_NAME ufo) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" TAG 1944119defaf82e0591683b29462faadcb1921b6 ) # develop on apr 18 + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) endif() # If UFO's current branch is available in ufo-data repo, that branch will be checked out @@ -208,7 +204,7 @@ branch_checkout (REPO_DIR_NAME ufo-data # same procedure for fv3-jedi-data find_branch_name(REPO_DIR_NAME fv3-jedi) if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) # updated from develop on apr 18 + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom ) endif() # If fv3-jedi's current branch is available in fv3-jedi-data repo, that branch will be checked out diff --git a/README.md b/README.md index 4de205e..f17dec8 100644 --- a/README.md +++ b/README.md @@ -9,37 +9,40 @@ This bundle requires the following spack-stack modules be loaded (all except `fm > module li Currently Loaded Modules: - 1) stack-apple-clang/13.1.6 22) esmf/8.4.2 43) ecbuild/3.7.2 64) netcdf-cxx4/4.3.1 85) py-pandas/1.4.0 - 2) zlib/1.2.13 23) llvm-openmp/16.0.0 44) openjpeg/2.3.1 65) json/3.10.5 86) py-pybind11/2.8.1 - 3) pigz/2.7 24) fms/2023.01 45) eccodes/2.27.0 66) json-schema-validator/2.1.0 87) py-pycodestyle/2.8.0 - 4) zstd/1.5.2 25) libjpeg/2.1.0 46) eigen/3.4.0 67) odc/1.4.6 88) py-pyhdf/0.10.4 - 5) tar/1.34 26) jasper/2.0.32 47) openblas/0.3.19 68) py-attrs/22.2.0 89) libyaml/0.2.5 - 6) gettext/0.21.1 27) libpng/1.6.37 48) eckit/1.23.0 69) py-pycparser/2.21 90) py-pyyaml/6.0 - 7) libxcrypt/4.4.33 28) g2/3.4.5 49) fftw/3.3.10 70) py-cffi/1.15.1 91) py-gast/0.5.3 - 8) sqlite/3.40.1 29) g2tmpl/1.10.2 50) fckit/0.10.1 71) py-findlibs/0.0.2 92) py-beniget/0.4.1 - 9) python/3.10.8 30) sp/2.3.3 51) fiat/1.1.0 72) py-setuptools/59.4.0 93) py-ply/3.11 - 10) stack-python/3.10.8 31) ip/3.3.3 52) ectrans/1.2.0 73) py-numpy/1.22.3 94) py-pythran/0.12.2 - 11) pmix/4.2.3 32) cmake/3.23.1 53) atlas/0.33.0 74) py-eccodes/1.4.2 95) py-scipy/1.9.3 - 12) openmpi/4.1.5 33) git/2.36.0 54) gsibec/1.1.2 75) py-f90nml/1.4.3 96) py-xarray/2022.3.0 - 13) stack-openmpi/4.1.5 34) libbacktrace/2020-02-19 55) gsl-lite/0.37.0 76) py-h5py/3.7.0 97) jedi-base-env/1.0.0 - 14) bacio/2.4.1 35) nccmp/1.9.0.1 56) hdf/4.2.15 77) py-cftime/1.0.3.4 98) gftl/1.10.0 - 15) curl/8.0.1 36) py-pip/23.0 57) jedi-cmake/1.4.0 78) py-netcdf4/1.5.3 99) gftl-shared/1.5.0 - 16) pkg-config/0.29.2 37) wget/1.21.3 58) libxt/1.1.5 79) py-bottleneck/1.3.5 100) mapl/2.35.2-esmf-8.4.2 - 17) hdf5/1.14.0 38) base-env/1.0.0 59) libxmu/1.1.2 80) py-packaging/23.0 101) w3nco/2.4.1 - 18) netcdf-c/4.9.2 39) boost/1.78.0 60) libxpm/3.5.12 81) py-numexpr/2.8.3 102) nemsio/2.5.2 - 19) netcdf-fortran/4.6.0 40) bufr/11.7.1 61) libxaw/1.0.13 82) py-six/1.16.0 103) sigio/2.3.2 - 20) parallel-netcdf/1.12.2 41) git-lfs/3.1.4 62) udunits/2.2.28 83) py-python-dateutil/2.8.2 104) w3emc/2.9.2 - 21) parallelio/2.5.9 42) crtm/v2.4.1-jedi 63) ncview/2.1.8 84) py-pytz/2022.2.1 105) jedi-ufs-env/unified-dev -``` -On a pre-configured platform (including an AWS single-node/parallelcluster instance based on a spack-stack AMI), load the spack-stack modules for `skylab-5.0.0` as described in https://spack-stack.readthedocs.io/en/1.4.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.8.0. - -On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.4.0 and https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/1.8.0 to build the `skylab-env` environment, then run (use the appropriate path, compiler/mpi/python versions for your system): -``` -module use /Users/heinzell/prod/spack-stack-v1/envs/skylab-env/install/modulefiles/Core + 1) stack-apple-clang/13.1.6 22) esmf/8.4.2 43) eccodes/2.27.0 64) json-schema-validator/2.1.0 85) py-pycodestyle/2.8.0 + 2) pmix/4.2.3 23) llvm-openmp/16.0.0 44) eigen/3.4.0 65) odc/1.4.6 86) py-pyhdf/0.10.4 + 3) zlib/1.2.13 24) libjpeg/2.1.0 45) openblas/0.3.19 66) py-attrs/22.2.0 87) libyaml/0.2.5 + 4) openmpi/4.1.5 25) jasper/2.0.32 46) eckit/1.23.1 67) py-pycparser/2.21 88) py-pyyaml/6.0 + 5) stack-openmpi/4.1.5 26) libpng/1.6.37 47) fftw/3.3.10 68) py-cffi/1.15.1 89) py-gast/0.5.3 + 6) pigz/2.7 27) g2/3.4.5 48) fckit/0.10.1 69) py-findlibs/0.0.2 90) py-beniget/0.4.1 + 7) zstd/1.5.2 28) g2tmpl/1.10.2 49) fiat/1.1.0 70) py-setuptools/59.4.0 91) py-ply/3.11 + 8) tar/1.34 29) sp/2.3.3 50) ectrans/1.2.0 71) py-numpy/1.22.3 92) py-pythran/0.12.2 + 9) gettext/0.21.1 30) ip/3.3.3 51) atlas/0.33.0 72) py-eccodes/1.4.2 93) py-scipy/1.9.3 + 10) libxcrypt/4.4.33 31) cmake/3.23.1 52) gsibec/1.1.2 73) py-f90nml/1.4.3 94) py-xarray/2022.3.0 + 11) sqlite/3.40.1 32) git/2.36.0 53) gsl-lite/0.37.0 74) py-h5py/3.7.0 95) jedi-base-env/1.0.0 + 12) python/3.10.8 33) libbacktrace/2020-02-19 54) hdf/4.2.15 75) py-cftime/1.0.3.4 96) gftl/1.8.3 + 13) stack-python/3.10.8 34) nccmp/1.9.0.1 55) jedi-cmake/1.4.0 76) py-netcdf4/1.5.3 97) gftl-shared/1.5.0 + 14) bacio/2.4.1 35) py-pip/23.0 56) libxt/1.1.5 77) py-bottleneck/1.3.5 98) yafyaml/0.5.1 + 15) curl/8.0.1 36) wget/1.21.3 57) libxmu/1.1.2 78) py-packaging/23.0 99) mapl/2.35.2-esmf-8.4.2 + 16) pkg-config/0.29.2 37) base-env/1.0.0 58) libxpm/3.5.12 79) py-numexpr/2.8.3 100) w3nco/2.4.1 + 17) hdf5/1.14.0 38) boost/1.78.0 59) libxaw/1.0.13 80) py-six/1.16.0 101) nemsio/2.5.2 + 18) netcdf-c/4.9.2 39) bufr/12.0.0 60) udunits/2.2.28 81) py-python-dateutil/2.8.2 102) sigio/2.3.2 + 19) netcdf-fortran/4.6.0 40) git-lfs/3.1.4 61) ncview/2.1.8 82) py-pytz/2022.2.1 103) w3emc/2.9.2 + 20) parallel-netcdf/1.12.2 41) ecbuild/3.7.2 62) netcdf-cxx4/4.3.1 83) py-pandas/1.4.0 104) jedi-ufs-env/unified-dev + 21) parallelio/2.5.9 42) openjpeg/2.3.1 63) json/3.10.5 84) py-pybind11/2.8.1 105) fms@2023.01 +``` +On a pre-configured platform that has spack-stack-1.4.1 (or 1.4.0) installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.4.1 (https://spack-stack.readthedocs.io/en/1.4.0), followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.4.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system): +``` +module use /Users/heinzell/prod/spack-stack-v1/envs/unified-dev/install/modulefiles/Core module load stack-apple-clang/13.1.6 module load stack-openmpi/4.1.5 module load stack-python/3.10.8 -module load jedi-ufs-env/skylab-dev +module load jedi-ufs-env/unified-dev +``` +Further, for testing of the newest `fms@2023.02-beta1` tag, verify if the platform you are on has this version available and if yes, run +``` +# Loading fms/2023.02-beta1 replaces the default fms@2023.01 +module load fms/2023.02-beta1 ``` ## Building ufs-bundle @@ -55,7 +58,33 @@ ecbuild --build=debug -DUFS_APP=ATMAERO .. 2>&1 | tee log.ecbuild ecbuild --build=debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.ecbuild ecbuild --build=debug -DUFS_APP=S2S .. 2>&1 | tee log.ecbuild ``` -When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="$llvm_openmp_ROOT/lib/libomp.dylib"` to the ecbuild command. +When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="${llvm_openmp_ROOT}/lib/libomp.dylib"` to the ecbuild command. + +macOS users may also encounter this error during the linker phase: +``` +dyld[47628]: dyld[47619]: Library not loaded: 'libpioc.dylib' + Referenced from: '/Users/hLibrary not loaded: 'libpioc.dyleinzell/work/ufs-bundle/20230714ib' +... +``` +This error comes from the `pioc` library not being installed correctly (a problem we have raised to the `parallelio` maintainers): +``` +> otool -L ${parallelio_ROOT}/lib/libpioc.dylib +/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallelio-2.5.9-r5txd2q/lib/libpioc.dylib: + libpioc.dylib (compatibility version 0.0.0, current version 0.0.0) + /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/netcdf-c-4.9.2-vrrvi2u/lib/libnetcdf.19.dylib (compatibility version 22.0.0, current version 22.2.0) + /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallel-netcdf-1.12.2-mfx2uut/lib/libpnetcdf.4.dylib (compatibility version 5.0.0, current version 5.2.0) + /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/openmpi-4.1.5-j7pjg6h/lib/libmpi.40.dylib (compatibility version 71.0.0, current version 71.5.0) + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3) +``` +The first identifier to the library itself must either contain `@rpath/` and the `RPATH` needs to be added correctly, or (easier) it must contain the full path to the library. This can be fixed by running: +``` +install_name_tool -id ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib +``` +A similar fix may be needed for `libpiof.dylib`, depending the `UFS_APP`: +``` +install_name_tool -id ${parallelio_ROOT}/lib/libpiof.dylib ${parallelio_ROOT}/lib/libpiof.dylib +install_name_tool -change libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpiof.dylib +``` While building with soca (NG-GODAS or S2S), there will be a long pause during configuration when `ecbuild` is downloading the input files for the test to be run.