From 528e0a1b1e294e057fec4651882ce4183e4cf0fc Mon Sep 17 00:00:00 2001 From: kgerheiser Date: Mon, 28 Sep 2020 09:37:51 -0400 Subject: [PATCH 01/10] Update CMake --- CMakeLists.txt | 5 ++++- Externals.cfg | 4 ++-- src/CMakeLists.txt | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 src/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e8457ce10..366e0e9d9d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,9 @@ if (${CMAKE_VERSION} VERSION_LESS 3.12) endif() # Set the project name and version. -project(ufs-srweather-app VERSION 1.0) +project(ufs-srweather-app VERSION 1.0 LANGUAGES C CXX Fortran) + +find_package(MPI REQUIRED COMPONENTS C CXX Fortran) # Set extended version info. SET(SRWA_VERSION_MAJOR 1) @@ -139,3 +141,4 @@ configure_file( FILE(COPY "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app_meta.h" DESTINATION include) +add_subdirectory(src) diff --git a/Externals.cfg b/Externals.cfg index cdc7aba41a..1b40dac961 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -10,8 +10,8 @@ required = True protocol = git repo_url = https://github.com/JeffBeck-NOAA/UFS_UTILS # Specify either a branch name or a hash but not both. -#branch = feature/regional_release -hash = e5419633 +branch = feature/regional_release +#hash = e5419633 local_path = src/UFS_UTILS required = True diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000000..36a19e1cab --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,32 @@ +include(ExternalProject) + +# ExternalProject_Add(UFS_UTILS +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/UFS_UTILS +# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/UFS_UTILS +# INSTALL_DIR ${CMAKE_INSTALL_PREFIX} +# CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" +# ) + +# set(CCPP_SUITES "FV3_GFS_2017_gfdlmp,FV3_GSD_v0,FV3_GSD_SAR,FV3_CPT_v0,FV3_GFS_v15p2,FV3_GFS_v16beta") + + +set(CCPP_SUITES "FV3_CPT_v0,FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_regional,FV3_GSD_SAR,FV3_GSD_v0,FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_RRFS_v1beta") + +ExternalProject_Add(ufs_weather_model + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs_weather_model + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs_weather_model + INSTALL_DIR ${CMAKE_INSTALL_PREFIX} + CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + "-DCCPP_SUITES=${CCPP_SUITES}" + "-DCMAKE_C_COMPILER=${MPI_C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER}" + "-DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER}" + "-DNETCDF_DIR=$ENV{NETCDF}" + ) + +# ExternalProject_Add(EMC_Post +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/EMC_Post +# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/EMC_Post +# INSTALL_DIR ${CMAKE_INSTALL_PREFIX} +# CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" +# ) From 35fbe10c7ad9a593aa9629b30f5b62b2e6afa293 Mon Sep 17 00:00:00 2001 From: kgerheiser Date: Mon, 28 Sep 2020 11:49:30 -0400 Subject: [PATCH 02/10] Enable building post and UFS_UTILS --- src/CMakeLists.txt | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 36a19e1cab..0a779b1b2a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,13 +1,11 @@ include(ExternalProject) -# ExternalProject_Add(UFS_UTILS -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/UFS_UTILS -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/UFS_UTILS -# INSTALL_DIR ${CMAKE_INSTALL_PREFIX} -# CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" -# ) - -# set(CCPP_SUITES "FV3_GFS_2017_gfdlmp,FV3_GSD_v0,FV3_GSD_SAR,FV3_CPT_v0,FV3_GFS_v15p2,FV3_GFS_v16beta") +ExternalProject_Add(UFS_UTILS + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/UFS_UTILS + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/UFS_UTILS + INSTALL_DIR ${CMAKE_INSTALL_PREFIX} + CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + ) set(CCPP_SUITES "FV3_CPT_v0,FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_regional,FV3_GSD_SAR,FV3_GSD_v0,FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_RRFS_v1beta") @@ -24,9 +22,9 @@ ExternalProject_Add(ufs_weather_model "-DNETCDF_DIR=$ENV{NETCDF}" ) -# ExternalProject_Add(EMC_Post -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/EMC_Post -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/EMC_Post -# INSTALL_DIR ${CMAKE_INSTALL_PREFIX} -# CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" -# ) +ExternalProject_Add(EMC_Post + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/EMC_Post + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/EMC_Post + INSTALL_DIR ${CMAKE_INSTALL_PREFIX} + CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + ) From 515e067be443f5881677d6888b6f6366a576c03c Mon Sep 17 00:00:00 2001 From: kgerheiser Date: Wed, 30 Sep 2020 13:00:08 -0400 Subject: [PATCH 03/10] Make EMC_post lowercase in CMakeLists --- src/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a779b1b2a..5a0079083f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,9 +22,9 @@ ExternalProject_Add(ufs_weather_model "-DNETCDF_DIR=$ENV{NETCDF}" ) -ExternalProject_Add(EMC_Post - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/EMC_Post - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/EMC_Post +ExternalProject_Add(EMC_post + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/EMC_post + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/EMC_post INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" ) From a9716b4bf49adcd71391c82bdb0a052305e472f2 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 5 Oct 2020 21:58:44 +0000 Subject: [PATCH 04/10] Add Hera build instructions --- docs/README_hera_intel.txt | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/README_hera_intel.txt diff --git a/docs/README_hera_intel.txt b/docs/README_hera_intel.txt new file mode 100644 index 0000000000..9e6c6da16d --- /dev/null +++ b/docs/README_hera_intel.txt @@ -0,0 +1,52 @@ +#Setup instructions for NOAA RDHPC Hera using Intel-18.0.5.274 (bash shell) + +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load cmake/3.16.1 + +export CC=icc +export CXX=icpc +export FC=ifort + +NCEPLIBS_INSTALL=/scratch1/BMC/gmtb/software/NCEPLIBS-ufs-v2.0.0/intel-18.0.5.274/impi-2018.0.4 + +module use -a ${NCEPLIBS_INSTALL}/modules + +module load bacio/2.4.1 +module load g2/3.4.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load sigio/2.3.2 + +module load sfcio/1.4.1 +module load gfsio/1.4.1 +module load nemsiogfs/2.5.3 +module load landsfcutil/2.4.1 +module load wgrib2/2.0.8 +module load netcdf/4.7.4 + +module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles +module load hdf5_parallel/1.10.6.release +module load netcdf_parallel/4.7.4.release +module load esmf/8.0.0_ParallelNetCDF + +export CMAKE_C_COMPILER=mpiicc +export CMAKE_CXX_COMPILER=mpiicpc +export CMAKE_Fortran_COMPILER=mpiifort +export CMAKE_Platform=hera.intel + +git clone -b feature/cmake git@github.com:kgerheiser/ufs-srweather-app +#git clone -b release/public-v1 git@github.com:ufs-community/ufs-srweather-app + +cd ufs-srweather-app/ +./manage_externals/checkout_externals + +mkdir build +cd build +cmake .. -DCMAKE_INSTALL_PREFIX=.. +make + From a545ace8f6b6bb4d7975f56db70453f09735b690 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 6 Oct 2020 10:56:13 -0600 Subject: [PATCH 05/10] Fix git clone location for hera, add Cheyenne build instructions --- docs/README_cheyenne_intel.txt | 50 ++++++++++++++++++++++++++++++++++ docs/README_hera_intel.txt | 4 +-- 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 docs/README_cheyenne_intel.txt diff --git a/docs/README_cheyenne_intel.txt b/docs/README_cheyenne_intel.txt new file mode 100644 index 0000000000..cdd8ab0c84 --- /dev/null +++ b/docs/README_cheyenne_intel.txt @@ -0,0 +1,50 @@ +#Setup instructions for CISL Cheyenne using Intel-19.1.1 (bash shell) + +module purge +module load ncarenv/1.3 +module load intel/19.1.1 +module load mpt/2.19 +module load ncarcompilers/0.5.0 +module load cmake/3.16.4 + +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx + +NCEPLIBS_INSTALL=/glade/p/ral/jntp/GMTB/tools/NCEPLIBS-ufs-v2.0.0/intel-19.1.1/mpt-2.19 + +module use -a ${NCEPLIBS_INSTALL}/modules + +module load bacio/2.4.1 +module load g2/3.4.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load sigio/2.3.2 +module load g2tmpl/1.9.1 +module load sfcio/1.4.1 +module load gfsio/1.4.1 +module load nemsiogfs/2.5.3 +module load landsfcutil/2.4.1 +module load wgrib2/2.0.8 +module load netcdf/4.7.4 +module load crtm/2.3.0 + +export ESMFMKFILE=/glade/p/ral/jntp/GMTB/tools/NCEPLIBS-ufs-v2.0.0/intel-19.1.1/mpt-2.19/lib64/esmf.mk + +export CMAKE_C_COMPILER=mpicc +export CMAKE_CXX_COMPILER=mpicxx +export CMAKE_Fortran_COMPILER=mpif90 +export CMAKE_Platform=cheyenne.intel + +git clone -b release/public-v1 git@github.com:ufs-community/ufs-srweather-app + +cd ufs-srweather-app/ +./manage_externals/checkout_externals + +mkdir build +cd build +cmake .. -DCMAKE_INSTALL_PREFIX=.. +make diff --git a/docs/README_hera_intel.txt b/docs/README_hera_intel.txt index 9e6c6da16d..3b0387fc88 100644 --- a/docs/README_hera_intel.txt +++ b/docs/README_hera_intel.txt @@ -39,8 +39,7 @@ export CMAKE_CXX_COMPILER=mpiicpc export CMAKE_Fortran_COMPILER=mpiifort export CMAKE_Platform=hera.intel -git clone -b feature/cmake git@github.com:kgerheiser/ufs-srweather-app -#git clone -b release/public-v1 git@github.com:ufs-community/ufs-srweather-app +git clone -b release/public-v1 git@github.com:ufs-community/ufs-srweather-app cd ufs-srweather-app/ ./manage_externals/checkout_externals @@ -49,4 +48,3 @@ mkdir build cd build cmake .. -DCMAKE_INSTALL_PREFIX=.. make - From 626866b66c9dd5a824e82d6e67c1127f9b3f4c18 Mon Sep 17 00:00:00 2001 From: kgerheiser Date: Tue, 6 Oct 2020 13:05:25 -0400 Subject: [PATCH 06/10] Install weather model executable to bin --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5a0079083f..b326922f49 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,7 @@ ExternalProject_Add(ufs_weather_model "-DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER}" "-DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER}" "-DNETCDF_DIR=$ENV{NETCDF}" + INSTALL_COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/bin && cp ${CMAKE_CURRENT_BINARY_DIR}/ufs_weather_model/src/ufs_weather_model-build/NEMS.exe ${CMAKE_INSTALL_PREFIX}/bin/ ) ExternalProject_Add(EMC_post From 43f67857d9d57a74dec8490eee14b502d47657ea Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 6 Oct 2020 19:04:19 -0600 Subject: [PATCH 07/10] Add Jet build instructions --- docs/README_jet_intel.txt | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/README_jet_intel.txt diff --git a/docs/README_jet_intel.txt b/docs/README_jet_intel.txt new file mode 100644 index 0000000000..7dffb7289c --- /dev/null +++ b/docs/README_jet_intel.txt @@ -0,0 +1,54 @@ +#Setup instructions for NOAA RDHPC Jet using Intel-18.0.5.274 (bash shell) + +module purge + +module use -a /contrib/sutils/modulefiles +module load sutils + +module load intel/18.0.5.274 +module load impi/2018.4.274 +module load hdf5/1.10.4 +module load netcdf/4.6.1 +module load cmake/3.16.1 + +export CC=icc +export CXX=icpc +export FC=ifort + +NCEPLIBS_INSTALL=/lfs4/HFIP/hfv3gfs/software/NCEPLIBS-ufs-v2.0.0/intel-18.0.5.274/impi-2018.4.274 + +module use -a ${NCEPLIBS_INSTALL}/modules + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load landsfcutil/2.4.1 +module load nceppost/dceca26 +module load nemsio/2.5.2 +module load nemsiogfs/2.5.3 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 + +module load gfsio/1.4.1 +module load sfcio/1.4.1 +module load sigio/2.3.2 +module load esmf/8.0.0 +module load wgrib2/2.0.8 + +export CMAKE_C_COMPILER=mpiicc +export CMAKE_CXX_COMPILER=mpiicpc +export CMAKE_Fortran_COMPILER=mpiifort +export CMAKE_Platform=jet.intel + +git clone -b release/public-v1 git@github.com:ufs-community/ufs-srweather-app + +cd ufs-srweather-app/ +./manage_externals/checkout_externals + +mkdir build +cd build +cmake .. -DCMAKE_INSTALL_PREFIX=.. +make From a4c154191c4a531fa7e678df3550c33a96bae1be Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 7 Oct 2020 15:00:51 -0600 Subject: [PATCH 08/10] Updates to Hera README that fixes runtime library issues; thanks Dom! --- docs/README_hera_intel.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/README_hera_intel.txt b/docs/README_hera_intel.txt index 3b0387fc88..41e6480651 100644 --- a/docs/README_hera_intel.txt +++ b/docs/README_hera_intel.txt @@ -11,10 +11,13 @@ export FC=ifort NCEPLIBS_INSTALL=/scratch1/BMC/gmtb/software/NCEPLIBS-ufs-v2.0.0/intel-18.0.5.274/impi-2018.0.4 -module use -a ${NCEPLIBS_INSTALL}/modules +module use ${NCEPLIBS_INSTALL}/modules + +module load libpng/1.6.35 module load bacio/2.4.1 module load g2/3.4.1 +module load g2tmpl/1.9.1 module load ip/3.3.3 module load nemsio/2.5.2 module load sp/2.3.3 @@ -28,11 +31,8 @@ module load nemsiogfs/2.5.3 module load landsfcutil/2.4.1 module load wgrib2/2.0.8 module load netcdf/4.7.4 - -module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles -module load hdf5_parallel/1.10.6.release -module load netcdf_parallel/4.7.4.release -module load esmf/8.0.0_ParallelNetCDF +module load esmf/8.0.0 +module load crtm/2.3.0 export CMAKE_C_COMPILER=mpiicc export CMAKE_CXX_COMPILER=mpiicpc From f5b32a524c8f2918049a435a80911c4576d08808 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 8 Oct 2020 14:21:46 -0600 Subject: [PATCH 09/10] Update Externals.cfg to point to hash on Jeff's fork rather than the HEAD --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 867c8c21b3..dc87d0f6d9 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -9,7 +9,7 @@ required = True [ufs_utils] protocol = git repo_url = https://github.com/JeffBeck-NOAA/UFS_UTILS -branch = feature/regional_release +hash = 4488766 local_path = src/UFS_UTILS required = True From 8dc1c7b8e2a645c2249f9c899594f1f8efa590d6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 8 Oct 2020 21:29:33 +0000 Subject: [PATCH 10/10] Use tag for Jeff's UFS_UTILS rather than static hash --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index dc87d0f6d9..655813a3e2 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -9,7 +9,7 @@ required = True [ufs_utils] protocol = git repo_url = https://github.com/JeffBeck-NOAA/UFS_UTILS -hash = 4488766 +tag = v2.0alpha01 local_path = src/UFS_UTILS required = True