diff --git a/.gitignore b/.gitignore index b8a4ffcbba7..49c4b1e78a4 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ sorc/logs sorc/fv3gfs.fd sorc/gfs_post.fd sorc/gfs_wafs.fd +sorc/gldas.fd sorc/gsi.fd sorc/ufs_utils.fd sorc/verif-global.fd @@ -44,14 +45,12 @@ sorc/calc_increment_ens.fd sorc/calc_increment_ens_ncio.fd sorc/emcsfc_ice_blend.fd sorc/emcsfc_snow2mdl.fd -sorc/filter_topo.fd sorc/fregrid.fd sorc/gdas2gldas.fd sorc/getsfcensmeanp.fd sorc/getsigensmeanp_smooth.fd sorc/getsigensstatp.fd sorc/gfs_ncep_post.fd -sorc/gldas.fd sorc/gldas2gdas.fd sorc/gldas_forcing.fd sorc/gldas_model.fd @@ -64,13 +63,8 @@ sorc/global_gsi.fd sorc/interp_inc.fd sorc/make_hgrid.fd sorc/make_solo_mosaic.fd -sorc/mkgfsnemsioctl.fd sorc/ncdiag_cat.fd -sorc/nemsio_chgdate.fd -sorc/nemsio_get.fd -sorc/nemsio_read.fd sorc/nst_tf_chg.fd -sorc/orog.fd sorc/oznmon_horiz.fd sorc/oznmon_time.fd sorc/radmon_angle.fd @@ -78,7 +72,6 @@ sorc/radmon_bcoef.fd sorc/radmon_bcor.fd sorc/radmon_time.fd sorc/recentersigp.fd -sorc/shave.fd sorc/wafs_awc_wafavn.fd sorc/wafs_blending.fd sorc/wafs_blending_0p25.fd @@ -144,10 +137,10 @@ scripts/exglobal_diag.sh # ush symlinks ush/calcanl_gfs.py ush/calcinc_gfs.py +ush/chgres_cube.sh ush/emcsfc_ice_blend.sh ush/emcsfc_snow.sh ush/fix_precip.sh -ush/fv3gfs_chgres.sh ush/fv3gfs_downstream_nems.sh ush/fv3gfs_driver_grid.sh ush/fv3gfs_dwn_nems.sh diff --git a/Externals.cfg b/Externals.cfg index 20c610f1bc5..6ecac4c8bf0 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,35 +1,36 @@ # External sub-modules of global-workflow [FV3GFS] -tag = GFS.v16.0.16 +hash = 9350745855aebe0790813e0ed2ba5ad680e3f75c local_path = sorc/fv3gfs.fd repo_url = https://github.com/ufs-community/ufs-weather-model.git protocol = git required = True [GSI] -tag = gfsda.v16.1.0 +hash = 9c1fc15d42573b398037319bbf8d5143ad126fb6 local_path = sorc/gsi.fd repo_url = https://github.com/NOAA-EMC/GSI.git protocol = git required = True [GLDAS] -tag = gldas_gfsv16_release.v1.13.0 +tag = gldas_gfsv16_release.v1.15.0 local_path = sorc/gldas.fd repo_url = https://github.com/NOAA-EMC/GLDAS.git protocol = git required = True [EMC_post] -tag = upp_gfsv16_release.v1.1.3 +#No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules +hash = ff42e0227d6100285d4179a2572b700fd5a959cb local_path = sorc/gfs_post.fd repo_url = https://github.com/NOAA-EMC/EMC_post.git protocol = git required = True [UFS_UTILS] -tag = ops-gfsv16.0.0 +tag = ufs_utils_1_4_0 local_path = sorc/ufs_utils.fd repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git protocol = git @@ -43,7 +44,7 @@ protocol = git required = True [EMC_gfs_wafs] -tag = gfs_wafs.v6.0.22 +hash = c2a29a67d9432b4d6fba99eac7797b81d05202b6 local_path = sorc/gfs_wafs.fd repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git protocol = git diff --git a/env/HERA.env b/env/HERA.env index b4d722b590e..d833c358a1b 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -160,14 +160,6 @@ elif [ $step = "efcs" ]; then elif [ $step = "post" ]; then - source /apps/lmod/lmod/init/sh - module unload esmf/8.0.1_ParallelNetCDF.release - module unload netcdf_parallel/4.7.4.release - module unload hdf5_parallel/1.10.6.release - module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles - module load hdf5_parallel/1.10.6 - module load netcdf_parallel/4.7.4 - nth_max=$(($npe_node_max / $npe_node_post)) export NTHREADS_NP=${nth_np:-1} diff --git a/modulefiles/OznMonBuild.orion b/modulefiles/OznMonBuild.orion index efac4df0576..f88717d1817 100644 --- a/modulefiles/OznMonBuild.orion +++ b/modulefiles/OznMonBuild.orion @@ -10,8 +10,5 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl" export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug" -module load intel/2018.4 -#module load w3nco/2.0.6 -module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles -module load w3nco-intel-sandybridge/2.0.6 - +module load hpc-intel/2018.4 +module load w3nco/2.4.1 diff --git a/modulefiles/OznMonBuild.wcoss_dell_p3 b/modulefiles/OznMonBuild.wcoss_dell_p3 index efb31a7da31..6b4193792b7 100644 --- a/modulefiles/OznMonBuild.wcoss_dell_p3 +++ b/modulefiles/OznMonBuild.wcoss_dell_p3 @@ -10,6 +10,8 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl" export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug" -module load ips/18.0.1.163 -module load w3nco/2.0.6 - +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 +module load w3nco/2.4.1 diff --git a/modulefiles/RadMonBuild.orion b/modulefiles/RadMonBuild.orion index 932bb6f95ba..411f2d8c8d8 100644 --- a/modulefiles/RadMonBuild.orion +++ b/modulefiles/RadMonBuild.orion @@ -11,7 +11,5 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash" export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug" -module load intel/2018.4 -#module load w3nco/2.0.6 -module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles -module load w3nco-intel-sandybridge/2.0.6 +module load hpc-intel/2018.4 +module load w3nco/2.4.1 diff --git a/modulefiles/RadMonBuild.wcoss_dell_p3 b/modulefiles/RadMonBuild.wcoss_dell_p3 index 9cece6e1166..1c85e93ded0 100644 --- a/modulefiles/RadMonBuild.wcoss_dell_p3 +++ b/modulefiles/RadMonBuild.wcoss_dell_p3 @@ -11,5 +11,8 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash" export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug" -module load ips/18.0.1.163 -module load w3nco/2.0.6 +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 +module load w3nco/2.4.1 diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index adb8370a29f..3b885306a1a 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -3,28 +3,49 @@ ## FV3GFS prerequisites ## -module load intel/18.0.5.274 -module load impi/2018.0.4 -module load wgrib2/2.0.8 +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.0.4 + module load hpss/hpss -module load nco/4.7.0 +module load nco/4.9.1 module load gempak/7.4.2 -#Load from official NCEPLIBS -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load prod_util/1.1.0 -module load grib_util/1.1.1 -module load g2tmpl/1.6.0 +#Load from hpc-stack +module load prod_util/1.2.2 +module load grib_util/1.2.2 + module load crtm/2.3.0 +setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 -module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/ -module load netcdf_parallel/4.7.4.release -module load hdf5_parallel/1.10.6.release -module load esmf/8.0.1_ParallelNetCDF.release +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.6 + +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 # python module use -a /contrib/anaconda/modulefiles module load anaconda/2.3.0 # waveprep -module load cdo/1.9.5 +module load cdo/1.9.5 + diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion index facf50d4009..dacf9e1e847 100755 --- a/modulefiles/module_base.orion +++ b/modulefiles/module_base.orion @@ -3,37 +3,50 @@ ## FV3GFS prerequisites ## -module load intel/2018.4 -module load impi/2018.4 -module load wgrib/2.0.8 +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-intel/2018.4 +module load hpc-impi/2018.4 + #module load hpss/hpss module load nco/4.8.1 -module load cdo/1.9.5 - module load gempak/7.5.1 -#Load from official NCEPLIBS -module use /apps/contrib/NCEPLIBS/orion/modulefiles -module load grib_util/1.2.0 -module load prod_util/1.2.0 -module load g2tmpl/1.6.0 +module load grib_util/1.2.2 +module load prod_util/1.2.2 + module load crtm/2.3.0 -#module load netcdf_parallel/4.7.4 -module load hdf5_parallel/1.10.6 - -## load ESMF library for above compiler / MPI combination -### use pre-compiled EMSF library for above compiler / MPI combination -### -module use /apps/contrib/NCEPLIBS/lib/modulefiles -module load netcdfp/4.7.4 -#module load hdf5_parallel/1.10.6 -module load esmflocal/8.0.1.08bs -module load post-intel-sandybridge/8.0.5 +setenv CRTM_FIX /apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.6 + +module load wgrib/2.0.8 +setenv WGRIB2 wgrib2 module load contrib -module load rocoto/1.3.2 +module load rocoto/1.3.3 module load slurm/19.05.3-2 # Python module load python/3.7.5 +# waveprep +module load cdo/1.9.5 diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3 index 0bd8bd436b0..9945cbbefd3 100755 --- a/modulefiles/module_base.wcoss_dell_p3 +++ b/modulefiles/module_base.wcoss_dell_p3 @@ -4,30 +4,53 @@ # From default environment -module load ips/18.0.1.163 -module load impi/18.0.1 +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 + module load lsf/10.1 module load EnvVars/1.0.3 - module load HPSS/5.0.2.5 -module load prod_util/1.1.4 + +module load prod_util/1.2.2 module load prod_envir/1.1.0 -module unload grib_util/1.0.6 -module load grib_util/1.1.0 +module load grib_util/1.2.2 +module load util_shared/1.3.0 + +module load crtm/2.3.0 +setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix module load NCO/4.7.0 module load CFP/2.0.2 setenv USE_CFP YES module load pm5 -module load NetCDF-parallel/4.7.4 -module load ESMF/8.0.1 -module load HDF5-parallel/1.10.6 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.6 + +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/ module load gempak/7.3.3 -module load g2tmpl/1.6.0 -module load util_shared/1.3.0 # Load for WAFS module load bufr_dumplist/2.0.0 @@ -36,8 +59,5 @@ module load dumpjb/5.1.0 # python module load python/3.6.3 -# Load crtm -module load crtm/2.3.0 - # waveprep module load cdo/1.9.8 diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera new file mode 100644 index 00000000000..fd3f0830315 --- /dev/null +++ b/modulefiles/modulefile.ww3.hera @@ -0,0 +1,32 @@ +#%Module###################################################################### +## +## S2S prerequisites +## + +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.0.4 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 +module load upp/10.0.6 diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion new file mode 100644 index 00000000000..1bc98a32f79 --- /dev/null +++ b/modulefiles/modulefile.ww3.orion @@ -0,0 +1,33 @@ +#%Module###################################################################### +# module for ww3 before base uses hpc-stack +module load contrib noaatools + +module load cmake/3.17.3 + +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/2018.4 +module load hpc-impi/2018.4 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 +module load upp/10.0.6 diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3 new file mode 100644 index 00000000000..20e84f704fc --- /dev/null +++ b/modulefiles/modulefile.ww3.wcoss_dell_p3 @@ -0,0 +1,28 @@ +#%Module###################################################################### + +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8_1_1 +module load fms/2020.04.03 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 +module load upp/10.0.6 diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index e68d0c0b0ab..588cca49233 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -71,8 +71,8 @@ export MODE="@MODE@" # cycled/free export FIXgsi="$HOMEgfs/fix/fix_gsi" export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd" export HOMEpost="$HOMEgfs" -export HOMEobsproc_prep="$BASE_GIT/obsproc/gfsv16b/obsproc_prep_RB-5.4.0" -export HOMEobsproc_network="$BASE_GIT/obsproc/gfsv16b/obsproc_global_RB-3.4.0" +export HOMEobsproc_prep="$BASE_GIT/obsproc/obsproc_prep_RB-5.4.0_hpc-stack" +export HOMEobsproc_network="$BASE_GIT/obsproc/obsproc_global_RB-3.4.0_hpc-stack" export HOMEobsproc_global=$HOMEobsproc_network export BASE_VERIF="$BASE_GIT/verif/global/tags/vsdb" diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 603582fc0ea..fd0b28cd737 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -125,8 +125,8 @@ export CASE="C768" export CASE_ENKF="C384" # Run with CCPP physics -export RUN_CCPP="NO" -export CCPP_SUITE="FV3_GFS_v16beta" +export RUN_CCPP="YES" +export CCPP_SUITE="FV3_GFS_v16" # Surface cycle update frequency if [[ "$CDUMP" == "gdas" ]] ; then diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index ea79d76d948..fb307abb7a6 100755 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -142,14 +142,8 @@ fi # Cyclone genesis and cyclone track verification #------------------------------------------------- -export ens_tracker_ver=v1.1.15.1 -if [ $machine = "WCOSS_DELL_P3" ] ; then - export ens_tracker_ver=v1.1.15.3 -fi -export HOMEens_tracker=$BASE_GIT/tracker/ens_tracker.${ens_tracker_ver} -if [ $machine = "ORION" ] ; then - export HOMEens_tracker=$BASE_GIT/tracker/TC_tracker.v1.1.15.2 -fi +export ens_tracker_ver=v1.1.15.4 +export HOMEens_tracker=$BASE_GIT/TC_tracker/TC_tracker.${ens_tracker_ver} if [ "$VRFYTRAK" = "YES" ]; then diff --git a/parm/config/config.wafsgcip b/parm/config/config.wafsgcip index 5c166a541b6..793dae1694c 100755 --- a/parm/config/config.wafsgcip +++ b/parm/config/config.wafsgcip @@ -11,5 +11,7 @@ export COMIN=$COMINatmos export COMINgfs=$COMIN export COMOUT=$COMOUTatmos export SENDCOM="YES" +export COMLISTROOT="$UTILROOT/save/config" +export COMDATEROOT="N/A" echo "END: config.wafsgcip" diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 6c027efc792..b2fae913a44 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -60,6 +60,20 @@ fi ((err+=$rc)) } +#------------------------------------ +# build WW3 pre & post execs +#------------------------------------ +$Build_ww3_prepost && { +echo " .... Building WW3 pre and post execs .... " +./build_ww3prepost.sh > $logs_dir/build_ww3_prepost.log 2>&1 +rc=$? +if [[ $rc -ne 0 ]] ; then + echo "Fatal error in building WW3 pre/post processing." + echo "The log file is in $logs_dir/build_ww3_prepost.log" +fi +((err+=$rc)) +} + #------------------------------------ # build gsi #------------------------------------ @@ -149,19 +163,16 @@ fi #------------------------------------ # build gfs_util #------------------------------------ -# Only build on WCOSS -if [ $target = wcoss -o $target = wcoss_cray -o $target = wcoss_dell_p3 ]; then - $Build_gfs_util && { - echo " .... Building gfs_util .... " - ./build_gfs_util.sh > $logs_dir/build_gfs_util.log 2>&1 - rc=$? - if [[ $rc -ne 0 ]] ; then - echo "Fatal error in building gfs_util." - echo "The log file is in $logs_dir/build_gfs_util.log" - fi - ((err+=$rc)) - } +$Build_gfs_util && { +echo " .... Building gfs_util .... " +./build_gfs_util.sh > $logs_dir/build_gfs_util.log 2>&1 +rc=$? +if [[ $rc -ne 0 ]] ; then + echo "Fatal error in building gfs_util." + echo "The log file is in $logs_dir/build_gfs_util.log" fi +((err+=$rc)) +} #------------------------------------ # Exception Handling diff --git a/sorc/build_gfs_wafs.sh b/sorc/build_gfs_wafs.sh index ceff4eae670..921d97f1173 100755 --- a/sorc/build_gfs_wafs.sh +++ b/sorc/build_gfs_wafs.sh @@ -17,6 +17,6 @@ if [ ! -d "../exec" ]; then fi cd gfs_wafs.fd/sorc -sh build_wafs.sh +./build_wafs_cmake.sh exit diff --git a/sorc/build_ncep_post.sh b/sorc/build_ncep_post.sh index 5bbc2f680af..95aa0caedd7 100755 --- a/sorc/build_ncep_post.sh +++ b/sorc/build_ncep_post.sh @@ -16,5 +16,5 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -cd gfs_post.fd/sorc -sh build_ncep_post.sh +cd gfs_post.fd/tests +./compile_upp.sh diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh new file mode 100755 index 00000000000..d2554d038e7 --- /dev/null +++ b/sorc/build_ww3prepost.sh @@ -0,0 +1,96 @@ +#!/bin/sh +set -x + +# Check final exec folder exists +if [ ! -d "../exec" ]; then + mkdir ../exec +fi + +finalexecdir=$( pwd -P )/../exec + +set +x +source ./machine-setup.sh > /dev/null 2>&1 +source ../modulefiles/modulefile.ww3.$target +set -x + + +if [ $target = hera ]; then target=hera.intel ; fi +if [ $target = orion ]; then target=orion.intel ; fi +if [ $target = stampede ]; then target=stampede.intel ; fi + +#cd ufs_coupled.fd/WW3 +cd fv3gfs.fd/WW3 +export WW3_DIR=$( pwd -P )/model +export WW3_BINDIR="${WW3_DIR}/bin" +export WW3_TMPDIR=${WW3_DIR}/tmp +export WW3_EXEDIR=${WW3_DIR}/exe +export WW3_COMP=$target +export WW3_CC=gcc +export WW3_F90=gfortran +export SWITCHFILE="${WW3_DIR}/esmf/switch" + +export WWATCH3_ENV=${WW3_BINDIR}/wwatch3.env +export PNG_LIB=${PNG_LIB:-$PNG_ROOT/lib64/libpng.a} +export Z_LIB=${Z_LIB:-$ZLIB_ROOT/lib/libz.a} +export JASPER_LIB=${JASPER_LIB:-$JASPER_ROOT/lib64/libjasper.a} +export WWATCH3_NETCDF=NC4 +export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config + +rm $WWATCH3_ENV +echo '#' > $WWATCH3_ENV +echo '# ---------------------------------------' >> $WWATCH3_ENV +echo '# Environment variables for wavewatch III' >> $WWATCH3_ENV +echo '# ---------------------------------------' >> $WWATCH3_ENV +echo '#' >> $WWATCH3_ENV +echo "WWATCH3_LPR $PRINTER" >> $WWATCH3_ENV +echo "WWATCH3_F90 $WW3_F90" >> $WWATCH3_ENV +echo "WWATCH3_CC $WW3_CC" >> $WWATCH3_ENV +echo "WWATCH3_DIR $WW3_DIR" >> $WWATCH3_ENV +echo "WWATCH3_TMP $WW3_TMPDIR" >> $WWATCH3_ENV +echo 'WWATCH3_SOURCE yes' >> $WWATCH3_ENV +echo 'WWATCH3_LIST yes' >> $WWATCH3_ENV +echo '' >> $WWATCH3_ENV + +${WW3_BINDIR}/w3_clean -m +${WW3_BINDIR}/w3_setup -q -c $WW3_COMP $WW3_DIR + +echo $(cat ${SWITCHFILE}) > ${WW3_BINDIR}/tempswitch + +sed -e "s/DIST/SHRD/g"\ + -e "s/OMPG/ /g"\ + -e "s/OMPH/ /g"\ + -e "s/MPIT/ /g"\ + -e "s/MPI/ /g"\ + -e "s/PDLIB/ /g"\ + ${WW3_BINDIR}/tempswitch > ${WW3_BINDIR}/switch + +#Build exes for prep jobs: +${WW3_BINDIR}/w3_make ww3_grid +${WW3_BINDIR}/w3_make ww3_prep +${WW3_BINDIR}/w3_make ww3_prnc + +#Build exes for post jobs (except grib)" +${WW3_BINDIR}/w3_make ww3_outp +${WW3_BINDIR}/w3_make ww3_outf +${WW3_BINDIR}/w3_make ww3_outp +${WW3_BINDIR}/w3_make ww3_gint +${WW3_BINDIR}/w3_make ww3_ounf +${WW3_BINDIR}/w3_make ww3_ounp + +#Update switch for grib: +echo $(cat ${SWITCHFILE}) > ${WW3_BINDIR}/tempswitch + +sed -e "s/DIST/SHRD/g"\ + -e "s/OMPG/ /g"\ + -e "s/OMPH/ /g"\ + -e "s/MPIT/ /g"\ + -e "s/MPI/ /g"\ + -e "s/PDLIB/ /g"\ + -e "s/NOGRB/NCEP2 NCO/g"\ + ${WW3_BINDIR}/tempswitch > ${WW3_BINDIR}/switch +#Build exe for grib +${WW3_BINDIR}/w3_make ww3_grib + +cp $WW3_EXEDIR/ww3_* $finalexecdir/ +${WW3_BINDIR}/w3_clean -c +rm ${WW3_BINDIR}/tempswitch diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 9ac4b97ea83..dbdcd97d844 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -2,12 +2,13 @@ #set -xue set -x -while getopts "oc" option; +while getopts "o" option; do case $option in o) - echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post" + echo "Received -o flag for optional checkout of operational-only codes" checkout_gtg="YES" + checkout_wafs="YES" ;; :) echo "option -$OPTARG needs an argument" @@ -39,7 +40,7 @@ if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1 cd gsi.fd - git checkout gfsda.v16.1.0 + git checkout 9c1fc15d42573b398037319bbf8d5143ad126fb6 git submodule update cd ${topdir} else @@ -51,7 +52,7 @@ if [[ ! -d gldas.fd ]] ; then rm -f ${topdir}/checkout-gldas.log git clone https://github.com/NOAA-EMC/GLDAS.git gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1 cd gldas.fd - git checkout gldas_gfsv16_release.v1.13.0 + git checkout gldas_gfsv16_release.v1.15.0 cd ${topdir} else echo 'Skip. Directory gldas.fd already exists.' @@ -60,7 +61,7 @@ fi echo ufs_utils checkout ... if [[ ! -d ufs_utils.fd ]] ; then rm -f ${topdir}/checkout-ufs_utils.log - git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 + git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1 cd ufs_utils.fd git checkout ufs_utils_1_4_0 cd ${topdir} @@ -73,7 +74,9 @@ if [[ ! -d gfs_post.fd ]] ; then rm -f ${topdir}/checkout-gfs_post.log git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1 cd gfs_post.fd - git checkout upp_gfsv16_release.v1.1.3 + git checkout ff42e0227d6100285d4179a2572b700fd5a959cb + git submodule update --init CMakeModules + ################################################################################ # checkout_gtg ## yes: The gtg code at NCAR private repository is available for ops. GFS only. @@ -83,7 +86,7 @@ if [[ ! -d gfs_post.fd ]] ; then checkout_gtg=${checkout_gtg:-"NO"} if [[ ${checkout_gtg} == "YES" ]] ; then ./manage_externals/checkout_externals - cp sorc/post_gtg.fd/*f90 sorc/ncep_post.fd/. + cp sorc/post_gtg.fd/*F90 sorc/ncep_post.fd/. cp sorc/post_gtg.fd/gtg.config.gfs parm/gtg.config.gfs fi cd ${topdir} @@ -91,15 +94,18 @@ else echo 'Skip. Directory gfs_post.fd already exists.' fi -echo EMC_gfs_wafs checkout ... -if [[ ! -d gfs_wafs.fd ]] ; then +checkout_wafs=${checkout_wafs:-"NO"} +if [[ ${checkout_wafs} == "YES" ]] ; then + echo EMC_gfs_wafs checkout ... + if [[ ! -d gfs_wafs.fd ]] ; then rm -f ${topdir}/checkout-gfs_wafs.log git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1 cd gfs_wafs.fd - git checkout gfs_wafs.v6.0.22 + git checkout c2a29a67d9432b4d6fba99eac7797b81d05202b6 cd ${topdir} -else + else echo 'Skip. Directory gfs_wafs.fd already exists.' + fi fi echo EMC_verif-global checkout ... diff --git a/sorc/fv3gfs_build.cfg b/sorc/fv3gfs_build.cfg index 161a21325ef..601b13713a7 100644 --- a/sorc/fv3gfs_build.cfg +++ b/sorc/fv3gfs_build.cfg @@ -2,6 +2,7 @@ # ***** configuration of global-workflow build ***** Building fv3gfs (fv3gfs) .............................. yes + Building ww3_prepost (ww3_prepost) .................... yes Building gsi (gsi) .................................... yes Building gldas (gldas) ................................ yes Building ncep_post (ncep_post) ........................ yes diff --git a/sorc/gfs_bufr.fd/makefile_module b/sorc/gfs_bufr.fd/makefile_module index c640d0a9278..d9d5374a7a2 100755 --- a/sorc/gfs_bufr.fd/makefile_module +++ b/sorc/gfs_bufr.fd/makefile_module @@ -8,20 +8,20 @@ FC = $(myFC) $(myFCFLAGS) CPP = $(myCPP) $(myCPPFLAGS) -FFLAGS = -I$(NETCDF_INC) \ +FFLAGS = -I$(NETCDF_INCLUDES) \ -I$(NEMSIO_INC) \ - -I$(SIGIO_INC4) \ + -I$(SIGIO_INC) \ -I$(W3EMC_INC4) -LIBS = -L$(NETCDF_LIB) -lnetcdf -lnetcdff \ - -L$(HDF5_LIB) -lhdf5_hl -lhdf5 \ +LIBS = -L$(NETCDF_LIBRARIES) -lnetcdff -lnetcdf \ + -L$(HDF5_LIBRARIES) -lhdf5_hl -lhdf5 -lz \ $(NEMSIO_LIB) \ $(W3EMC_LIB4) \ $(W3NCO_LIB4) \ $(BUFR_LIB4) \ $(BACIO_LIB4) \ $(SP_LIB4) \ - $(SIGIO_LIB4) + $(SIGIO_LIB) SRCM = gfsbufr.f OBJS = physcons.o funcphys.o meteorg.o bfrhdr.o newsig1.o terp3.o\ diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index fe823d2ee99..7869afb22e3 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -40,7 +40,8 @@ elif [ $machine = "orion" ]; then FIX_DIR="/work/noaa/global/glopara/fix" fi cd ${pwd}/../fix ||exit 8 -for dir in fix_aer fix_am fix_chem fix_fv3_gmted2010 fix_gldas fix_lut fix_orog fix_sfc_climo fix_verif fix_wave_gfs ; do +dirs=`ls $FIX_DIR` +for dir in $dirs ; do if [ -d $dir ]; then [[ $RUN_ENVIR = nco ]] && chmod -R 755 $dir rm -rf $dir @@ -77,7 +78,8 @@ cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gfs_post.fd/ush/$file . done for file in emcsfc_ice_blend.sh fv3gfs_driver_grid.sh fv3gfs_make_orog.sh global_cycle_driver.sh \ - emcsfc_snow.sh fv3gfs_filter_topo.sh global_cycle.sh fv3gfs_make_grid.sh ; do + emcsfc_snow.sh fv3gfs_filter_topo.sh global_cycle.sh \ + chgres_cube.sh fv3gfs_make_grid.sh ; do $LINK ../sorc/ufs_utils.fd/ush/$file . done for file in gldas_archive.sh gldas_forcing.sh gldas_get_data.sh gldas_process_data.sh gldas_liscrd.sh gldas_post.sh ; do @@ -212,21 +214,15 @@ done [[ -s global_fv3gfs.x ]] && rm -f global_fv3gfs.x $LINK ../sorc/fv3gfs.fd/NEMS/exe/global_fv3gfs.x . -if [ -d ../sorc/fv3gfs.fd/WW3/exec ]; then # Wave execs - for waveexe in ww3_gint ww3_grib ww3_grid ww3_multi ww3_ounf ww3_ounp ww3_outf ww3_outp ww3_prep ww3_prnc; do - [[ -s $waveexe ]] && rm -f $waveexe - $LINK ../sorc/fv3gfs.fd/WW3/exec/$waveexe . - done -fi [[ -s gfs_ncep_post ]] && rm -f gfs_ncep_post -$LINK ../sorc/gfs_post.fd/exec/ncep_post gfs_ncep_post +$LINK ../sorc/gfs_post.fd/exec/upp.x gfs_ncep_post if [ -d ${pwd}/gfs_wafs.fd ]; then for wafsexe in \ - wafs_awc_wafavn wafs_blending wafs_blending_0p25 \ - wafs_cnvgrib2 wafs_gcip wafs_grib2_0p25 \ - wafs_makewafs wafs_setmissing; do + wafs_awc_wafavn.x wafs_blending.x wafs_blending_0p25.x \ + wafs_cnvgrib2.x wafs_gcip.x wafs_grib2_0p25.x \ + wafs_makewafs.x wafs_setmissing.x ; do [[ -s $wafsexe ]] && rm -f $wafsexe $LINK ../sorc/gfs_wafs.fd/exec/$wafsexe . done diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 5f28d6abaa9..59e9ad89479 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -27,12 +27,13 @@ if [[ -d /work ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi target=orion + module purge - module load intel/2018.4 - module load impi/2018.4 - export NCEPLIBS=/apps/contrib/NCEPLIBS/orion - export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src - module use $NCEPLIBS/modulefiles + module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack + module load hpc/1.1.0 + module load hpc-intel/2018.4 + module load hpc-impi/2018.4 + export myFC=mpiifort export FCOMP=mpiifort @@ -44,9 +45,12 @@ elif [[ -d /scratch1 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi target=hera + module purge module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc/1.1.0 + module load hpc-intel/18.0.5.274 + module load hpc-impi/2018.0.4 module load cmake/3.20.0 export myFC=mpiifort @@ -94,21 +98,12 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then fi target=wcoss_dell_p3 module purge - module use /gpfs/dell2/usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles + module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc/1.1.0 + module load hpc-ips/18.0.1.163 + module load hpc-impi/18.0.1 module load cmake/3.20.0 -##--------------------------------------------------------------------------- - -elif [[ -d /dcom && -d /hwrf ]] ; then - # We are on NOAA Tide or Gyre - if ( ! eval module help > /dev/null 2>&1 ) ; then - echo load the module command 1>&2 - source /usrx/local/Modules/default/init/$__ms_shell - fi - target=wcoss - module purge - ##--------------------------------------------------------------------------- elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh index 691a3c7b3a1..3be40418805 100755 --- a/sorc/partial_build.sh +++ b/sorc/partial_build.sh @@ -2,6 +2,7 @@ # define the array of the name of build program # declare -a Build_prg=("Build_fv3gfs" \ + "Build_ww3_prepost" \ "Build_gsi" \ "Build_gldas" \ "Build_ncep_post" \ diff --git a/sorc/tocsbufr.fd/makefile_module b/sorc/tocsbufr.fd/makefile_module index b6310ff7659..06f5ba7092b 100755 --- a/sorc/tocsbufr.fd/makefile_module +++ b/sorc/tocsbufr.fd/makefile_module @@ -47,7 +47,7 @@ LIBS = $(W3EMC_LIB4) \ $(BUFR_LIB4) \ $(BACIO_LIB4) \ $(SP_LIB4) \ - $(SIGIO_LIB4) + $(SIGIO_LIB) CMD = ../../exec/tocsbufr PROFLIB = -lprof diff --git a/util/modulefiles/gfs_util.hera b/util/modulefiles/gfs_util.hera index cf880d66b66..ac8a7d941cd 100644 --- a/util/modulefiles/gfs_util.hera +++ b/util/modulefiles/gfs_util.hera @@ -5,25 +5,24 @@ # Loading required system modules # -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -# Loading Intel Compiler Suite -module load intel/18.0.5.274 +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.0.4 -module load bacio/2.0.3 -module load w3emc/2.3.1 -module load w3nco/2.0.6 -module load ip/3.0.1 -module load sp/2.0.2 -module load bufr/11.3.0 +module load bacio/2.4.1 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load ip/3.3.3 +module load sp/2.3.3 +module load bufr/11.4.0 -module load jasper/1.900.1 -module load png/1.2.44 -module load z/1.2.11 +module load jasper/2.0.22 +module load png/1.6.35 +module load zlib/1.2.11 module load ncl/6.5.0 -module load gempak/7.4.2 - -module load impi/2018.0.4 +module load gempak/7.4.2 export GEMINC=/apps/gempak/7.4.2/gempak/include export GEMOLB=/apps/gempak/7.4.2/os/linux64/lib diff --git a/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh b/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh index f6800a1943b..ccce82b0e52 100755 --- a/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh +++ b/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh @@ -12,7 +12,7 @@ cwd=`pwd` if [ "$target" = "wcoss_dell_p3" ] || [ "$target" = "wcoss_cray" ] || [ "$target" = "hera" ] ; then echo " " - echo " You are on WCOSS: $target " + echo " You are on $target " echo " " elif [ "$target" = "wcoss" ] ; then echo " "