diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index 8eadf74e2d6..146af1ef443 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -3,54 +3,40 @@ ## S2S prerequisites ## -module load intel/18.0.5.274 -module load impi/2018.0.4 -module load wgrib2/2.0.8 +# Modules not under hpc-stack module load hpss/hpss -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load hdf5_parallel/1.10.6 -module load netcdf_parallel/4.7.4 -module load netcdf/4.7.0 -module load nco/4.7.0 - +module load nco/4.9.1 module load gempak/7.4.2 module load ncl/6.5.0 - -#Load from official NCEPLIBS -module load g2tmpl/1.6.0 -module load grib_util/1.1.1 -module load crtm/2.2.6 -module load prod_util/1.1.0 -module load grib_util/1.1.1 -module load g2tmpl/1.6.0 -module load crtm/2.3.0 - -module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/ -module load netcdf_parallel/4.7.4.release -module load hdf5_parallel/1.10.6.release -module load esmf/8.0.1_ParallelNetCDF.release - -# python -module use -a /contrib/anaconda/modulefiles -module load anaconda/2.3.0 - -module load bacio/2.0.3 -module load ip/3.0.2 -module load nemsio/2.2.4 -module load sp/2.0.3 -module load w3emc/2.3.1 -module load w3nco/2.0.7 -module load g2/3.1.1 -module load jasper/1.900.1 -module load png/1.2.44 -module load z/1.2.11 - - -#Load from emc.nemspara -module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles -module load netcdf_parallel/4.7.4.release -module load esmf/8.1.0bs27 -module load hdf5_parallel/1.10.6.release - -# waveprep module load cdo/1.9.5 + +# Remaining modules availble through hpc-stack +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack + +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.0.4 + +# Non-MPI +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load grib_util/1.2.2 +module load crtm 2.3.0 +module load prod_util/1.2.2 +module load grib_util/1.2.2 +module load ip/3.3.3 +module load sp/2.3.3 +module load w3nco/2.4.1 +module load bacio/2.4.1 +module load landsfcutil/2.4.1 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +# MPI +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load esmf/8_1_0_beta_snapshot_27 +module load w3emc/2.7.3 +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion index 2f780597066..9fbc6a081b4 100644 --- a/modulefiles/module_base.orion +++ b/modulefiles/module_base.orion @@ -8,53 +8,40 @@ ## module load contrib noaatools -## -## load programming environment -## this typically includes compiler, MPI and job scheduler -## -module load intel/2018 -module load impi/2018 - -#For ocean post: +module load nco/4.9.3 +module load gempak/7.5.1 module load ncl/6.6.2 -module load nco/4.8.1 - -## -## NCEP libraries (temporary version to match the CCPP requirements) -## -module use -a /apps/contrib/NCEPLIBS/orion/cmake/install/NCEPLIBS/modules -module load bacio/2.4.0 -module load crtm_dev/2.3.0 -module load g2/3.4.0 -module load g2tmpl/1.9.0 -module load ip/3.3.0 -module load nceppost/dceca26 -module load nemsio/2.5.1 -module load sp/2.3.0 -module load w3emc/2.7.0 -module load w3nco/2.4.0 - -module load gfsio/1.4.0 -module load sfcio/1.4.0 -module load sigio/2.3.0 - -module use /apps/contrib/NCEPLIBS/orion/modulefiles -module load jasper/1.900.2 -module load png/1.2.44 -module load z/1.2.6 - -module load prod_util/1.2.0 -## -## load ESMF library for above compiler / MPI combination -## use pre-compiled EMSF library for above compiler / MPI combination -## -module use /apps/contrib/NCEPLIBS/lib/modulefiles -module load netcdfp/4.7.4.release -module load esmflocal/8.1.0.27bs.release -module load post-intel-sandybridge/8.0.5 - -module load wgrib/2.0.8 -module load grib_util/1.2.0 +module load cdo/1.9.8 + +# Remaining modules availble through hpc-stack +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-intel/2018.4 +module load hpc-impi/2018.4 + +# Non-MPI +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load grib_util/1.2.2 +module load crtm 2.3.0 +module load prod_util/1.2.2 +module load grib_util/1.2.2 +module load ip/3.3.3 +module load sp/2.3.3 +module load w3nco/2.4.1 +module load bacio/2.4.1 +module load landsfcutil/2.4.1 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +# MPI +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load esmf/8_1_0_beta_snapshot_27 +module load w3emc/2.7.3 +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 module load slurm/19.05.3-2 diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3 index 4cfed7fb5ff..e6dee12e6d8 100644 --- a/modulefiles/module_base.wcoss_dell_p3 +++ b/modulefiles/module_base.wcoss_dell_p3 @@ -1,43 +1,49 @@ #%Module###################################################################### -## -## FV3GFS prerequisites -# From default environment +# Remaining modules availble through hpc-stack +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 -module load ips/18.0.1.163 -module load impi/18.0.1 -module load lsf/10.1 -module load EnvVars/1.0.3 - -module load HPSS/5.0.2.5 -module load prod_util/1.1.4 -module load prod_envir/1.1.0 -module unload grib_util/1.0.6 -module load grib_util/1.1.0 - -module load NCO/4.7.0 +# Not hpc-stack, but must come after impi load module load CFP/2.0.2 setenv USE_CFP YES -module load pm5 - -module load NetCDF-parallel/4.7.4 -module load ESMF/8.0.1 -module load HDF5-parallel/1.10.6 - -module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/ -module load gempak/7.3.1 -module load g2tmpl/1.6.0 -# Load for WAFS -module load bufr_dumplist/2.0.0 -module load dumpjb/5.1.0 -module load util_shared/1.1.2 - -# python -module load python/3.6.3 +# Non-MPI +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load grib_util/1.2.2 +module load crtm 2.3.0 +module load prod_util/1.2.2 +module load ip/3.3.3 +module load sp/2.3.3 +module load w3nco/2.4.1 +module load bacio/2.4.1 +module load landsfcutil/2.4.1 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +# MPI +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load esmf/8_1_0_beta_snapshot_27 +module load w3emc/2.7.3 +module load wgrib2/2.0.8 +setenv WGRIB2 wgrib2 + + +# Modules not under hpc-stack +module load HPSS/5.0.2.5 +module load NCO/4.7.0 +module load gempak/7.3.3 +module load NCL/6.4.0 +module load cdo/1.9.8 -# Load crtm -module load crtm/2.3.0 +# WCOSS-specific +module load lsf/10.1 +module load EnvVars/1.0.3 +module load prod_envir/1.1.0 +module load pm5/1.0 -# waveprep -module load cdo/1.9.8 diff --git a/modulefiles/modulefile.reg2grb2.hera b/modulefiles/modulefile.reg2grb2.hera index abe64952274..dd87cbcc7db 100644 --- a/modulefiles/modulefile.reg2grb2.hera +++ b/modulefiles/modulefile.reg2grb2.hera @@ -2,18 +2,20 @@ # reg2grb2 - hera ############################################################# -module use /scratch2/NCEPDEV/nwprod/hpc-stack/test/modulefiles/stack +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack -module load hpc/1.0.0-beta1 +module load hpc/1.1.0 module load hpc-intel/18.0.5.274 module load hpc-impi/2018.0.4 +export FCMP="mpiifort" module load netcdf/4.7.4 +module load hdf5/1.10.6 module load ip/3.3.3 module load sp/2.3.3 module load w3nco/2.4.1 module load bacio/2.4.1 module load landsfcutil/2.4.1 - -module load wgrib2/2.0.8 -set FCMP ifort +module load jasper/2.0.25 +module load zlib/1.2.11 +module load wgrib2/2.0.8 diff --git a/modulefiles/modulefile.reg2grb2.orion b/modulefiles/modulefile.reg2grb2.orion index a3cfff38a69..7be76ec2b10 100644 --- a/modulefiles/modulefile.reg2grb2.orion +++ b/modulefiles/modulefile.reg2grb2.orion @@ -2,17 +2,20 @@ # reg2grb2 - orion ############################################################# -module use /apps/contrib/NCEP/test/hpc-stack-nco/modulefiles/stack -module load hpc/1.0.0-beta1 +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack + +module load hpc/1.1.0 module load hpc-intel/2018.4 module load hpc-impi/2018.4 +export FCMP="mpiifort" -export FCOMP=ifort module load netcdf/4.7.4 +module load hdf5/1.10.6 module load ip/3.3.3 module load sp/2.3.3 module load w3nco/2.4.1 module load bacio/2.4.1 module load landsfcutil/2.4.1 - -module load wgrib/2.0.8 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load wgrib2/2.0.8 diff --git a/modulefiles/modulefile.reg2grb2.wcoss_dell_p3 b/modulefiles/modulefile.reg2grb2.wcoss_dell_p3 new file mode 100644 index 00000000000..2900ec0b4ff --- /dev/null +++ b/modulefiles/modulefile.reg2grb2.wcoss_dell_p3 @@ -0,0 +1,26 @@ +#%Module###################################################################### + +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load esmf/8_1_0_beta_snapshot_36 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nceppost/dceca26 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 \ No newline at end of file diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera index b77fbe6e864..a58a482c34e 100644 --- a/modulefiles/modulefile.ww3.hera +++ b/modulefiles/modulefile.ww3.hera @@ -3,40 +3,28 @@ ## S2S prerequisites ## -module load intel/18.0.5.274 -module load impi/2018.0.4 -module load wgrib2/2.0.8 -module load hpss/hpss -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load hdf5_parallel/1.10.6 -module load netcdf_parallel/4.7.4 -module load netcdf/4.7.0 -module load nco/4.7.0 - -module load gempak/7.4.2 -module load ncl/6.5.0 +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack -#Load from official NCEPLIBS -module load g2tmpl/1.6.0 -module load grib_util/1.1.1 -module load crtm/2.2.6 -module load prod_util/1.1.0 +module load hpc/1.1.0 +module load hpc-intel/18.0.5.274 +module load hpc-impi/2018.0.4 -module load bacio/2.0.3 -module load ip/3.0.2 -module load nemsio/2.2.4 -module load sp/2.0.3 -module load w3emc/2.3.1 -module load w3nco/2.0.7 -module load g2/3.1.1 -module load jasper/1.900.1 -module load png/1.2.44 -module load z/1.2.11 +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load esmf/8_1_0_beta_snapshot_36 -#Load from emc.nemspara -module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles -module load netcdf_parallel/4.7.4.release -module load esmf/8.1.0bs27 -module load hdf5_parallel/1.10.6.release - +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nceppost/dceca26 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion index 381c2886d1b..19c370de43e 100644 --- a/modulefiles/modulefile.ww3.orion +++ b/modulefiles/modulefile.ww3.orion @@ -4,28 +4,28 @@ module load contrib noaatools module load cmake/3.17.3 -module use /apps/contrib/NCEP/libs/modulefiles/stack +module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack -module load hpc/1.0.0 +module load hpc/1.1.0 module load hpc-intel/2018.4 module load hpc-impi/2018.4 -#module load jasper/2.0.15 -module load jasper/1.900.1 +module load jasper/2.0.25 module load zlib/1.2.11 module load png/1.6.35 module load hdf5/1.10.6 module load netcdf/4.7.4 -module load esmf/8_1_0_beta_snapshot_21 +module load esmf/8_1_0_beta_snapshot_27 -module load bacio/2.4.0 +module load bacio/2.4.1 module load crtm/2.3.0 -module load g2/3.4.0 -module load g2tmpl/1.9.0 -module load ip/3.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 module load nceppost/dceca26 -module load nemsio/2.5.1 -module load sp/2.3.0 -module load w3emc/2.7.0 -module load w3nco/2.4.0 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3 new file mode 100644 index 00000000000..16784b7d1d2 --- /dev/null +++ b/modulefiles/modulefile.ww3.wcoss_dell_p3 @@ -0,0 +1,26 @@ +#%Module###################################################################### + +module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack +module load hpc/1.1.0 +module load hpc-ips/18.0.1.163 +module load hpc-impi/18.0.1 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load esmf/8_1_0_beta_snapshot_36 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nceppost/dceca26 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load wgrib2/2.0.8 diff --git a/sorc/build_reg2grb2.sh b/sorc/build_reg2grb2.sh index 4817fefbf0e..a15e4edb42f 100755 --- a/sorc/build_reg2grb2.sh +++ b/sorc/build_reg2grb2.sh @@ -16,9 +16,9 @@ export CPP=fpp export INCS="-I${LANDSFCUTIL_INCd} \ -I${IP_INCd} \ - -I${NETCDF}/include" + -I${NETCDF_INCLUDES}" -export FFLAGSM="-O3 -free -convert big_endian -traceback -qopenmp -fp-model precise -assume byterecl ${INCS}" +export FFLAGSM="-O3 -free -convert big_endian -traceback -qopenmp -fp-model precise -assume byterecl ${INCS}" export FFLAGSM2="-O3 -free -r8" export LIBSM="${LANDSFCUTIL_LIBd} \ @@ -26,11 +26,15 @@ export LIBSM="${LANDSFCUTIL_LIBd} \ ${SP_LIBd} \ ${W3NCO_LIB4} \ ${BACIO_LIB4} \ - -L${NETCDF}/lib -lnetcdff -lnetcdf" - -WGRIB2DIR=$wgrib2_ROOT -export LIB2="-L$WGRIB2DIR/lib -lwgrib2_api -lwgrib2 -L$Jasper_ROOT/lib -ljasper -L$PNG_ROOT/lib -lpng" -export MOD2="-I$WGRIB2DIR/include" + -L${NETCDF_LIBRARIES} -lnetcdff -lnetcdf -L${HDF5_LIBRARIES} -lhdf5_hl -lhdf5 -L${ZLIB_LIBRARIES} -lz -ldl -lm" + +WGRIB2_ROOT=${WGRIB2_ROOT:-$wgrib2_ROOT} +WGRIB2_LIB=${WGRIB2_LIB:-${WGRIB_LIB:-"$WGRIB2_ROOT/lib/libwgrib2.a"}} +WGRIB2_LIBAPI=${WGRIB2_LIBAPI:-"$WGRIB2_ROOT/lib/libwgrib2_api.a"} +WGRIB2_INC=${WGRIB2_INC:-"$WGRIB2_ROOT/include"} +JASPER_LIB=${JASPER_LIB:-"$JASPER_LIBRARIES/libjasper.a"} +export LIB2="${WGRIB2_LIBAPI} ${WGRIB2_LIB} ${JASPER_LIB}" +export MOD2="-I${WGRIB2_INC}" cd reg2grb2.fd diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index b5f6d7bc970..134f94f8678 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -18,5 +18,5 @@ module purge module use $MOD_PATH module load fv3 cd ufs_coupled.fd/ +if [[ -d build ]]; then rm -Rf build; fi CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh - diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index f9e62c58450..80204f67411 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -26,13 +26,12 @@ export WW3_F90=gfortran export SWITCHFILE="${WW3_DIR}/esmf/switch" export WWATCH3_ENV=${WW3_BINDIR}/wwatch3.env -if [ $target = orion ]; then - export PNG_LIB=$PNG_ROOT/lib64/libpng.a - export Z_LIB=$ZLIB_ROOT/lib/libz.a - export JASPER_LIB=$JASPER_ROOT/lib/libjasper.a - export WWATCH3_NETCDF=NC4 - export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config -fi +export PNG_LIB=${PNG_LIB:-$PNG_ROOT/lib64/libpng.a} +export Z_LIB=${Z_LIB:-$ZLIB_ROOT/lib/libz.a} +export JASPER_LIB=${JASPER_LIB:-$JASPER_ROOT/lib64/libjasper.a} +export WWATCH3_NETCDF=NC4 +export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config + rm $WWATCH3_ENV echo '#' > $WWATCH3_ENV echo '# ---------------------------------------' >> $WWATCH3_ENV diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index 823e3687b53..c710b8405d5 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -44,7 +44,8 @@ pwd=$(pwd -P) if [ $machine == "cray" ]; then FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix_nco_gfsv16" elif [ $machine = "dell" ]; then - FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_nco_gfsv16" + # FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_nco_gfsv16" + FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/Walter.Kolczynski/global-workflow/fix_UFSp6" elif [ $machine = "hera" ]; then if [ $model = "coupled" ]; then FIX_DIR="/scratch2/NCEPDEV/climate/climpara/S2S/FIX/fix_UFSp6" diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml index fbb9cf625c1..79657de755b 100644 --- a/workflow/config/base.yaml +++ b/workflow/config/base.yaml @@ -68,6 +68,7 @@ config_base: export HOMEDIR="{doc.places.HOMEgfs}" export STMP="{doc.places.DATAROOT}" export PTMP="{doc.places.COMROOT}" + export COMROOT="{doc.places.COMROOT}" export NOSCRUB="{doc.places.NOSCRUB}" # Base directories for various builds diff --git a/workflow/platforms/wcoss_dell_p3.yaml b/workflow/platforms/wcoss_dell_p3.yaml index bf9f5204464..abf26f98481 100644 --- a/workflow/platforms/wcoss_dell_p3.yaml +++ b/workflow/platforms/wcoss_dell_p3.yaml @@ -34,9 +34,9 @@ platform: !Platform # DMPDIR - location of the global dump data. This is used by the observation # processing scripts, which are not included in the public release. DMPDIR: !FirstTrue - - do: "/gpfs/gp1/emc/globaldump" + - do: "/gpfs/dell3/emc/global/dump" when: !calc tools.isdir(do) - - do: "/gpfs/tp1/emc/globaldump" + - do: "/gpfs/hps3/emc/global/dump" when: !calc tools.isdir(do) - otherwise: !error "Cannot find globaldump directory." @@ -58,9 +58,9 @@ platform: !Platform BASE_GIT: "/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git" #BASE_CPLIC - is the base directory of ICs for coupled s2s runs - BASE_CPLIC: "/path/to/base/dir/of/IC" - #ncks - NCO netcdf operator used in ocean post - ncks: "/path/to/ncks" + BASE_CPLIC: "/gpfs/dell2/emc/modeling/noscrub/Walter.Kolczynski/global-workflow/IC" + #ncks - NCO netcdf operator used in ocean post + ncks: "/usrx/local/prod/packages/ips/18.0.1/nco/4.7.0/bin/ncks" # config_base_extras - Additional configuration data to put in the # config.base file