diff --git a/fix b/fix index 0be26971f8..6a42a29dbb 160000 --- a/fix +++ b/fix @@ -1 +1 @@ -Subproject commit 0be26971f834fe9b1d5b118e1e0ffed53facf671 +Subproject commit 6a42a29dbbc9fca3453cc9e829601185555890b9 diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua index 494ec6fb18..a4e7452e96 100644 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ b/modulefiles/gsi_cheyenne.gnu.lua @@ -9,7 +9,7 @@ load("mpt/2.22") load("ncarcompilers/0.5.0") unload("netcdf") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/GMTB/tools/gnu/10.1.0/hpc-stack-v1.2.0/modulefiles/stack") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/gnu10.1.0/modulefiles/stack") load("hpc/1.2.0") load("hpc-gnu/10.1.0") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index cafc1973be..d02a0c3885 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -17,8 +17,8 @@ load("mkl/2022.1") load("gsi_common") -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index b2929bcdf8..585572a049 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -7,7 +7,6 @@ local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" -local w3nco_ver=os.getenv("w3nco_ver") or "2.4.1" local sp_ver=os.getenv("sp_ver") or "2.3.3" local ip_ver=os.getenv("ip_ver") or "3.3.3" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" @@ -32,4 +31,7 @@ load(pathJoin("nemsio", nemsio_ver)) load(pathJoin("wrf_io", wrf_io_ver)) load(pathJoin("ncio", ncio_ver)) load(pathJoin("crtm", crtm_ver)) -load(pathJoin("ncdiag",ncdiag_ver)) +--load(pathJoin("ncdiag",ncdiag_ver)) + +setenv("ncdiag_ROOT", "/scratch1/NESDIS/nesdis-rdo2/David.Huber/GSI/ncdiag/1.1.1") +setenv("ncdiag_VERSION", "1.1.1") diff --git a/modulefiles/gsi_gaea b/modulefiles/gsi_gaea deleted file mode 100644 index 641f3d0fcf..0000000000 --- a/modulefiles/gsi_gaea +++ /dev/null @@ -1,62 +0,0 @@ -#%Module1.0 -###################################################################### -## NOAA-EMC/GSI -##_____________________________________________________ -proc ModulesHelp { } { -puts stderr "Set environment variables for NOAA-EMC/GSI" -puts stderr "This module initializes the environment " -puts stderr "for the Intel Compiler Suite $version\n" -} -module-whatis " NOAA-EMC/GSI whatis description" - -set COMPILER intel - -setenv FFLAGS_COM "-fp-model strict" -setenv LDFLAGS_COM " " - -#set WRF_SHARED_VER v1.1.0 -#set WRF_SHARED_ROOT /gpfs/hps/nco/ops/nwprod/wrf_shared -#set WRF_SHARED_ROOT /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/EXTERNAL/wrf_shared -#setenv WRF_SHARED_PATH ${WRF_SHARED_ROOT}.${WRF_SHARED_VER} - -setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - -# Loading ncep environment -##module load ncep/1.0 -module use /opt/cray/pe/craype/2.5.5/modulefiles - -# Loading Intel Compiler Suite -module load PrgEnv-intel - -# Loading pe environment -module load cray-mpich -module load cray-libsci -module unload craype-broadwell -module load craype-haswell - -module use /sw/gaea/modulefiles -module load cmake - -# Loading nceplibs modules -module use /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/modulefiles -#module load HDF5-serial-intel-haswell/1.8.9 -#module load NetCDF-intel-haswell/4.2 -module load cray-hdf5 -module load cray-netcdf - -#module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module load bufr-intel-sandybridge/11.0.1 -module load nemsio-intel-sandybridge/2.2.2 -module load sfcio-intel-sandybridge/1.0.0 -module load sigio-intel-sandybridge/2.0.1 -module load sp-intel-sandybridge/2.0.2 -module load w3nco-intel-sandybridge/2.0.6 -module load w3emc-intel-sandybridge/2.2.0 -module load bacio-intel-sandybridge/2.0.2 -setenv CRAYOS_VERSION $::env(CRAYPE_VERSION) -#setenv CRAYOS_VERSION ${CRAYPE_VERSION} - -# Compiler flags specific to this platform -setenv CFLAGS "-xCORE-AVX2" -setenv FFLAGS "-xCORE-AVX2" - diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua new file mode 100644 index 0000000000..dc008dcdbe --- /dev/null +++ b/modulefiles/gsi_gaea.lua @@ -0,0 +1,29 @@ +help([[ +]]) + +load("cmake/3.20.1") + +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) + +load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) +load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) +load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) + +load("gsi_common") + +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("prod_util", prod_util_ver)) + +local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix") +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") +pushenv("CRAYPE_LINK_TYPE","dynamic") + +whatis("Description: GSI environment on Gaea with Intel Compilers") diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 3ed9fbddb0..4f0253ba4d 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -20,6 +20,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("MKLROOT", "/apps/oneapi/mkl/2022.0.2") -pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601") whatis("Description: GSI environment on Hera with GNU Compilers") diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index 7c98d48ad9..866af02d50 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601") whatis("Description: GSI environment on Hera with Intel Compilers") diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index 2f8cacff8a..e2ea2ef1d0 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -26,6 +26,6 @@ pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") -pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601") whatis("Description: GSI environment on Jet with Intel Compilers") diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 1f0787be50..43276b7cbe 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230601") whatis("Description: GSI environment on Orion with Intel Compilers") diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index 52e942f458..efdc6c4bfb 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -23,6 +23,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-march=ivybridge") pushenv("FFLAGS", "-march=ivybridge") -pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20230601") whatis("Description: GSI environment on S4 with Intel Compilers") diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index 209af8c8a9..1872f89d17 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -20,6 +20,6 @@ load(pathJoin("prod_util", prod_util_ver)) load("gsi_common") -pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230601") whatis("Description: GSI environment on WCOSS2") diff --git a/src/gsi/gsimod.F90 b/src/gsi/gsimod.F90 index cf885c2b64..9360d9225f 100644 --- a/src/gsi/gsimod.F90 +++ b/src/gsi/gsimod.F90 @@ -2213,7 +2213,7 @@ subroutine gsimain_initialize endif ! Set up directories (or pe specific filenames) - call init_directories(mype) + call init_directories(mype,npe) ! Initialize space for qc call create_qcvars diff --git a/src/gsi/obsmod.F90 b/src/gsi/obsmod.F90 index b755bdf1cc..f32aa8ae68 100644 --- a/src/gsi/obsmod.F90 +++ b/src/gsi/obsmod.F90 @@ -981,7 +981,7 @@ subroutine init_obsmod_dflts return end subroutine init_obsmod_dflts - subroutine init_directories(in_pe) + subroutine init_directories(in_pe,num_pe) !$$$ subprogram documentation block ! . . . . ! subprogram: create sub-directories @@ -1012,24 +1012,34 @@ subroutine init_directories(in_pe) implicit none integer(i_kind),intent(in ) :: in_pe + integer(i_kind),intent(in ) :: num_pe logical :: l_mkdir_stat character(len=144):: command - character(len=8):: pe_name + character(len=8):: pe_name, loc_pe_name + character(len=128):: loc_dirname + integer(i_kind) :: i if (lrun_subdirs) then write(pe_name,'(i4.4)') in_pe dirname = 'dir.'//trim(pe_name)//'/' - command = 'mkdir -p -m 755 ' // trim(dirname) +! Only create directories on one PE + if(in_pe == 0) then + do i = 0, num_pe + write(loc_pe_name,'(i4.4)') i + loc_dirname = 'dir.'//trim(loc_pe_name) #ifdef __INTEL_COMPILER - l_mkdir_stat = MAKEDIRQQ(trim(dirname)) - if(.not. l_mkdir_stat) then - write(6, *) "Failed to create directory ", trim(dirname), " for PE ", pe_name - call stop2(678) - endif + l_mkdir_stat = MAKEDIRQQ(trim(loc_dirname)) + if(.not. l_mkdir_stat) then + write(6, *) "Failed to create directory ", trim(loc_dirname), " for PE ", loc_pe_name + call stop2(678) + endif #else - call system(command) + command = 'mkdir -p -m 755 ' // trim(loc_dirname) + call system(command) #endif + enddo + endif else write(pe_name,100) in_pe 100 format('pe',i4.4,'.') diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3c5..d32846f316 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -57,33 +57,10 @@ elif [[ $MACHINE_ID = gaea* ]] ; then # /etc/profile here. source /etc/profile __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge - # clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - if [[ "$__ms_source_etc_profile" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile fi + source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh + elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse if ( ! eval module help > /dev/null 2>&1 ) ; then