From cff1111b97a8f3042a0a672a5b06915ae71eda01 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 29 May 2025 14:09:24 -0400 Subject: [PATCH 01/16] Separate Gaea C5/6 module initialization procedures --- ush/module-setup.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 458b7195..60eec2f9 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -59,7 +59,7 @@ elif [[ $MACHINE_ID = stampede* ]] ; then fi module purge -elif [[ $MACHINE_ID = gaea* ]] ; then +elif [[ $MACHINE_ID = gaeac5 ]] ; then # We are on GAEA. if ( ! eval module help > /dev/null 2>&1 ) ; then # We cannot simply load the module command. The GAEA @@ -71,6 +71,13 @@ elif [[ $MACHINE_ID = gaea* ]] ; then fi module reset +elif [[ ${MACHINE_ID} = gaeac6 ]]; then + # We are on GAEA C6. + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /opt/cray/pe/lmod/lmod/init/bash + fi + module reset + elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse if ( ! eval module help > /dev/null 2>&1 ) ; then From f5ab738008a05927f04e0f7609ee868eba28fce4 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 29 May 2025 14:21:51 -0400 Subject: [PATCH 02/16] Remove support for S4 --- modulefiles/gsiutils_s4.intel.lua | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 modulefiles/gsiutils_s4.intel.lua diff --git a/modulefiles/gsiutils_s4.intel.lua b/modulefiles/gsiutils_s4.intel.lua deleted file mode 100644 index 22640cfe..00000000 --- a/modulefiles/gsiutils_s4.intel.lua +++ /dev/null @@ -1,24 +0,0 @@ -help([[ -]]) - -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") - -local python_ver=os.getenv("python_ver") or "3.11.6" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("python", python_ver)) -load(pathJoin("cmake", cmake_ver)) - -load("gsiutils_common") - -load(pathJoin("prod_util", prod_util_ver)) - -pushenv("CFLAGS", "-march=ivybridge") -pushenv("FFLAGS", "-march=ivybridge") - -whatis("Description: GSI utilities environment on S4 with Intel Compilers") From a0fa5a81f7efe72f69cb1f151757e9db7d12f980 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 29 May 2025 15:40:13 -0400 Subject: [PATCH 03/16] Update to spack-stack 1.9.1 --- CMakeLists.txt | 1 - modulefiles/gsiutils_common.lua | 12 +- modulefiles/gsiutils_gaeac6.intel.lua | 8 +- src/EFSOI_Utilities/src/CMakeLists.txt | 2 +- src/EnKF/gfs/src/adderrspec.fd/CMakeLists.txt | 2 +- .../adderrspec_nmcmeth_ncep_spec.f90 | 43 +++---- .../src/getsfcnstensupdp.fd/CMakeLists.txt | 2 +- .../getsfcnstensupdp.fd/getsfcnstensupdp.f90 | 3 +- .../getsigensmeanp_smooth.fd/CMakeLists.txt | 2 +- .../getsigensmeanp_smooth_ncep.f90 | 1 + .../gfs/src/getsigensstatp.fd/CMakeLists.txt | 2 +- .../src/getsigensstatp.fd/getsigensstatp.f90 | 9 +- src/EnKF/gfs/src/preproc/CMakeLists.txt | 2 +- src/NMC_Bkerror/sorc/CMakeLists.txt | 2 +- src/NMC_Bkerror/sorc/horizsc.f90 | 5 +- src/NMC_Bkerror/sorc/postmod.f90 | 1 + src/NMC_Bkerror/sorc/readpairs.f90 | 19 +-- src/NMC_Bkerror/sorc/specgrid.f90 | 21 ++-- src/NMC_Bkerror/sorc_aero/CMakeLists.txt | 2 +- src/NMC_Bkerror/sorc_aero/horizsc_aerosol.f90 | 13 ++- src/NMC_Bkerror/sorc_aero/specgrid.f90 | 9 +- src/Single_Observation/CMakeLists.txt | 3 +- src/bufr_tools/CMakeLists.txt | 2 +- src/netcdf_io/interp_inc.fd/CMakeLists.txt | 1 - src/netcdf_io/interp_inc.fd/driver.F90 | 108 +++++++++--------- src/radar_process/radialwind/CMakeLists.txt | 2 +- src/radar_process/reflectivity/CMakeLists.txt | 2 +- 27 files changed, 139 insertions(+), 140 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e6ec870..9bd3b312 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,7 +95,6 @@ find_package(sigio REQUIRED) find_package(sfcio REQUIRED) find_package(nemsio REQUIRED) find_package(ncio REQUIRED) -find_package(sp REQUIRED) find_package(ip REQUIRED) find_package(w3emc REQUIRED) find_package(bufr REQUIRED) diff --git a/modulefiles/gsiutils_common.lua b/modulefiles/gsiutils_common.lua index 8d798f4c..482fae78 100644 --- a/modulefiles/gsiutils_common.lua +++ b/modulefiles/gsiutils_common.lua @@ -5,13 +5,13 @@ Load common modules to build GSI utilities on all machines local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" -local bufr_ver=os.getenv("bufr_ver") or "11.7.0" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" +local bufr_ver=os.getenv("bufr_ver") or "12.1.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -local sp_ver=os.getenv("sp_ver") or "2.5.0" -local ip_ver=os.getenv("ip_ver") or "4.3.0" -local sigio_ver=os.getenv("sigio_ver") or "2.3.2" -local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +local ip_ver=os.getenv("ip_ver") or "5.1.0" +local sigio_ver=os.getenv("sigio_ver") or "2.3.3" +local sfcio_ver=os.getenv("sfcio_ver") or "1.4.2" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" @@ -21,10 +21,10 @@ local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2" load(pathJoin("netcdf-c", netcdf_c_ver)) load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("cmake", cmake_ver)) load(pathJoin("bufr", bufr_ver)) load(pathJoin("bacio", bacio_ver)) load(pathJoin("w3emc", w3emc_ver)) -load(pathJoin("sp", sp_ver)) load(pathJoin("ip", ip_ver)) load(pathJoin("sigio", sigio_ver)) load(pathJoin("sfcio", sfcio_ver)) diff --git a/modulefiles/gsiutils_gaeac6.intel.lua b/modulefiles/gsiutils_gaeac6.intel.lua index 7cf9a72f..f291205b 100644 --- a/modulefiles/gsiutils_gaeac6.intel.lua +++ b/modulefiles/gsiutils_gaeac6.intel.lua @@ -1,18 +1,16 @@ help([[ ]]) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.1/envs/ue-intel-2023.2.0/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.11.6" +local python_ver=os.getenv("python_ver") or "3.11.7" local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.30" local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) load(pathJoin("python", python_ver)) -load(pathJoin("cmake", cmake_ver)) load("gsiutils_common") diff --git a/src/EFSOI_Utilities/src/CMakeLists.txt b/src/EFSOI_Utilities/src/CMakeLists.txt index 016fd0f2..0f63d9ef 100644 --- a/src/EFSOI_Utilities/src/CMakeLists.txt +++ b/src/EFSOI_Utilities/src/CMakeLists.txt @@ -25,7 +25,7 @@ target_link_libraries(efsoi.x PRIVATE sfcio::sfcio) target_link_libraries(efsoi.x PRIVATE nemsio::nemsio) target_link_libraries(efsoi.x PRIVATE ncio::ncio) target_link_libraries(efsoi.x PRIVATE w3emc::w3emc_d) -target_link_libraries(efsoi.x PRIVATE sp::sp_d) +target_link_libraries(efsoi.x PRIVATE ip::ip_d) if(OpenMP_Fortran_FOUND) target_link_libraries(efsoi.x PRIVATE OpenMP::OpenMP_Fortran) endif() diff --git a/src/EnKF/gfs/src/adderrspec.fd/CMakeLists.txt b/src/EnKF/gfs/src/adderrspec.fd/CMakeLists.txt index d7a39cb6..0723df98 100644 --- a/src/EnKF/gfs/src/adderrspec.fd/CMakeLists.txt +++ b/src/EnKF/gfs/src/adderrspec.fd/CMakeLists.txt @@ -11,7 +11,7 @@ target_link_libraries(adderrspec.x PRIVATE bacio::bacio_4) target_link_libraries(adderrspec.x PRIVATE sigio::sigio) target_link_libraries(adderrspec.x PRIVATE nemsio::nemsio) target_link_libraries(adderrspec.x PRIVATE w3emc::w3emc_d) -target_link_libraries(adderrspec.x PRIVATE sp::sp_d) +target_link_libraries(adderrspec.x PRIVATE ip::ip_4) if(OpenMP_Fortran_FOUND) target_link_libraries(adderrspec.x PRIVATE OpenMP::OpenMP_Fortran) endif() diff --git a/src/EnKF/gfs/src/adderrspec.fd/adderrspec_nmcmeth_ncep_spec.f90 b/src/EnKF/gfs/src/adderrspec.fd/adderrspec_nmcmeth_ncep_spec.f90 index 62f06ffb..78b7631d 100644 --- a/src/EnKF/gfs/src/adderrspec.fd/adderrspec_nmcmeth_ncep_spec.f90 +++ b/src/EnKF/gfs/src/adderrspec.fd/adderrspec_nmcmeth_ncep_spec.f90 @@ -6,9 +6,9 @@ program adderrspec_nmcmeth ! prgmmr: whitaker org: esrl/psd date: 2009-02-23 ! ! abstract: Add samples of 48-24 forecast differences with a -! a specified amplitude and zero mean to analysis ensemle. -! Initial dates for forecasts are read in from dates.dat -! (this file must be created beforehand). Ensemble perts +! a specified amplitude and zero mean to analysis ensemle. +! Initial dates for forecasts are read in from dates.dat +! (this file must be created beforehand). Ensemble perts ! are smoothed, ens mean written out. ! ! program history log: @@ -34,6 +34,7 @@ program adderrspec_nmcmeth !$$$ use sigio_module + use sp_mod, only: sppad use nemsio_module, only: nemsio_init,nemsio_open,nemsio_close use nemsio_module, only: nemsio_gfile,nemsio_getfilehead,& nemsio_readrec,nemsio_writerec,nemsio_readrecv,nemsio_writerecv @@ -45,10 +46,10 @@ program adderrspec_nmcmeth ! Declare externals external :: mpi_init, mpi_comm_rank, mpi_comm_size, w3tagb, mpi_abort, mpi_comm_group, & mpi_group_incl, mpi_comm_create, mpi_barrier, copy_sigdata, mpi_allreduce, & - sppad, smooth, w3tage, mpi_finalize - + smooth, w3tage, mpi_finalize + logical :: meanonly,lexist,nemsio,sigio - + character(len=3) :: charnanal character(len=4) :: string character(len=10) :: datestring, datestringpert @@ -87,7 +88,7 @@ program adderrspec_nmcmeth call mpi_init(iret) call mpi_comm_rank(mpi_comm_world,mype,iret) call mpi_comm_size(mpi_comm_world,npe,iret) - + if (mype==0) call w3tagb('ADDERRSPEC_NMCMETH',2011,0319,0055,'NP25') ! Get command line arguments @@ -127,7 +128,7 @@ program adderrspec_nmcmeth write(6,*)' npert= ',npert,' meanonly ',meanonly write(6,*)' window= ',window endif - + if (npe < nanals) then write(6,*)'***ERROR** npe=',npe,' too small. nanals=',nanals flush(6) @@ -156,13 +157,13 @@ program adderrspec_nmcmeth write(6,*)'***ERROR*** after mpi_comm_create with iret=',iret call mpi_abort(mpi_comm_world,101,iret) endif - -! Generate random numbers to select perturbations. + +! Generate random numbers to select perturbations. ! Use analysis date as random seed rseed = 1.0e6_8*iadate(1) + 1.0e4_8*iadate(2) + 1.0e2_8*iadate(3) + iadate(4) iseed = rseed call random_setseed(iseed) - + ! Generate random number and map into range allocate(rwork(nanals),iwork(nanals)) call random_number(rwork) @@ -174,7 +175,7 @@ program adderrspec_nmcmeth ! Randomize dates (done on single task) if (mype == 0) then allocate(datepert(npert)) - + ! Read file with sequential pertubration dates. open(9,form='formatted',file='dates_seq.dat') do i=1,npert @@ -182,7 +183,7 @@ program adderrspec_nmcmeth end do close(9) write(6,*)'perturbation dates range from ',datepert(1),' to ',datepert(npert) - + ! Write file with randomized perturbation dates. write(6,*)'iseed=',iseed,' nanals=',nanals,' npert=',npert open(59,form='formatted',file='dates_ran.dat') @@ -314,7 +315,7 @@ program adderrspec_nmcmeth sigdataim%t = rnanals*sigdataim%t sigdataim%q = rnanals*sigdataim%q sigdataim%ps = rnanals*sigdataim%ps - + ! Write out ensemble mean from task 0 if (mype == 0) then sigheadim%iens(1) = 1 ! unperturbed control @@ -356,7 +357,7 @@ program adderrspec_nmcmeth deallocate(rwork1d) deallocate(swork1d) deallocate(twork1d) - + endif ! Jump here if more mpi processors than files to process @@ -379,10 +380,10 @@ program adderrspec_nmcmeth enddo close(9) filenamepert = trim(datapath)//'sigf48_f24.gfs.'//trim(datestringpert) - + call sigio_srohdc(iunitsf,trim(filenamepert),sigheadpin,sigdatapin,iret) write(6,*)'member=',trim(filenamein),' perturbation=',trim(filenamepert) - + if (iret /= 0) then write(6,*)'***ERROR*** problem opening ',trim(filenamepert),' iret=',iret flush(6) @@ -390,7 +391,7 @@ program adderrspec_nmcmeth call mpi_abort(mpi_comm_world,101,iret) stop end if - + write(6,*) 'compare resolution, jcapout, jcappert = ',sigheado%jcap,sigheadpin%jcap ! Change resolution of spectral perturbations if necessary @@ -469,14 +470,14 @@ program adderrspec_nmcmeth sigdatap%t = sigdatai%t - sigdataim%t + sigdatap%t sigdatap%q = sigdatai%q - sigdataim%q + sigdatap%q sigdatap%ps = sigdatai%ps - sigdataim%ps + sigdatap%ps - + ! Optionally smooth perturbations if (maxval(smoothparm) > 0) then if (mype == 0) write(6,*)'call smooth' call smooth(sigdatap%z,ntrunc,nlevs,smoothparm,window) call smooth(sigdatap%d,ntrunc,nlevs,smoothparm,window) call smooth(sigdatap%t,ntrunc,nlevs,smoothparm,window) - + ! Only smooth q field? this is what's done in getsigensmeanp_smooth.f90 ! call smooth(sigdatap%q,ntrunc,nlevs,smoothparm,window) do k=1,ntrac @@ -540,7 +541,7 @@ program adderrspec_nmcmeth write(91,*) datestring close(91) endif - + end program adderrspec_nmcmeth subroutine smooth(specdat,ntrunc,nlevs,smoothparm,window) diff --git a/src/EnKF/gfs/src/getsfcnstensupdp.fd/CMakeLists.txt b/src/EnKF/gfs/src/getsfcnstensupdp.fd/CMakeLists.txt index 0080d12a..8b650dba 100644 --- a/src/EnKF/gfs/src/getsfcnstensupdp.fd/CMakeLists.txt +++ b/src/EnKF/gfs/src/getsfcnstensupdp.fd/CMakeLists.txt @@ -20,7 +20,7 @@ target_link_libraries(getsfcnstensupdp.x PRIVATE bacio::bacio_4) target_link_libraries(getsfcnstensupdp.x PRIVATE w3emc::w3emc_d) target_link_libraries(getsfcnstensupdp.x PRIVATE nemsio::nemsio) target_link_libraries(getsfcnstensupdp.x PRIVATE sfcio::sfcio) -target_link_libraries(getsfcnstensupdp.x PRIVATE sp::sp_4) +target_link_libraries(getsfcnstensupdp.x PRIVATE ip::ip_d) # Install executable targets install(TARGETS getsfcnstensupdp.x RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/EnKF/gfs/src/getsfcnstensupdp.fd/getsfcnstensupdp.f90 b/src/EnKF/gfs/src/getsfcnstensupdp.fd/getsfcnstensupdp.f90 index d6bc700c..13083adf 100644 --- a/src/EnKF/gfs/src/getsfcnstensupdp.fd/getsfcnstensupdp.f90 +++ b/src/EnKF/gfs/src/getsfcnstensupdp.fd/getsfcnstensupdp.f90 @@ -24,6 +24,7 @@ program getsfcnstensupdp !$$$ use mpi + use sp_mod, only: splat use kinds, only: r_kind,i_kind,r_single use constants, only: two,half,zero,z_w_max,tfrozen,init_constants_derived,pi use sfcio_module, only: sfcio_srohdc,sfcio_head,sfcio_data,sfcio_swohdc @@ -44,7 +45,7 @@ program getsfcnstensupdp integer(i_kind), parameter :: idrt=4 ! Declare externals - external :: w3tagb, splat, int2_msk_glb_prep, int22_msk_glb, dtzm_2d, w3tage + external :: w3tagb, int2_msk_glb_prep, int22_msk_glb, dtzm_2d, w3tage character(len=80) :: fname_dtfanl,fname_nstges,fname_sfcgcy,fname_nstanl,fname_sfcanl character(len=3) :: charnanal diff --git a/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/CMakeLists.txt b/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/CMakeLists.txt index d0d24065..bc7e76b3 100644 --- a/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/CMakeLists.txt +++ b/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries(getsigensmeanp_smooth.x PRIVATE MPI::MPI_Fortran) target_link_libraries(getsigensmeanp_smooth.x PRIVATE bacio::bacio_4) target_link_libraries(getsigensmeanp_smooth.x PRIVATE w3emc::w3emc_d) target_link_libraries(getsigensmeanp_smooth.x PRIVATE nemsio::nemsio) -target_link_libraries(getsigensmeanp_smooth.x PRIVATE sp::sp_4) +target_link_libraries(getsigensmeanp_smooth.x PRIVATE ip::ip_4) target_link_libraries(getsigensmeanp_smooth.x PRIVATE sigio::sigio) target_link_libraries(getsigensmeanp_smooth.x PRIVATE ncio::ncio) diff --git a/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/getsigensmeanp_smooth_ncep.f90 b/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/getsigensmeanp_smooth_ncep.f90 index db28e2d7..e697221d 100644 --- a/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/getsigensmeanp_smooth_ncep.f90 +++ b/src/EnKF/gfs/src/getsigensmeanp_smooth.fd/getsigensmeanp_smooth_ncep.f90 @@ -21,6 +21,7 @@ program getsigensmeanp_smooth !$$$ use netcdf + use sp_mod, only: sptezv, sptez, splat use sigio_module, only: sigio_head,sigio_data,sigio_srohdc, & sigio_swohdc,sigio_aldata,sigio_axdata use nemsio_module, only: nemsio_init,nemsio_open,nemsio_close diff --git a/src/EnKF/gfs/src/getsigensstatp.fd/CMakeLists.txt b/src/EnKF/gfs/src/getsigensstatp.fd/CMakeLists.txt index 8dd1b976..a8106ca5 100644 --- a/src/EnKF/gfs/src/getsigensstatp.fd/CMakeLists.txt +++ b/src/EnKF/gfs/src/getsigensstatp.fd/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries(getsigensstatp.x PRIVATE MPI::MPI_Fortran) target_link_libraries(getsigensstatp.x PRIVATE bacio::bacio_4) target_link_libraries(getsigensstatp.x PRIVATE w3emc::w3emc_d) target_link_libraries(getsigensstatp.x PRIVATE nemsio::nemsio) -target_link_libraries(getsigensstatp.x PRIVATE sp::sp_4) +target_link_libraries(getsigensstatp.x PRIVATE ip::ip_4) target_link_libraries(getsigensstatp.x PRIVATE sigio::sigio) target_link_libraries(getsigensstatp.x PRIVATE ncio::ncio) diff --git a/src/EnKF/gfs/src/getsigensstatp.fd/getsigensstatp.f90 b/src/EnKF/gfs/src/getsigensstatp.fd/getsigensstatp.f90 index efe496f5..53f8da7b 100644 --- a/src/EnKF/gfs/src/getsigensstatp.fd/getsigensstatp.f90 +++ b/src/EnKF/gfs/src/getsigensstatp.fd/getsigensstatp.f90 @@ -15,7 +15,7 @@ program getsigensstatp ! ! program history log: ! 2014-08-23 Initial version. -! 2018-07-21 Add hydrometeor (optional) +! 2018-07-21 Add hydrometeor (optional) ! ! usage: ! input files: @@ -29,6 +29,7 @@ program getsigensstatp use netcdf use mpi + use sp_mod, only: sptezv, sptez, splat use sigio_module, only: sigio_head,sigio_data,sigio_srohdc, & sigio_axdata,sigio_sclose use nemsio_module, only: nemsio_init,nemsio_open,nemsio_close, & @@ -46,7 +47,7 @@ program getsigensstatp integer,parameter :: iunit=21 integer,parameter :: idrt=4 ! Declare externals - external :: w3tagb, splat, sptez, sptezv, w3tage + external :: w3tagb, w3tage character(nemsio_charkind8) :: dtype character(len=3) :: charnanal character(len=500) :: filenamein,datapath,filepref @@ -276,7 +277,7 @@ program getsigensstatp call nemsio_readrecv(gfile,'spfh', 'mid layer',k,rwork_mem(:,krecq), iret=iret) call nemsio_readrecv(gfile,'o3mr', 'mid layer',k,rwork_mem(:,krecoz), iret=iret) call nemsio_readrecv(gfile,'clwmr','mid layer',k,rwork_mem(:,kreccwmr),iret=iret) - ! if ( do_icmr ) call nemsio_readrecv(gfile,'icmr', 'mid layer',k,rwork_mem(:,krecicmr),iret=iret) + ! if ( do_icmr ) call nemsio_readrecv(gfile,'icmr', 'mid layer',k,rwork_mem(:,krecicmr),iret=iret) if ( do_hydro ) then call nemsio_readrecv(gfile,'icmr', 'mid layer',k,rwork_mem(:,krecicmr), iret=iret) call nemsio_readrecv(gfile,'rwmr', 'mid layer',k,rwork_mem(:,krecrwmr), iret=iret) @@ -288,7 +289,7 @@ program getsigensstatp call nemsio_close(gfile,iret=iret) elseif ( ncio ) then - call read_vardata(dset,'pressfc',values_2d) + call read_vardata(dset,'pressfc',values_2d) rwork_mem(:,1) = reshape(values_2d,(/npts/)) deallocate(values_2d) call read_vardata(dset,'ugrd',values_3d) diff --git a/src/EnKF/gfs/src/preproc/CMakeLists.txt b/src/EnKF/gfs/src/preproc/CMakeLists.txt index 65955d93..9a9576fb 100644 --- a/src/EnKF/gfs/src/preproc/CMakeLists.txt +++ b/src/EnKF/gfs/src/preproc/CMakeLists.txt @@ -14,7 +14,7 @@ list(APPEND _targets enspreproc.x) foreach(_tgt ${_targets}) - target_link_libraries(${_tgt} PRIVATE sp::sp_4) + target_link_libraries(${_tgt} PRIVATE ip::ip_4) target_link_libraries(${_tgt} PRIVATE sigio::sigio) if(OpenMP_Fortran_FOUND) target_link_libraries(${_tgt} PRIVATE OpenMP::OpenMP_Fortran) diff --git a/src/NMC_Bkerror/sorc/CMakeLists.txt b/src/NMC_Bkerror/sorc/CMakeLists.txt index 3714a76f..77a8d1d9 100644 --- a/src/NMC_Bkerror/sorc/CMakeLists.txt +++ b/src/NMC_Bkerror/sorc/CMakeLists.txt @@ -37,7 +37,7 @@ target_link_libraries(calcstats.x PRIVATE MPI::MPI_Fortran) target_link_libraries(calcstats.x PRIVATE ${LAPACK_LIBRARIES}) target_link_libraries(calcstats.x PRIVATE bacio::bacio_4) target_link_libraries(calcstats.x PRIVATE w3emc::w3emc_d) -target_link_libraries(calcstats.x PRIVATE sp::sp_4) +target_link_libraries(calcstats.x PRIVATE ip::ip_4) target_link_libraries(calcstats.x PRIVATE sigio::sigio) target_link_libraries(calcstats.x PRIVATE nemsio::nemsio) target_link_libraries(calcstats.x PRIVATE ncio::ncio) diff --git a/src/NMC_Bkerror/sorc/horizsc.f90 b/src/NMC_Bkerror/sorc/horizsc.f90 index 10d65d61..950b2fff 100644 --- a/src/NMC_Bkerror/sorc/horizsc.f90 +++ b/src/NMC_Bkerror/sorc/horizsc.f90 @@ -1,5 +1,6 @@ subroutine horizsc(numcases,mype) use kinds, only: r_kind,i_kind + use sp_mod, only: splaplac, splat use postmod, only: smoothlat use variables,only: nlat,nlon,nsig,lat1,lon1,zero,& displs_g,ijn,db_prec,filunit1,filunit2,npe,& @@ -143,7 +144,7 @@ subroutine horizsc(numcases,mype) do i=1,lat1 ps3(i,j) = ps3(i,j) + ps1(i,j)*ps1(i,j)*r_norm end do - end do + end do end do ! end do over numcases close(filunit1) close(filunit2) @@ -305,5 +306,5 @@ subroutine horizsc(numcases,mype) call mpi_bcast(chln,nlat*nsig,mpi_rtype,mype_work,mpi_comm_world,ierror) call mpi_bcast(pshln,nlat,mpi_rtype,mype_work,mpi_comm_world,ierror) - return + return end subroutine horizsc diff --git a/src/NMC_Bkerror/sorc/postmod.f90 b/src/NMC_Bkerror/sorc/postmod.f90 index ced6277e..9ade3818 100644 --- a/src/NMC_Bkerror/sorc/postmod.f90 +++ b/src/NMC_Bkerror/sorc/postmod.f90 @@ -1,6 +1,7 @@ module postmod use kinds, only: r_kind,r_single + use sp_mod, only: splat implicit none diff --git a/src/NMC_Bkerror/sorc/readpairs.f90 b/src/NMC_Bkerror/sorc/readpairs.f90 index 5330032a..0d835cee 100644 --- a/src/NMC_Bkerror/sorc/readpairs.f90 +++ b/src/NMC_Bkerror/sorc/readpairs.f90 @@ -21,6 +21,7 @@ subroutine readpairs(npe,mype,numcases) use module_ncio, only: Dataset, Variable, Dimension, open_dataset,& get_idate_from_time_units,& read_attribute, close_dataset, get_dim, read_vardata + use sp_mod, only: splaplac implicit none include 'mpif.h' @@ -74,7 +75,7 @@ subroutine readpairs(npe,mype,numcases) z4all = 0.0 z4all2 = 0.0 - + if (use_gfs_nemsio .or. use_gfs_ncio) then nfields = 1+5*nsig !ps, (u,v), t, q, oz, cw allocate(taskid(nfields)) @@ -201,7 +202,7 @@ subroutine readpairs(npe,mype,numcases) call sptezv_sin(z4all2(:,nsig+k),z4all2(:,k),grid1in,grid2in,-1) end if end do - + ! need to improve in the future ! broadcast the data on various processors to all processors icount = 0 @@ -262,10 +263,10 @@ subroutine readpairs(npe,mype,numcases) z42,spec_send(mm1),mpi_rtype,0,mpi_comm_world,ierror) call mpi_barrier(mpi_comm_world,iret2) - + call nemsio_close(gfile1,iret=iret) call nemsio_close(gfile2,iret=iret) - + else if (use_gfs_ncio) then if (mype==0) write(6,*)'reading from', trim(filename(na(n))) dset1 = open_dataset(trim(adjustl(filename(na(n)))),errcode=iret) @@ -343,7 +344,7 @@ subroutine readpairs(npe,mype,numcases) endif grid1in = values_3d_1(:,:,1) grid2in = values_3d_2(:,:,1) - if ( iret == 0 ) then + if ( iret == 0 ) then ! if icmr exists, add ice to cloud water to get total condensate grid1in = grid1in + values_3d_3(:,:,1) grid2in = grid2in + values_3d_4(:,:,1) @@ -369,7 +370,7 @@ subroutine readpairs(npe,mype,numcases) end if end do call mpi_barrier(mpi_comm_world,ierror) - + ! need to improve in the future ! broadcast the data on various processors to all processors icount = 0 @@ -430,14 +431,14 @@ subroutine readpairs(npe,mype,numcases) z42,spec_send(mm1),mpi_rtype,0,mpi_comm_world,ierror) call mpi_barrier(mpi_comm_world,iret2) - + call close_dataset(dset1) call close_dataset(dset2) else !if not use_gfs_nemsio and not use_gfs_ncio if (mype==0) write(6,*)'opening=', inges,filename(na(n)) if (mype==0) write(6,*)'opening=', inge2,filename(nb(n)) - ! Get spectral information from + ! Get spectral information from if (mype==proc1) call sigio_srohdc(inges,filename(na(n)),sighead1,sigdata1,iret) if (mype==proc2) call sigio_srohdc(inge2,filename(nb(n)),sighead1,sigdata1,iret) call mpi_barrier(mpi_comm_world,iret2) @@ -479,7 +480,7 @@ subroutine readpairs(npe,mype,numcases) if (allocated(values_3d_4)) deallocate(values_3d_4) endif - work1=zero ; work2=zero + work1=zero ; work2=zero do k=1,nsig1o ! Check: Streamfunction level? diff --git a/src/NMC_Bkerror/sorc/specgrid.f90 b/src/NMC_Bkerror/sorc/specgrid.f90 index 727c5175..85ceaf77 100644 --- a/src/NMC_Bkerror/sorc/specgrid.f90 +++ b/src/NMC_Bkerror/sorc/specgrid.f90 @@ -1,5 +1,6 @@ module specgrid use kinds, only: r_kind,r_double,i_kind + use sp_mod, only: sptranf0, sptranf1, spdz2uv, spuv2dz implicit none integer jcap,jcapin,jcapsmooth,nc,ncin,ncd2,ncd2in @@ -20,7 +21,7 @@ subroutine init_spec_vars(nlat,nlon,nsig) implicit none integer,intent(in):: nlat,nlon,nsig - integer ii,ii1,l,m,ncpus + integer ii,ii1,l,m real(r_kind) zero1 ! Set constants @@ -54,7 +55,6 @@ subroutine init_spec_vars(nlat,nlon,nsig) kw=2*ncd2 jb=1 je=(jmax+1)/2 - jc=ncpus() ! Allocate arrays allocate( eps(ncd2) ) @@ -149,7 +149,7 @@ subroutine sptranf_s(wave,gridn,grids,idir) ! Initialize local variables mp=0 - + do i=1,2*(jcap+1) wtop(i)=0. end do @@ -476,7 +476,7 @@ subroutine sptranf_v(waved,wavez,gridun,gridus,gridvn,gridvs,idir) ! spdz2uv compute winds from divergence and vorticity ! spuv2dz compute divergence and vorticity from winds ! -! remarks: +! remarks: ! This routine assumes that splib routine sptranf0 has been ! previously called. sptranf0 initializes arrays needed in ! the transforms. @@ -503,7 +503,7 @@ subroutine sptranf_v(waved,wavez,gridun,gridus,gridvn,gridvs,idir) ! Declare local variables integer(i_kind) i,j,jj,ijn,ijs - integer(i_kind),dimension(2):: mp + integer(i_kind) :: mp real(r_kind),dimension(ncd2*2,2):: w real(r_kind),dimension(2*(jcap+1),2):: wtop real(r_kind),dimension(imax,2,2):: g @@ -580,7 +580,7 @@ subroutine init_spec_varsin(nlat,nlon,nsig) implicit none integer,intent(in):: nlat,nlon,nsig - integer ii,ii1,l,m,ncpus + integer ii,ii1,l,m real(r_kind) zero1 ! Set constants @@ -615,7 +615,6 @@ subroutine init_spec_varsin(nlat,nlon,nsig) kwin=2*ncd2in jbin=1 jein=(jmaxin+1)/2 -! jc=ncpus() ! Allocate arrays allocate( epsin(ncd2in) ) @@ -710,7 +709,7 @@ subroutine sptranf_sin(wave,gridn,grids,idir) ! Initialize local variables mp=0 - + do i=1,2*(jcapin+1) wtop(i)=0. end do @@ -907,7 +906,7 @@ subroutine sptranf_vin(waved,wavez,gridun,gridus,gridvn,gridvs,idir) ! spdz2uv compute winds from divergence and vorticity ! spuv2dz compute divergence and vorticity from winds ! -! remarks: +! remarks: ! This routine assumes that splib routine sptranf0 has been ! previously called. sptranf0 initializes arrays needed in ! the transforms. @@ -934,7 +933,7 @@ subroutine sptranf_vin(waved,wavez,gridun,gridus,gridvn,gridvs,idir) ! Declare local variables integer(i_kind) i,j,jj,ijn,ijs - integer(i_kind),dimension(2):: mp + integer(i_kind) :: mp real(r_kind),dimension(ncd2in*2,2):: w real(r_kind),dimension(2*(jcapin+1),2):: wtop real(r_kind),dimension(imaxin,2,2):: g @@ -1007,6 +1006,6 @@ subroutine sptranf_vin(waved,wavez,gridun,gridus,gridvn,gridvs,idir) endif end subroutine sptranf_vin - + end module specgrid diff --git a/src/NMC_Bkerror/sorc_aero/CMakeLists.txt b/src/NMC_Bkerror/sorc_aero/CMakeLists.txt index ae48aa40..97612281 100644 --- a/src/NMC_Bkerror/sorc_aero/CMakeLists.txt +++ b/src/NMC_Bkerror/sorc_aero/CMakeLists.txt @@ -29,7 +29,7 @@ target_link_libraries(calcstats_aero.x PRIVATE MPI::MPI_Fortran) target_link_libraries(calcstats_aero.x PRIVATE ${LAPACK_LIBRARIES}) target_link_libraries(calcstats_aero.x PRIVATE bacio::bacio_4) target_link_libraries(calcstats_aero.x PRIVATE w3emc::w3emc_d) -target_link_libraries(calcstats_aero.x PRIVATE sp::sp_4) +target_link_libraries(calcstats_aero.x PRIVATE ip::ip_4) target_link_libraries(calcstats_aero.x PRIVATE nemsio::nemsio) install(TARGETS calcstats_aero.x RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/NMC_Bkerror/sorc_aero/horizsc_aerosol.f90 b/src/NMC_Bkerror/sorc_aero/horizsc_aerosol.f90 index 1d7c6276..c1446fa7 100644 --- a/src/NMC_Bkerror/sorc_aero/horizsc_aerosol.f90 +++ b/src/NMC_Bkerror/sorc_aero/horizsc_aerosol.f90 @@ -1,6 +1,7 @@ subroutine horizsc_aerosol(numcases,mype) use kinds, only: r_kind,i_kind use postmod, only: smoothlat + use sp_mod, only: splaplac use variables,only: nlat,nlon,nsig,lat1,lon1,zero,& displs_g,ijn,db_prec,filunit1,filunit2,npe,& d1hln,d2hln,d3hln,d4hln,d5hln,s1hln,s2hln,s3hln,s4hln, & @@ -19,7 +20,7 @@ subroutine horizsc_aerosol(numcases,mype) real(r_kind),dimension(lat1,lon1,nsig):: d1a,d2a,d3a,d4a,d5a, & s1a,s2a,s3a,s4a,so4a,oc1a,oc2a,bc1a,bc2a real(r_kind),dimension(lat1,lon1,nsig):: d1b,d2b,d3b,d4b,d5b, & - s1b,s2b,s3b,s4b,so4b,oc1b,oc2b,bc1b,bc2b + s1b,s2b,s3b,s4b,so4b,oc1b,oc2b,bc1b,bc2b real(r_kind),dimension(lat1,lon1,nsig):: d1c,d2c,d3c,d4c,d5c, & s1c,s2c,s3c,s4c,so4c,oc1c,oc2c,bc1c,bc2c @@ -49,12 +50,12 @@ subroutine horizsc_aerosol(numcases,mype) eight=8.0_r_kind quarter=0.25_r_kind - d1c=zero ; d2c=zero ; d3c=zero ; d4c=zero ; d5c=zero ; - s1c=zero ; s2c=zero ; s3c=zero ; s4c=zero ; so4c=zero ; + d1c=zero ; d2c=zero ; d3c=zero ; d4c=zero ; d5c=zero ; + s1c=zero ; s2c=zero ; s3c=zero ; s4c=zero ; so4c=zero ; oc1c=zero ; oc2c=zero ; bc1c=zero ; bc2c=zero - d1lap=zero ; d2lap=zero ; d3lap=zero ; d4lap=zero ; d5lap=zero ; - s1lap=zero ; s2lap=zero ; s3lap=zero ; s4lap=zero ; so4lap=zero ; + d1lap=zero ; d2lap=zero ; d3lap=zero ; d4lap=zero ; d5lap=zero ; + s1lap=zero ; s2lap=zero ; s3lap=zero ; s4lap=zero ; so4lap=zero ; oc1lap=zero ; oc2lap=zero ; bc1lap=zero ; bc2lap=zero open(filunit1,form='unformatted',action='read') @@ -471,5 +472,5 @@ subroutine horizsc_aerosol(numcases,mype) call mpi_bcast(bc1hln,nlat*nsig,mpi_rtype,mype_work,mpi_comm_world,ierror) call mpi_bcast(bc2hln,nlat*nsig,mpi_rtype,mype_work,mpi_comm_world,ierror) - return + return end subroutine horizsc_aerosol diff --git a/src/NMC_Bkerror/sorc_aero/specgrid.f90 b/src/NMC_Bkerror/sorc_aero/specgrid.f90 index e0cc5748..b8f61da5 100644 --- a/src/NMC_Bkerror/sorc_aero/specgrid.f90 +++ b/src/NMC_Bkerror/sorc_aero/specgrid.f90 @@ -1,6 +1,8 @@ module specgrid use kinds, only: r_kind,r_double use variables, only: use_nemsio + use sp_mod, only: sptranf1, sptranf0, spuv2dz, spdz2uv + use mpi implicit none integer jcap,jcapin,jcapsmooth,nc,ncin,ncd2 @@ -47,7 +49,7 @@ subroutine init_spec_vars(nlat,nlon,nsig) ! Set other constants used in transforms !>swei - iromb=0 + iromb=0 !