From ec074b77f674d4434c13c3506a973a19e4bda277 Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Mon, 3 Aug 2020 17:17:49 +0000 Subject: [PATCH 01/73] Add CMEPS (mediator) to hafs_forecast submodule component and build. --- .gitmodules | 2 +- sorc/build_forecast.sh | 2 +- sorc/hafs_forecast.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index db8148ef5..22f206484 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-hafs-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-hafs-model.git - branch = feature/hafs_couplehycom + branch = feature/hafs_couplehycom_cmeps [submodule "EMC_post"] path = sorc/hafs_post.fd url = https://github.com/hafs-community/EMC_post.git diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 86df1b2b0..16e67e215 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -24,7 +24,7 @@ if [ $target = wcoss_cray ]; then module load python/2.7.14; fi cd hafs_forecast.fd/ FV3=$( pwd -P )/FV3 cd tests/ -./compile.sh "$FV3" "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y" 32bit YES NO +./compile.sh "$FV3" "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y" 32bit YES NO #./compile.sh "$FV3" "$target" "CCPP=Y REPRO=Y 32BIT=Y HYCOM=Y" 32bit YES NO #./compile.sh "$FV3" "$target" "CCPP=Y 32BIT=Y HYCOM=Y" 32bit YES NO #cp -p fv3_32bit.exe ../NEMS/exe/ diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index c1682a47d..c2f6fe311 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit c1682a47dc8d7f0fd03378cec2437c5fe7135c14 +Subproject commit c2f6fe311f0d1ccd71e39006802174d30c321c78 From e8aa540cb627ac8598b1b214817e966767f287a0 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sun, 9 Aug 2020 19:58:06 +0000 Subject: [PATCH 02/73] Script changes in order to support run a CMEPS build for hafs-hycom coupling, even when CMEPS is disabled, according to the instructions from Dan R. --- parm/forecast/regional/nems.configure.atm_ocn.tmp | 9 +++++++++ .../regional_hafsv0p1a/nems.configure.atm_ocn.tmp | 9 +++++++++ parm/forecast/regional_hwrf/nems.configure.atm_ocn.tmp | 9 +++++++++ scripts/exhafs_forecast.sh | 3 +++ 4 files changed, 30 insertions(+) diff --git a/parm/forecast/regional/nems.configure.atm_ocn.tmp b/parm/forecast/regional/nems.configure.atm_ocn.tmp index a2ec3fb4f..83bdac3bf 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn.tmp @@ -49,3 +49,12 @@ OCN_attributes:: end_min = 0 end_sec = 0 :: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: + diff --git a/parm/forecast/regional_hafsv0p1a/nems.configure.atm_ocn.tmp b/parm/forecast/regional_hafsv0p1a/nems.configure.atm_ocn.tmp index cf5070020..e4f4eef7c 100644 --- a/parm/forecast/regional_hafsv0p1a/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional_hafsv0p1a/nems.configure.atm_ocn.tmp @@ -49,3 +49,12 @@ OCN_attributes:: end_min = 0 end_sec = 0 :: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: + diff --git a/parm/forecast/regional_hwrf/nems.configure.atm_ocn.tmp b/parm/forecast/regional_hwrf/nems.configure.atm_ocn.tmp index a2ec3fb4f..83bdac3bf 100644 --- a/parm/forecast/regional_hwrf/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional_hwrf/nems.configure.atm_ocn.tmp @@ -49,3 +49,12 @@ OCN_attributes:: end_min = 0 end_sec = 0 :: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: + diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index eba5e7d04..b907d3bc3 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -358,6 +358,9 @@ cp ${PARMhycom}/hafs_${hycom_basin}.basin.fcst.blkdat.input blkdat.input cp ${PARMhycom}/hafs_${hycom_basin}.basin.ports.input ports.input cp ${PARMhycom}/hafs_${hycom_basin}.basin.patch.input.${ocean_tasks} patch.input +# copy fd_nems.yaml +cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS/mediator/fd_nems.yaml ./ + # create hycom limits ${USHhafs}/hafs_hycom_limits.py ${yr}${mn}${dy}${cyc} From 1c08214c9cda62640cbb4ac22f59ff967e99e794 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 11 Aug 2020 14:18:31 +0000 Subject: [PATCH 03/73] Enable atmospheric surface pressure forcing in hafs-hycom coupling (Dan R. and Hyun-Sook) --- parm/hycom/regional/hafs_hycom_hat10.basin.fcst.blkdat.input | 2 +- parm/hycom/regional/hafs_hycom_hcp70.basin.fcst.blkdat.input | 2 +- parm/hycom/regional/hafs_hycom_hep20.basin.fcst.blkdat.input | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/hycom/regional/hafs_hycom_hat10.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_hat10.basin.fcst.blkdat.input index b0294bcec..6b4c8b4c0 100644 --- a/parm/hycom/regional/hafs_hycom_hat10.basin.fcst.blkdat.input +++ b/parm/hycom/regional/hafs_hycom_hat10.basin.fcst.blkdat.input @@ -256,7 +256,7 @@ sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) 1E+5 'prsbas' = msl pressure is input field + prsbas (Pa) - 0 'mslprf' = msl pressure forcing flag (0=F,1=T) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) 0 'stroff' = net stress offset flag (0=F,1=T) 0 'flxoff' = net flux offset flag (0=F,1=T) 0 'flxsmo' = smooth surface fluxes (0=F,1=T) diff --git a/parm/hycom/regional/hafs_hycom_hcp70.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_hcp70.basin.fcst.blkdat.input index 8582dea2d..e68770f2a 100644 --- a/parm/hycom/regional/hafs_hycom_hcp70.basin.fcst.blkdat.input +++ b/parm/hycom/regional/hafs_hycom_hcp70.basin.fcst.blkdat.input @@ -256,7 +256,7 @@ sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) 1E+5 'prsbas' = msl pressure is input field + prsbas (Pa) - 0 'mslprf' = msl pressure forcing flag (0=F,1=T) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) 0 'stroff' = net stress offset flag (0=F,1=T) 0 'flxoff' = net flux offset flag (0=F,1=T) 0 'flxsmo' = smooth surface fluxes (0=F,1=T) diff --git a/parm/hycom/regional/hafs_hycom_hep20.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_hep20.basin.fcst.blkdat.input index 2d518b4a6..ae2bc2d63 100644 --- a/parm/hycom/regional/hafs_hycom_hep20.basin.fcst.blkdat.input +++ b/parm/hycom/regional/hafs_hycom_hep20.basin.fcst.blkdat.input @@ -256,7 +256,7 @@ sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) 1E+5 'prsbas' = msl pressure is input field + prsbas (Pa) - 0 'mslprf' = msl pressure forcing flag (0=F,1=T) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) 0 'stroff' = net stress offset flag (0=F,1=T) 0 'flxoff' = net flux offset flag (0=F,1=T) 0 'flxsmo' = smooth surface fluxes (0=F,1=T) From be431350264e686611b75dccb1f715585661907b Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Tue, 27 Oct 2020 20:36:23 +0000 Subject: [PATCH 04/73] Update CMEPS submodule. Includes ocean fraction computation fix. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index c2f6fe311..f9047d16a 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit c2f6fe311f0d1ccd71e39006802174d30c321c78 +Subproject commit f9047d16a42b99486d96d3d3c3e2637234bb7d1e From 768dfde9a61efc05c16cee00a25eaea7940bf6fa Mon Sep 17 00:00:00 2001 From: Evan Kalina Date: Thu, 19 Nov 2020 12:41:02 -0600 Subject: [PATCH 05/73] Add support for running a CDEPS data atmosphere (datm) to the HAFS workflow. --- parm/cdeps/datm.streams.xml | 47 +++++++ parm/cdeps/datm_in | 15 +++ parm/cdeps/modelio.nml | 8 ++ parm/cdeps/pio_in | 34 +++++ .../regional/nems.configure.datm_ocn.tmp | 121 ++++++++++++++++++ parm/hafs.conf | 8 ++ parm/hafs_basic.conf | 3 + parm/hafs_cdeps.conf | 20 +++ parm/hafs_holdvars.conf | 4 + parm/hafs_holdvars.txt | 9 ++ rocoto/hafs_workflow.xml.in | 41 ++++-- rocoto/sites/orion.ent | 3 + scripts/exhafs_forecast.sh | 58 ++++++++- ush/hafs/launcher.py | 5 + 14 files changed, 363 insertions(+), 13 deletions(-) create mode 100755 parm/cdeps/datm.streams.xml create mode 100755 parm/cdeps/datm_in create mode 100755 parm/cdeps/modelio.nml create mode 100755 parm/cdeps/pio_in create mode 100755 parm/forecast/regional/nems.configure.datm_ocn.tmp create mode 100644 parm/hafs_cdeps.conf diff --git a/parm/cdeps/datm.streams.xml b/parm/cdeps/datm.streams.xml new file mode 100755 index 000000000..e15ae1bbb --- /dev/null +++ b/parm/cdeps/datm.streams.xml @@ -0,0 +1,47 @@ + + + + + cycle + linear + single + bilinear + 1.5 + 2019 + 2019 + 1900 + u:v + _mesh_atm_ + null + + + + u10 Sa_u + v10 Sa_v + t2m Sa_tbot + skt Sa_tskn + d2m Sa_tdew + msl Sa_pslv + sp Sa_pbot + tp Faxa_rain + cp Faxa_rainc + lsp Faxa_rainl + csf Faxa_snowc + lsf Faxa_snowl + ssrd Faxa_swdn + ssr Faxa_swnet + strd Faxa_lwdn + str Faxa_lwnet + aluvp Faxa_swvdr + aluvd Faxa_swvdf + alnip Faxa_swndr + alnid Faxa_swndf + sshf Faxa_sen + slhf Faxa_lat + ewss Faxa_taux + nsss Faxa_tauy + + 0 + + + diff --git a/parm/cdeps/datm_in b/parm/cdeps/datm_in new file mode 100755 index 000000000..3c481f34e --- /dev/null +++ b/parm/cdeps/datm_in @@ -0,0 +1,15 @@ +&datm_nml + datamode = "ERA5" + factorfn_data = "null" + factorfn_mesh = "null" + flds_co2 = .false. + flds_presaero = .false. + flds_wiso = .false. + iradsw = 1 + model_createmesh_fromfile = "null" + model_maskfile = "_mesh_atm_" + model_meshfile = "_mesh_atm_" + nx_global = 1440 + ny_global = 721 + restfilm = "null" +/ diff --git a/parm/cdeps/modelio.nml b/parm/cdeps/modelio.nml new file mode 100755 index 000000000..959488b19 --- /dev/null +++ b/parm/cdeps/modelio.nml @@ -0,0 +1,8 @@ +&pio_inparm + pio_netcdf_format = "64bit_offset" + pio_numiotasks = -99 + pio_rearranger = 1 + pio_root = 1 + pio_stride = 36 + pio_typename = "netcdf" +/ diff --git a/parm/cdeps/pio_in b/parm/cdeps/pio_in new file mode 100755 index 000000000..0282ee872 --- /dev/null +++ b/parm/cdeps/pio_in @@ -0,0 +1,34 @@ +&papi_inparm + papi_ctr1_str = "PAPI_FP_OPS" + papi_ctr2_str = "PAPI_NO_CTR" + papi_ctr3_str = "PAPI_NO_CTR" + papi_ctr4_str = "PAPI_NO_CTR" +/ +&pio_default_inparm + pio_async_interface = .false. + pio_blocksize = -1 + pio_buffer_size_limit = -1 + pio_debug_level = 0 + pio_rearr_comm_enable_hs_comp2io = .true. + pio_rearr_comm_enable_hs_io2comp = .false. + pio_rearr_comm_enable_isend_comp2io = .false. + pio_rearr_comm_enable_isend_io2comp = .true. + pio_rearr_comm_fcd = "2denable" + pio_rearr_comm_max_pend_req_comp2io = 0 + pio_rearr_comm_max_pend_req_io2comp = 64 + pio_rearr_comm_type = "p2p" +/ +&prof_inparm + profile_add_detail = .false. + profile_barrier = .false. + profile_depth_limit = 4 + profile_detail_limit = 2 + profile_disable = .false. + profile_global_stats = .true. + profile_outpe_num = 1 + profile_outpe_stride = 0 + profile_ovhd_measurement = .false. + profile_papi_enable = .false. + profile_single_file = .false. + profile_timer = 4 +/ diff --git a/parm/forecast/regional/nems.configure.datm_ocn.tmp b/parm/forecast/regional/nems.configure.datm_ocn.tmp new file mode 100755 index 000000000..692036f5d --- /dev/null +++ b/parm/forecast/regional/nems.configure.datm_ocn.tmp @@ -0,0 +1,121 @@ +############################################# +##### NEMS Run-Time Configuration File ##### +############################################## + +# EARTH # +EARTH_component_list: MED ATM OCN +EARTH_attributes:: + Verbosity = 0 +:: + +# MED # +MED_model: nems +_MED_petlist_bounds_ +:: + +# ATM # +ATM_model: datm +_ATM_petlist_bounds_ +ATM_attributes:: + Verbosity = low + domain_atm = _domain_atm_ + mesh_atm = _mesh_atm_ +:: + +# OCN # +OCN_model: hycom +_OCN_petlist_bounds_ +OCN_attributes:: + Verbosity = 1 + Diagnostic = 0 + cdf_impexp_freq = 3 + cpl_hour = 0 + cpl_min = 0 + cpl_sec = _cpl_dt_ + base_dtg = _base_dtg_ + merge_import = _merge_import_ + skip_first_import = .true. + hycom_arche_output = .true. + hyc_esmf_exp_output = .true. + hyc_esmf_imp_output = .true. + hyc_impexp_file = dummy_file + espc_show_impexp_minmax = .true. + ocean_start_dtg = _ocean_start_dtg_ + start_hour = 0 + start_min = 0 + start_sec = 0 + end_hour = _end_hour_ + end_min = 0 + end_sec = 0 +:: + +# Run Sequence # +runSeq:: +@_cpl_dt_ + MED med_phases_prep_ocn_accum_avg + MED -> OCN :remapMethod=redist + OCN + OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true + MED med_phases_prep_ocn_map + MED med_phases_prep_ocn_merge + MED med_phases_prep_ocn_accum_fast + MED med_phases_ocnalb_run + ATM + ATM -> MED :remapMethod=redist + MED med_phases_history_write + MED med_phases_restart_write + MED med_phases_profile +@ +:: + +# Other Attributes # +DRIVER_attributes:: + mediator_read_restart = false +:: + +MED_attributes:: + ATM_model = datm + OCN_model = hycom + history_n = 1 + history_option = nhours + history_ymd = -999 + coupling_mode = hafs + atm2ice_fmapname = unset + atm2ice_smapname = unset + atm2ice_vmapname = unset + atm2ocn_fmapname = unset + atm2ocn_smapname = unset + atm2ocn_vmapname = unset + ocn2atm_fmapname = unset + ocn2atm_smapname = unset + ice2atm_fmapname = unset + ice2atm_smapname = unset +:: + +ALLCOMP_attributes:: + ATM_model = datm + OCN_model = hycom + ScalarFieldCount = 3 + ScalarFieldIdxGridNX = 1 + ScalarFieldIdxGridNY = 2 + ScalarFieldIdxNextSwCday = 3 + ScalarFieldName = cpl_scalars + start_type = startup + case_name = ufs.hafs + restart_n = 48 + restart_option = nhours + restart_ymd = -999 + dbug_flag = 0 + use_coldstart = true + orb_eccen = 1.e36 + orb_iyear = 2000 + orb_iyear_align = 2000 + orb_mode = fixed_year + orb_mvelp = 1.e36 + orb_obliq = 1.e36 +:: + +ATM_modelio:: + diro = ./ + logfile = atm.log +:: diff --git a/parm/hafs.conf b/parm/hafs.conf index 2d54ebed9..ebefd04bf 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -40,6 +40,7 @@ realstormwork={CDSCRUB}/{RUNhafs}/{YMDH}/{realstorm} oldsid={oldvit[stormid3]} oldcom={CDSCRUB}/{RUNhafs}/com/{oldvit[YMDH]}/{oldvit[stormid3]} intercom={WORKhafs}/intercom ;; dir for communicating data files between jobs +INPdir={intercom}/chgres ;; input dir for forecast job outatcf={CDNOSCRUB}/{SUBEXPT} ;; Delivery location for ATCF files outdiag={CDNOSCRUB}/diagtrak/{SUBEXPT} ;; Delivery location for wrfdiag files outstatus={CDNOSCRUB}/cycstatus/{SUBEXPT} ;; Delivery location for status files @@ -249,10 +250,14 @@ runSeq_ATM2OCN=auto runSeq_ATM=auto runSeq_OCN=auto ATM_petlist_bounds=auto +MED_petlist_bounds=auto OCN_petlist_bounds=auto ocean_start_dtg=auto merge_import=.false. +domain_atm=dummy.nc +mesh_atm=dummy.nc + [post] # Grid definition for post and tracker, used by wgrib2 # Example: @@ -313,6 +318,9 @@ FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layo RUN_GSI={run_gsi} ;; Do we run GSI? RUN_OCEAN={run_ocean} ;; Do we run with ocean coupling? RUN_WAVE={run_wave} ;; Do we run with wave coupling? +RUN_DATM={run_datm} ;; Do we run with a data atmosphere using CDEPS? +RUN_DOCN={run_docn} ;; Do we run with a data ocean using CDEPS? +RUN_DWAV={run_dwav} ;; Do we run with data waves using CDEPS? RUN_VORTEXINIT={run_vortexinit} ;; Do we enable vortex initialization? RUN_HRDGRAPHICS={run_hrdgraphics} ;; Do we run HRD graphics? RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index 2612fab3d..9c532de20 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -31,6 +31,9 @@ run_gsi=no ;; GSI and FGAT initialization run_wave=no ;; Wave coupling (placeholder) run_ocean=no ;; Ocean coupling ocean_model=hycom +run_datm=no ;; Data atmosphere using CDEPS +run_docn=no ;; Data ocean using CDEPS +run_dwav=no ;; Data waves using CDEPS run_hrdgraphics=no ;; Run HRD graphics run_emcgraphics=no ;; Run EMC graphics diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf new file mode 100644 index 000000000..dd74d7807 --- /dev/null +++ b/parm/hafs_cdeps.conf @@ -0,0 +1,20 @@ +[config] +run_datm=yes ;; Data atmosphere using CDEPS +run_docn=no ;; Data ocean using CDEPS +run_dwav=no ;; Data waves using CDEPS + +[dir] +INPdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input + +[forecast] +domain_atm={INPdir}/static/domain.lnd.TL639_Atlantic8.200618.nc +mesh_atm={INPdir}/static/TL639_200618_ESMFmesh.nc + +layoutx=10 +layouty=8 +write_groups=1 +write_tasks_per_group=40 +ocean_tasks=120 + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 29ee47ea3..3c6c0c885 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -93,6 +93,10 @@ runSeq_ATM={forecast/runSeq_ATM} runSeq_OCN={forecast/runSeq_OCN} ocean_tasks={forecast/ocean_tasks} ATM_petlist_bounds={forecast/ATM_petlist_bounds} +MED_petlist_bounds={forecast/MED_petlist_bounds} OCN_petlist_bounds={forecast/OCN_petlist_bounds} ocean_start_dtg={forecast/ocean_start_dtg} merge_import={forecast/merge_import} + +domain_atm={forecast/domain_atm} +mesh_atm={forecast/mesh_atm} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 88ada9bc9..1af09a2b6 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -31,6 +31,14 @@ export COMhafs={COMhafs} export COMIN={COMIN} export COMOUT={COMOUT} export COMgfs={COMgfs} +export INPdir={INPdir} + +export run_datm={run_datm} +export run_docn={run_docn} +export run_dwav={run_dwav} + +export domain_atm={domain_atm} +export mesh_atm={mesh_atm} export SYNDAThafs={syndat} export ADECKhafs={ADECKhafs} @@ -122,6 +130,7 @@ export runSeq_ATM={runSeq_ATM} export runSeq_OCN={runSeq_OCN} export ocean_tasks={ocean_tasks} export ATM_petlist_bounds={ATM_petlist_bounds} +export MED_petlist_bounds={MED_petlist_bounds} export OCN_petlist_bounds={OCN_petlist_bounds} export ocean_start_dtg={ocean_start_dtg} export merge_import={merge_import} diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index aaa086219..b6f2514a9 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -63,6 +63,7 @@ + @@ -181,6 +182,7 @@ &FETCH_INPUT;YES + &RUN_DATM;NO @@ -217,6 +219,7 @@ + &RUN_DATM;NO &FETCH_INPUT;YES @@ -251,6 +254,7 @@ + &RUN_DATM;NO &FETCH_INPUT;YES @@ -360,9 +364,16 @@ - + + &RUN_DATM;YES + + +@** if gtype==regional + +@** endif + + @** if gtype==regional - @** if RUN_OCEAN==YES @@ -385,11 +396,14 @@ - - - - + already completed and we are not using a data atmosphere. --> + + &RUN_DATM;NO + + + + + @@ -430,11 +444,14 @@ - - - - + already completed and we are not using a data atmosphere. --> + + &RUN_DATM;NO + + + + + diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 414b53ffd..c9107d54f 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -50,6 +50,9 @@ 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=40:tpp=1TOTAL_TASKS240NCTSK40OMP_THREADS103:00:00"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> 56:ppn=12:tpp=2TOTAL_TASKS672NCTSK12&FORECAST_EXTRA;"> diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index b907d3bc3..4c23f6d4b 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -55,6 +55,10 @@ export ccpp_suite_regional=${ccpp_suite_regional:-HAFS_v0_gfdlmp_nocp} export ccpp_suite_glob=${ccpp_suite_glob:-HAFS_v0_gfdlmp} export ccpp_suite_nest=${ccpp_suite_nest:-HAFS_v0_gfdlmp_nocp} +export run_datm=${run_datm:-no} +export domain_atm=${domain_atm:-''} +export mesh_atm=${mesh_atm:-''} + export run_ocean=${run_ocean:-no} export ocean_model=${ocean_model:-hycom} export cpl_ocean=${cpl_ocean:-0} @@ -108,6 +112,8 @@ ln -sf ${INPdir}/*.nc INPUT/ #---------------------------------------------- # Copy all the necessary fix files #---------------------------------------------- +if [ ${run_datm} = no ]; then + cp $FIXam/global_solarconstant_noaa_an.txt solarconstant_noaa_an.txt cp $FIXam/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77 global_o3prdlos.f77 cp $FIXam/global_h2o_pltc.f77 global_h2oprdlos.f77 @@ -315,6 +321,12 @@ sed -e "s/_fhmax_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.false.}/g" \ input.nml.tmp > input.nml +fi # if regional + +fi # if not cdeps datm + +if [ $gtype = regional ]; then + if [ ${run_ocean} = yes ]; then # Copy hycom related files @@ -359,7 +371,11 @@ cp ${PARMhycom}/hafs_${hycom_basin}.basin.ports.input ports.input cp ${PARMhycom}/hafs_${hycom_basin}.basin.patch.input.${ocean_tasks} patch.input # copy fd_nems.yaml -cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS/mediator/fd_nems.yaml ./ +if [ ${run_datm} = no ]; then + cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS/mediator/fd_nems.yaml ./ +else + cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS/mediator/fd_cdeps.yaml fd.yaml +fi # create hycom limits ${USHhafs}/hafs_hycom_limits.py ${yr}${mn}${dy}${cyc} @@ -378,8 +394,48 @@ ${yr}${mn}${dy}.${cyc}Z.${CASE}.32bit.non-hydro $yr $mn $dy $cyc 0 0 EOF +if [ ${run_datm} = no ]; then + cat temp diag_table.tmp > diag_table +else + +#---------------------------------------------- +# Copy CDEPS parm files if required. +#---------------------------------------------- + cp ${PARMforecast}/model_configure.tmp . + cp ${PARMhafs}/cdeps/datm_in . + cp ${PARMhafs}/cdeps/datm.streams.xml . + cp ${PARMhafs}/cdeps/pio_in . + cp ${PARMhafs}/cdeps/modelio.nml atm_modelio.nml + cp ${PARMhafs}/cdeps/modelio.nml med_modelio.nml + cp ${PARMhafs}/cdeps/modelio.nml ocn_modelio.nml + + sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in + + sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams.xml + for file in `ls INPUT/*.nc ` ; do + sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" datm.streams.xml + done + + ln -sf ${domain_atm} INPUT/ + ln -sf ${mesh_atm} INPUT/ + + cp ${PARMforecast}/nems.configure.datm_ocn.tmp ./ + sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ + -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ + -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ + -e "s/_cpl_dt_/${cpl_dt}/g" \ + -e "s/_base_dtg_/${CDATE}/g" \ + -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ + -e "s/_end_hour_/${NHRS}/g" \ + -e "s/_merge_import_/${merge_import:-.true.}/g" \ + -e "s/_domain_atm_/INPUT\/$(basename $domain_atm)/g" \ + -e "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" \ + nems.configure.datm_ocn.tmp > nems.configure + +fi + cat model_configure.tmp | sed s/NTASKS/$TOTAL_TASKS/ | sed s/YR/$yr/ | \ sed s/MN/$mn/ | sed s/DY/$dy/ | sed s/H_R/$cyc/ | \ sed s/NHRS/$NHRS/ | sed s/NTHRD/$OMP_NUM_THREADS/ | \ diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index bececfe33..8d7fa7834 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -1403,6 +1403,7 @@ def make_holdvars(self,part1='{PARMhafs}/hafs_holdvars.txt',part2=None): runSeq_ATM=self.getstr('forecast','runSeq_ATM','auto') runSeq_OCN=self.getstr('forecast','runSeq_OCN','auto') ATM_petlist_bounds=self.getstr('forecast','ATM_petlist_bounds','auto') + MED_petlist_bounds=self.getstr('forecast','MED_petlist_bounds','auto') OCN_petlist_bounds=self.getstr('forecast','OCN_petlist_bounds','auto') gtype=self.getstr('grid','gtype','regional') glob_layoutx=self.getint('forecast','glob_layoutx',8) @@ -1450,6 +1451,10 @@ def make_holdvars(self,part1='{PARMhafs}/hafs_holdvars.txt',part2=None): ATM_petlist_bounds='"ATM_petlist_bounds: %04d %04d"'%(0, ATM_tasks-1) self.set('holdvars','ATM_petlist_bounds',ATM_petlist_bounds) + if MED_petlist_bounds=='auto': + MED_petlist_bounds='"MED_petlist_bounds: %04d %04d"'%(0, ATM_tasks-1) + self.set('holdvars','MED_petlist_bounds',MED_petlist_bounds) + if run_ocean and OCN_petlist_bounds=='auto': OCN_petlist_bounds='"OCN_petlist_bounds: %04d %04d"'%(ATM_tasks, ATM_tasks+OCN_tasks-1) self.set('holdvars','OCN_petlist_bounds',OCN_petlist_bounds) From 153ebdfff36118da6d5451eb62b4cc41a58c0674 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 11 Dec 2020 23:26:33 -0600 Subject: [PATCH 06/73] update model to support data components with CDEPS --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index f9047d16a..6b9796642 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit f9047d16a42b99486d96d3d3c3e2637234bb7d1e +Subproject commit 6b97966427b5cf5ddc25eae3f3f484bb93620f50 From 939f7e2892c6421989f2ca9814e62dca4b1f82b7 Mon Sep 17 00:00:00 2001 From: evankalina Date: Wed, 6 Jan 2021 17:40:06 -0600 Subject: [PATCH 07/73] Do not use INPdir as the CDEPS input data directory, since this variable is already used locally by multiple scripts in the workflow. The new CDEPS input data directory is called DATMdir and is specific to the CDEPS DATM. --- parm/hafs.conf | 2 +- parm/hafs_cdeps.conf | 6 +++--- parm/hafs_holdvars.txt | 2 +- scripts/exhafs_forecast.sh | 21 ++++++++++----------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index ebefd04bf..6347482a6 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -40,7 +40,7 @@ realstormwork={CDSCRUB}/{RUNhafs}/{YMDH}/{realstorm} oldsid={oldvit[stormid3]} oldcom={CDSCRUB}/{RUNhafs}/com/{oldvit[YMDH]}/{oldvit[stormid3]} intercom={WORKhafs}/intercom ;; dir for communicating data files between jobs -INPdir={intercom}/chgres ;; input dir for forecast job +DATMdir=dummy ;; input dir for cdeps datm outatcf={CDNOSCRUB}/{SUBEXPT} ;; Delivery location for ATCF files outdiag={CDNOSCRUB}/diagtrak/{SUBEXPT} ;; Delivery location for wrfdiag files outstatus={CDNOSCRUB}/cycstatus/{SUBEXPT} ;; Delivery location for status files diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index dd74d7807..176aad1fa 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -4,11 +4,11 @@ run_docn=no ;; Data ocean using CDEPS run_dwav=no ;; Data waves using CDEPS [dir] -INPdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input +DATMdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input [forecast] -domain_atm={INPdir}/static/domain.lnd.TL639_Atlantic8.200618.nc -mesh_atm={INPdir}/static/TL639_200618_ESMFmesh.nc +domain_atm={DATMdir}/static/domain.lnd.TL639_Atlantic8.200618.nc +mesh_atm={DATMdir}/static/TL639_200618_ESMFmesh.nc layoutx=10 layouty=8 diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 1af09a2b6..94804e234 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -31,7 +31,7 @@ export COMhafs={COMhafs} export COMIN={COMIN} export COMOUT={COMOUT} export COMgfs={COMgfs} -export INPdir={INPdir} +export DATMdir={DATMdir} export run_datm={run_datm} export run_docn={run_docn} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index fc808309d..46fd1f0d5 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -99,21 +99,22 @@ yr=`echo $CDATE | cut -c1-4` mn=`echo $CDATE | cut -c5-6` dy=`echo $CDATE | cut -c7-8` +mkdir -p INPUT RESTART + +#---------------------------------------------- +# Copy all the necessary fix files +#---------------------------------------------- +if [ ${run_datm} = no ]; then + if [ ! -d $INPdir ]; then echo Cannot find $INPdir ... exit exit 1 fi -mkdir -p INPUT RESTART ln -sf ${INPdir}/*.nc INPUT/ #cp ${INPdir}/*.nc INPUT/ #rsync ${INPdir}/*.nc INPUT/ -#---------------------------------------------- -# Copy all the necessary fix files -#---------------------------------------------- -if [ ${run_datm} = no ]; then - cp $FIXam/global_solarconstant_noaa_an.txt solarconstant_noaa_an.txt cp $FIXam/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77 global_o3prdlos.f77 cp $FIXam/global_h2o_pltc.f77 global_h2oprdlos.f77 @@ -371,11 +372,7 @@ cp ${PARMhycom}/hafs_${hycom_basin}.basin.ports.input ports.input cp ${PARMhycom}/hafs_${hycom_basin}.basin.patch.input.${ocean_tasks} patch.input # copy fd_nems.yaml -if [ ${run_datm} = no ]; then - cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml ./ -else - cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS-interface/CMEPS/mediator/fd_cdeps.yaml fd.yaml -fi +cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml ./ # create hycom limits ${USHhafs}/hafs_hycom_limits.py ${yr}${mn}${dy}${cyc} @@ -400,6 +397,8 @@ cat temp diag_table.tmp > diag_table else + ln -sf ${DATMdir}/*.nc INPUT/ + #---------------------------------------------- # Copy CDEPS parm files if required. #---------------------------------------------- From 64ea33e3957f8f2d77c89aba53f340b6386ce65e Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 8 Jan 2021 16:37:41 -0600 Subject: [PATCH 08/73] update model to enable data components with cmake build --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 41f4e60bb..483f62775 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 41f4e60bb6333bc4a0374ef06b5ec4e8f2ba1708 +Subproject commit 483f627758342bd776285872c646a7c6854265d7 From 2e167738d79e0cde9ada4c34f6cd9c005804aca4 Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Tue, 12 Jan 2021 01:08:49 +0000 Subject: [PATCH 09/73] Update the hafs_forecast.fd submodule after bringing in FV3 regional grid corners, HYCOM_merged branch, ability to set NEMS connector attributes through config, and CMEPS datm mask value fix. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 483f62775..accfe8d65 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 483f627758342bd776285872c646a7c6854265d7 +Subproject commit accfe8d658a9f42fe6e083901f643a73019b12f4 From 6c45e699ff8bfac3d0b0bc3d5867656bec98d6bc Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 12 Feb 2021 16:40:11 -0600 Subject: [PATCH 10/73] Point to feature/hafs_couplehycom_cdeps of ufs-weather-model --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 0ced7cb0b..516a9a292 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-weather-model.git - branch = feature/hafs_couplehycom_cmeps + branch = feature/hafs_couplehycom_cdeps [submodule "EMC_post"] path = sorc/hafs_post.fd url = https://github.com/hafs-community/EMC_post.git From 9c96b3668d1ebd0b516c57d32d9f268a60d3a184 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 12 Feb 2021 16:59:54 -0600 Subject: [PATCH 11/73] Finish pointing to feature/hafs_couplehycom_cdeps of ufs-weather-model --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 883303404..db3d4c813 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 88330340413564c6f9a64318d55e24a5cbd7529b +Subproject commit db3d4c8138b458a9c5cddd5c29dcb987d6e1d58f From 10482f5fdeb561bcb0d76315a89de3a5ca9c47ec Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 12 Feb 2021 18:13:40 -0600 Subject: [PATCH 12/73] Advance hafs_gsi, hafs_post, and hafs_utils to the latest versions used by b/develop. --- sorc/hafs_gsi.fd | 2 +- sorc/hafs_post.fd | 2 +- sorc/hafs_utils.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 22b9b6b52..1fa9aa61f 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 22b9b6b52ba55fab25ed22c47385581ef0eef307 +Subproject commit 1fa9aa61f00c4af54b44d8bfa075e5ef71657ee4 diff --git a/sorc/hafs_post.fd b/sorc/hafs_post.fd index 6e328c907..108a5c512 160000 --- a/sorc/hafs_post.fd +++ b/sorc/hafs_post.fd @@ -1 +1 @@ -Subproject commit 6e328c907ca56eb3e6fbf02a2a077a66a8274c96 +Subproject commit 108a5c512244299a55f737fd41129093f8a9b6c9 diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index 4185d3561..a8efc940c 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit 4185d356199b2655183060d8499d1f8f5611623e +Subproject commit a8efc940c027bd785a2908f39dabe4c44d65beac From 83b8799f46aa2067620f7f86e603d126d83567db Mon Sep 17 00:00:00 2001 From: evankalina Date: Wed, 17 Feb 2021 14:37:05 -0600 Subject: [PATCH 13/73] A few small updates for CDEPS DATM --- .../regional/nems.configure.datm_ocn.tmp | 30 ++++++------------- parm/hafs_cdeps.conf | 1 - parm/hafs_holdvars.conf | 1 - parm/hafs_holdvars.txt | 1 - scripts/exhafs_forecast.sh | 11 ++++--- 5 files changed, 16 insertions(+), 28 deletions(-) diff --git a/parm/forecast/regional/nems.configure.datm_ocn.tmp b/parm/forecast/regional/nems.configure.datm_ocn.tmp index 692036f5d..206c6504e 100755 --- a/parm/forecast/regional/nems.configure.datm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.datm_ocn.tmp @@ -1,4 +1,4 @@ -############################################# +############################################## ##### NEMS Run-Time Configuration File ##### ############################################## @@ -18,7 +18,6 @@ ATM_model: datm _ATM_petlist_bounds_ ATM_attributes:: Verbosity = low - domain_atm = _domain_atm_ mesh_atm = _mesh_atm_ :: @@ -52,19 +51,18 @@ OCN_attributes:: # Run Sequence # runSeq:: @_cpl_dt_ - MED med_phases_prep_ocn_accum_avg + MED med_phases_prep_ocn_avg MED -> OCN :remapMethod=redist OCN - OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true - MED med_phases_prep_ocn_map - MED med_phases_prep_ocn_merge - MED med_phases_prep_ocn_accum_fast - MED med_phases_ocnalb_run ATM ATM -> MED :remapMethod=redist - MED med_phases_history_write - MED med_phases_restart_write - MED med_phases_profile + MED med_phases_post_atm + MED med_phases_aofluxes_run + MED med_phases_prep_ocn_accum + OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true + MED med_phases_post_ocn + MED med_phases_history_write + MED med_phases_restart_write @ :: @@ -80,16 +78,6 @@ MED_attributes:: history_option = nhours history_ymd = -999 coupling_mode = hafs - atm2ice_fmapname = unset - atm2ice_smapname = unset - atm2ice_vmapname = unset - atm2ocn_fmapname = unset - atm2ocn_smapname = unset - atm2ocn_vmapname = unset - ocn2atm_fmapname = unset - ocn2atm_smapname = unset - ice2atm_fmapname = unset - ice2atm_smapname = unset :: ALLCOMP_attributes:: diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index 176aad1fa..7110ae7df 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -7,7 +7,6 @@ run_dwav=no ;; Data waves using CDEPS DATMdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input [forecast] -domain_atm={DATMdir}/static/domain.lnd.TL639_Atlantic8.200618.nc mesh_atm={DATMdir}/static/TL639_200618_ESMFmesh.nc layoutx=10 diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 832c70db7..7790c2755 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -90,5 +90,4 @@ ocean_tasks={forecast/ocean_tasks} ocean_start_dtg={forecast/ocean_start_dtg} merge_import={forecast/merge_import} -domain_atm={forecast/domain_atm} mesh_atm={forecast/mesh_atm} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 2c2297bb8..5ca6a9b29 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -37,7 +37,6 @@ export run_datm={run_datm} export run_docn={run_docn} export run_dwav={run_dwav} -export domain_atm={domain_atm} export mesh_atm={mesh_atm} export SYNDAThafs={syndat} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 0302ce386..0b8a90596 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -56,7 +56,6 @@ export ccpp_suite_glob=${ccpp_suite_glob:-HAFS_v0_gfdlmp} export ccpp_suite_nest=${ccpp_suite_nest:-HAFS_v0_gfdlmp_nocp} export run_datm=${run_datm:-no} -export domain_atm=${domain_atm:-''} export mesh_atm=${mesh_atm:-''} export run_ocean=${run_ocean:-no} @@ -107,12 +106,18 @@ if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 2 ]; then export runSeq_ALL="OCN -> ATM :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=0\n ATM -> OCN :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=1:dstmaskvalues=0\n ATM\n OCN" fi # CMEPS based coupling through the bilinear regridding method -if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ]; then +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ] && [ ${run_datm} = no ]; then export cplflx=.true. export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) export runSeq_ALL="ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n OCN -> MED :remapMethod=redist\n MED med_phases_post_ocn\n MED med_phases_prep_atm\n MED med_phases_prep_ocn_accum\n MED med_phases_prep_ocn_avg\n MED -> ATM :remapMethod=redist\n MED -> OCN :remapMethod=redist\n ATM\n OCN" fi +# CMEPS based coupling between HYCOM and DATM through the bilinear regridding method +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ] && [ ${run_datm} = yes ]; then + export cplflx=.true. + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" 0 $(($ATM_tasks-1))) +fi export ocean_start_dtg=${ocean_start_dtg:-43340.00000} #export base_dtg=${CDATE:-2019082900} @@ -462,7 +467,6 @@ else sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" datm.streams.xml done - ln -sf ${domain_atm} INPUT/ ln -sf ${mesh_atm} INPUT/ cp ${PARMforecast}/nems.configure.datm_ocn.tmp ./ @@ -474,7 +478,6 @@ else -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.true.}/g" \ - -e "s/_domain_atm_/INPUT\/$(basename $domain_atm)/g" \ -e "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" \ nems.configure.datm_ocn.tmp > nems.configure From 31a29cabd9bf211cfe8edd493f34c3e68351746a Mon Sep 17 00:00:00 2001 From: evankalina Date: Wed, 3 Mar 2021 18:02:05 -0600 Subject: [PATCH 14/73] Update to the most recent version of the feature/hafs_couplehycom_cdeps branch of UFS weather model. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index db3d4c813..ee0f23fb9 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit db3d4c8138b458a9c5cddd5c29dcb987d6e1d58f +Subproject commit ee0f23fb93b601bc58c66e92982317a8c00f4a7d From 29b111edf02aedf0c9172bd9166eb7afbe6fe1b2 Mon Sep 17 00:00:00 2001 From: evankalina Date: Wed, 3 Mar 2021 18:05:14 -0600 Subject: [PATCH 15/73] CDEPS no longer needs the pio_in and X_modelio.nml files. --- parm/cdeps/modelio.nml | 8 -------- parm/cdeps/pio_in | 34 ---------------------------------- scripts/exhafs_forecast.sh | 4 ---- 3 files changed, 46 deletions(-) delete mode 100755 parm/cdeps/modelio.nml delete mode 100755 parm/cdeps/pio_in diff --git a/parm/cdeps/modelio.nml b/parm/cdeps/modelio.nml deleted file mode 100755 index 959488b19..000000000 --- a/parm/cdeps/modelio.nml +++ /dev/null @@ -1,8 +0,0 @@ -&pio_inparm - pio_netcdf_format = "64bit_offset" - pio_numiotasks = -99 - pio_rearranger = 1 - pio_root = 1 - pio_stride = 36 - pio_typename = "netcdf" -/ diff --git a/parm/cdeps/pio_in b/parm/cdeps/pio_in deleted file mode 100755 index 0282ee872..000000000 --- a/parm/cdeps/pio_in +++ /dev/null @@ -1,34 +0,0 @@ -&papi_inparm - papi_ctr1_str = "PAPI_FP_OPS" - papi_ctr2_str = "PAPI_NO_CTR" - papi_ctr3_str = "PAPI_NO_CTR" - papi_ctr4_str = "PAPI_NO_CTR" -/ -&pio_default_inparm - pio_async_interface = .false. - pio_blocksize = -1 - pio_buffer_size_limit = -1 - pio_debug_level = 0 - pio_rearr_comm_enable_hs_comp2io = .true. - pio_rearr_comm_enable_hs_io2comp = .false. - pio_rearr_comm_enable_isend_comp2io = .false. - pio_rearr_comm_enable_isend_io2comp = .true. - pio_rearr_comm_fcd = "2denable" - pio_rearr_comm_max_pend_req_comp2io = 0 - pio_rearr_comm_max_pend_req_io2comp = 64 - pio_rearr_comm_type = "p2p" -/ -&prof_inparm - profile_add_detail = .false. - profile_barrier = .false. - profile_depth_limit = 4 - profile_detail_limit = 2 - profile_disable = .false. - profile_global_stats = .true. - profile_outpe_num = 1 - profile_outpe_stride = 0 - profile_ovhd_measurement = .false. - profile_papi_enable = .false. - profile_single_file = .false. - profile_timer = 4 -/ diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 0b8a90596..a2adb793a 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -455,10 +455,6 @@ else cp ${PARMforecast}/model_configure.tmp . cp ${PARMhafs}/cdeps/datm_in . cp ${PARMhafs}/cdeps/datm.streams.xml . - cp ${PARMhafs}/cdeps/pio_in . - cp ${PARMhafs}/cdeps/modelio.nml atm_modelio.nml - cp ${PARMhafs}/cdeps/modelio.nml med_modelio.nml - cp ${PARMhafs}/cdeps/modelio.nml ocn_modelio.nml sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in From a5972bf6cd953b0d7436b7d543def8664576032d Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 9 Mar 2021 09:44:49 -0600 Subject: [PATCH 16/73] Bug fix to set the year at the beginning and end of the forecast properly in datm.streams.xml --- parm/cdeps/datm.streams.xml | 4 ++-- scripts/exhafs_forecast.sh | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/parm/cdeps/datm.streams.xml b/parm/cdeps/datm.streams.xml index e15ae1bbb..27369ca29 100755 --- a/parm/cdeps/datm.streams.xml +++ b/parm/cdeps/datm.streams.xml @@ -7,8 +7,8 @@ single bilinear 1.5 - 2019 - 2019 + _yearFirst_ + _yearLast_ 1900 u:v _mesh_atm_ diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index a2adb793a..81bd100cb 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -458,6 +458,13 @@ else sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in + sed -i "s/_yearFirst_/$yr/g" datm.streams.xml + + enddate=`${NDATE} +${NHRS} $CDATE` + endyr=`echo $enddate | cut -c1-4` + + sed -i "s/_yearLast_/$endyr/g" datm.streams.xml + sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams.xml for file in `ls INPUT/*.nc ` ; do sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" datm.streams.xml From e0bff555ec414381bc4bca1800ce4bbde676cdb2 Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 9 Mar 2021 09:46:43 -0600 Subject: [PATCH 17/73] From Sam Trahan (CIRES at NOAA/GSL, DTC): Add a download script for ERA5 reanalysis data for the CDEPS DATM. --- ush/hafs_era5_downloader.py | 225 ++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100755 ush/hafs_era5_downloader.py diff --git a/ush/hafs_era5_downloader.py b/ush/hafs_era5_downloader.py new file mode 100755 index 000000000..4ce88efd0 --- /dev/null +++ b/ush/hafs_era5_downloader.py @@ -0,0 +1,225 @@ +#! /usr/bin/env python3 + +# This next line will abort in any version earlier than Python 3.6: +f'This script requires Python 3.6 or newer.' + +import time +import subprocess +import contextlib +import os +import tempfile +import getopt +import re +import logging +import datetime +import sys +import cdsapi +import produtil.setup, produtil.fileop, produtil.locking + +# Constants +UTILITY_NAME = 'era5_downloader' +VERSION_STRING = '0.0.1' +LOGGING_DOMAIN = UTILITY_NAME +DATASET = 'reanalysis-era5-single-levels' +PRODUCT_TYPE = 'reanalysis' +VARIABLES = [ + '10m_u_component_of_wind', '10m_v_component_of_wind', '2m_dewpoint_temperature', + '2m_temperature', 'convective_precipitation', 'convective_snowfall', + 'large_scale_precipitation', 'large_scale_snowfall', 'mean_sea_level_pressure', + 'near_ir_albedo_for_diffuse_radiation', 'near_ir_albedo_for_direct_radiation', + 'uv_visible_albedo_for_diffuse_radiation', 'uv_visible_albedo_for_direct_radiation', + 'surface_latent_heat_flux', 'surface_sensible_heat_flux', + 'surface_solar_radiation_downwards', 'surface_thermal_radiation_downwards', + 'surface_pressure', 'total_precipitation', 'skin_temperature', + 'eastward_turbulent_surface_stress', 'northward_turbulent_surface_stress', + 'surface_net_solar_radiation', 'surface_net_thermal_radiation' +] +FILE_FORMAT = 'netcdf' +CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) +EPSILON = datetime.timedelta(seconds=5) # epsilon for time comparison: five seconds + +# Non-constant globals: +dayset=set() # list of YYYYMMDD strings +happy=True # False = something failed +filename_format = 'ERA5_%Y%m%d' +swap_latitudes=True + +def usage(why=None): + print(f'''Synopsis: {UTILITY_NAME} [options] day [day [...]] + +Downloads the listed days of data. Days can be specified as: + 20210815 = specify one day: August 15, 2021 + 20210815-20210819 = specify a range of days: August 15th to 19th, 2021 + 2018 = specify an entire year (2018) + +Options: + -q | --quiet = log only warnings and errors + -v | --verbose = log all messages + -n | --no-invertlat = do not run "cdo invertlat" on downloaded files + -F format | --format format = filename format as in strftime(3) + -i | --invertlat = DO run "cdo inverlat". This is the default + --version = print {UTILITY_NAME} {VERSION_STRING} + --help = this message + +Format example: ERA5_%Y%m%d = ERA5_20210815 +Script will automatically append ".nc" +''') + if why: + sys.stderr.write(f'SCRIPT IS ABORTING BECAUSE: {why}\n') + return 1 + return 0 + +# Function that makes the singleton for cdsapi client: +_client = None +def client(): + global _client + if not _client: + logger.info('creating cdsapi client') + _client=cdsapi.Client() + return _client + +# Tell CDO to flip latitudes in a NetCDF file: +def cdo_swap_latitudes(filename_in,filename_out): + logger.info('Flip latitudes in "'+str(filename_in)+'" and write to "'+str(filename_out)+'"') + cmd = [ 'cdo', 'invertlat', filename_in, filename_out ] + logger.info(f'''Run "{'" "'.join(cmd) }"''') + result = subprocess.run(cmd) + result.check_returncode() + +def quiet_remove(filename): + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + +# The meat of the program: retrieve a file +def request(when): + filename_base = when.strftime(filename_format) + filename_download = filename_base+'_download.nc' + filename_invert = filename_base+'_invert.nc' + filename_lock = filename_base+'.lock' + filename_final = filename_base+'.nc' + if os.path.exists(filename_final): + logger.info(filename_final+': already exists. Skipping.') + return + with produtil.locking.LockFile(filename_lock,logger): + try: + quiet_remove(filename_download) + quiet_remove(filename_invert) + logger.info(filename_download+': retrieve '+str(when)+'...') + request = { + 'product_type': PRODUCT_TYPE, + 'variable': VARIABLES, + 'year': '%04d'%int(when.year), + 'month': [ '%02d'%int(when.month) ], + 'day': [ '%02d'%int(when.day) ], + 'time': [ '%02d'%hour for hour in range(24) ], + 'format': FILE_FORMAT, + } + # super-wordy debugging: logger.debug(filename_download+': request is '+str(request)) + client().retrieve(DATASET,request,filename_download) + filename_copy=filename_download + if swap_latitudes: + cdo_swap_latitudes(filename_download,filename_invert) + filename_copy=filename_invert + produtil.fileop.deliver_file(filename_copy,filename_final,logger=logger, + keep=False,verify=False,moveok=True,force=True) + quiet_remove(filename_download) + quiet_remove(filename_invert) + quiet_remove(filename_lock) + except Exception as e: + quiet_remove(filename_download) + quiet_remove(filename_invert) + raise e + +# Parse arguments and initialize logging: +log_level = logging.INFO +optlist,args = getopt.getopt(sys.argv[1:],'qveniF:',[ + 'version','help','verbose','quiet','invertlat','no-invertlat','format']) +if len(args)<1: + exit(usage("No arguments provided!")) +for optarg in optlist: + if optarg[0] in ['-q', '--quiet']: + log_level = logging.WARNING + elif optarg[0] in ['-v', '--verbose']: + log_level = logging.DEBUG + elif optarg[0] in ['-i', '--invertlat']: + invertlat = True + elif optarg[0] in ['-n', '--no-invertlat']: + invertlat = False + elif optarg[0] in ['-F', '--format']: + filename_format = optarg[1] + elif optarg[0]=='--help': + exit(usage()) + elif optarg[0]=='--version': + print(UTILITY_NAME+' '+VERSION_STRING) + exit(0) +logger = logging.getLogger(LOGGING_DOMAIN) + +produtil.setup.setup(level=log_level,send_dbn=False) + +# Parse the days. This loop was modified from run_hafs.py: +for arg in args: + if re.match('\A\d{8}\Z',arg): + logger.info('single date/time') + # Single date/time + dayset.add(arg) + elif re.match('\A\d{4}\Z',arg): + logger.info('year') + # Year + start=datetime.datetime(int(arg,8),1,1,0,0,0) + end=datetime.datetime(int(arg,8),12,31,23,59,0) + now=start + while now=len(daylist): + logger.info(f'{day}: sleep for a little while... 30 second snooze...') + time.sleep(30) + logger.info(f'{day}: done sleeping.') + iloop=0 + except Exception as ex: # Unfortunately, cdsapi raises Exception + happy = False + logger.error(f'CDSAPI failed to download day {day}: {ex}',exc_info=ex) + +# Exit 0 on success, 1 on failure: +exit( 0 if happy else 1 ) From 99144c2eba3e3102437969fa810b00afeef06ce2 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 12 Mar 2021 14:44:06 -0600 Subject: [PATCH 18/73] Generate DATM mesh in workflow if make_mesh_atm=yes --- jobs/JHAFS_DATM_MESH | 78 ++++++ parm/hafs.conf | 4 +- parm/hafs_basic.conf | 1 + parm/hafs_cdeps.conf | 5 +- parm/hafs_holdvars.txt | 1 + rocoto/hafs_workflow.xml.in | 30 +++ rocoto/run_hafs.py | 3 + rocoto/sites/hera.ent | 1 + rocoto/sites/kjet.ent | 1 + rocoto/sites/orion.ent | 1 + rocoto/sites/wcoss_cray.ent | 1 + rocoto/sites/wcoss_dell_p3.ent | 1 + rocoto/sites/xjet.ent | 1 + rocoto/sites/xjet_hafsv0p1a.ent | 1 + rocoto/sites/xjet_hafsv0p2a.ent | 1 + scripts/exhafs_datm_mesh.sh | 56 +++++ ush/hafs/launcher.py | 29 +++ ush/hafs_datm_mesh.py | 405 ++++++++++++++++++++++++++++++++ 18 files changed, 617 insertions(+), 3 deletions(-) create mode 100755 jobs/JHAFS_DATM_MESH create mode 100755 scripts/exhafs_datm_mesh.sh create mode 100755 ush/hafs_datm_mesh.py diff --git a/jobs/JHAFS_DATM_MESH b/jobs/JHAFS_DATM_MESH new file mode 100755 index 000000000..7ad8ba237 --- /dev/null +++ b/jobs/JHAFS_DATM_MESH @@ -0,0 +1,78 @@ +#!/bin/sh + +set -xe + +date +source ${USHhafs}/hafs_pre_job.sh.inc +HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} +source ${HOLDVARS} + +export EXPT=${EXPT:-HAFS} +export SUBEXPT=${SUBEXPT:-${EXPT}} + +export CDATE=${CDATE:-${YMDH}} +export cyc=${cyc:-00} +export STORM=${STORM:-FAKE} +export STORMID=${STORMID:-00L} + +#################################### +# Specify Execution Areas +#################################### +export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +export USHhafs=${USHhafs:-${HOMEhafs}/ush} + +#################################### +# Load Modules if Needed +#################################### +export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet +#export RUN_ENVIR=${RUN_ENVIR:-prod} +export RUN_ENVIR=${envir:-prod} +if [ "$RUN_ENVIR" = prod ]; then + #module use ${HOMEhafs}/modulefiles + #module load modulefile.hafs.run.${machine} + if [ "$machine" = wcoss_dell_p3 ]; then + module unload NetCDF + module unload HDF5-serial + fi + #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} + module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles + module load build.${machine}.intel + module list +fi + +##################################### +# Set up job node/core/run environment variables +##################################### +source ${USHhafs}/hafs_runcmd.sh.inc + +#################################### +# Run setpdy and initialize PDY variables +#################################### +#setpdy.sh +#. ./PDY + +PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) +export PDY=${PDY:-${PDYtmp}} + +##################################### +# Working directories +##################################### +export DATA=${WORKhafs}/datm_mesh + +export SCRUBDATA=${SCRUBDATA:-YES} + +if [ "${SCRUBDATA}" = YES ]; then + rm -rf $DATA +fi + +mkdir -p $DATA +cd $DATA + +##################################### +# Execute the script. +${HOMEhafs}/scripts/exhafs_datm_mesh.sh +export err=$? +##################################### + +exit $err diff --git a/parm/hafs.conf b/parm/hafs.conf index 17adab8c4..6301c065c 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -233,7 +233,7 @@ ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31 merge_import=.true. domain_atm=dummy.nc -mesh_atm=dummy.nc +# mesh_atm=dummy.nc ;; This variable is now set by ush/hafs/launcher.py [post] # Grid definition for post and tracker, used by wgrib2 @@ -304,4 +304,4 @@ RUN_HRDGRAPHICS={run_hrdgraphics} ;; Do we run HRD graphics? RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? - +MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index 8a7465008..b00d335bf 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -37,6 +37,7 @@ run_docn=no ;; Data ocean using CDEPS run_dwav=no ;; Data waves using CDEPS run_hrdgraphics=no ;; Run HRD graphics run_emcgraphics=no ;; Run EMC graphics +make_mesh_atm=no ;; Generate DATM mesh in workflow (only for run_datm=yes) # Scrub options scrub=no ;; do we delete temporary files? (placeholder) diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index 7110ae7df..840299923 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -2,12 +2,15 @@ run_datm=yes ;; Data atmosphere using CDEPS run_docn=no ;; Data ocean using CDEPS run_dwav=no ;; Data waves using CDEPS +make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen [dir] DATMdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input [forecast] -mesh_atm={DATMdir}/static/TL639_200618_ESMFmesh.nc +mesh_atm_in={DATMdir}/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change +# mesh_atm=... ;; This variable is now set automatically by ush/hafs/launcher.py layoutx=10 layouty=8 diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 5ca6a9b29..67b4d069d 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -37,6 +37,7 @@ export run_datm={run_datm} export run_docn={run_docn} export run_dwav={run_dwav} +export make_mesh_atm={make_mesh_atm} export mesh_atm={mesh_atm} export SYNDAThafs={syndat} diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index ce4f7fb61..4faa5702a 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -70,6 +70,7 @@ + @@ -365,6 +366,29 @@ +@** if RUN_DATM==YES + + + &JOBhafs;/JHAFS_DATM_MESH + hafs_datm_mesh_&SID;_@Y@m@d@H + &WORKhafs;/hafs_datm_mesh.log + &ACCOUNT; + &RESERVATION; + &QUEUE_PE; + &PE_EXTRA; + &MESH_ATM_RESOURCES; + &ENV_VARS; + + + + + &MAKE_MESH_ATM;YES + + + + +@** endif + @** endif @** endif @@ -395,6 +419,12 @@ @** if RUN_OCEAN==YES +@** if RUN_DATM==YES + + &MAKE_MESH_ATM;YES + + +@** endif @** endif @** endif diff --git a/rocoto/run_hafs.py b/rocoto/run_hafs.py index 64144bdd4..65fcdb159 100755 --- a/rocoto/run_hafs.py +++ b/rocoto/run_hafs.py @@ -418,6 +418,9 @@ def fullify(s): conf.timeless_sanity_check(enset,logger) except Exception as e: tcutil.rocoto.sanity_check_failed(logger,e) + logger.critical("HAFS Sanity Checker Designation: INSANE!") + logger.critical("Check your configuration for errors.") + logger.critical("See earlier messages for clues.") sys.exit(1) logger.info("I think I'm sane.") diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index c62a5c0f2..09dc9de8f 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 385bc05f4..8cbaf77f9 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 6fa0c0b89..03fcdd903 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index 48439d4a4..8864b779b 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index cda8e2a5a..321ea7ff8 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -15,6 +15,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 1:ppn=6:tpp=1TOTAL_TASKS6NCTSK6OMP_THREADS4-R affinity[core\(4\):distribute=balance]00:30:00"> 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK24OMP_THREADS100:30:00"> 2:ppn=24:tpp=1TOTAL_TASKS48NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 7a0024977..c28e7e0a0 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p1a.ent b/rocoto/sites/xjet_hafsv0p1a.ent index a00405257..2977e0173 100644 --- a/rocoto/sites/xjet_hafsv0p1a.ent +++ b/rocoto/sites/xjet_hafsv0p1a.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 6:ppn=24:tpp=1TOTAL_TASKS144NCTSK24OMP_THREADS100:30:00"> 6:ppn=24:tpp=1TOTAL_TASKS144NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index d3a046ad2..0dbbf2c50 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -14,6 +14,7 @@ 1G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 30:ppn=24:tpp=1TOTAL_TASKS720NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/scripts/exhafs_datm_mesh.sh b/scripts/exhafs_datm_mesh.sh new file mode 100755 index 000000000..d0fda6ced --- /dev/null +++ b/scripts/exhafs_datm_mesh.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [[ "$make_mesh_atm" != yes || "$run_datm" != yes ]] ; then + echo "This job should only be run if \$run_datm and \$make_mesh_atm are both yes." + echo " \$make_mesh_atm=\"$make_mesh_atm\"" + echo " \$run_datm=\"$run_datm\"" + echo "Billions of electrons have whizzed by, the universe's entropy has increased, and yet nothing has been accomplished." + echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " + exit 0 +fi + +set -xe + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} +APRUNS=${APRUNS:-"aprun -b -j1 -n1 -N1 -d1 -cc depth"} + +ofile=ofile.nc +mesh_atm="$mesh_atm" +mesh_dir=$( dirname "$mesh_atm" ) + +# Pick an input file. This will get the newest one, if the names are +# identical except for YYYYMMDD date string: +test -d "$DATMdir" +ifile=$( ls -1 "$DATMdir"/*nc | sort -ur | head -1 ) + +test -n "$ifile" +test -s "$ifile" +test -r "$ifile" + +[ -d "$mesh_dir" ] || mkdir "$mesh_dir" +rm -f "$mesh_atm" +[ -e "$ofile" -o -L "$ofile" ]] && rm -f "$ofile" + +echo "Generating ESMF mesh from ERA5 files." +echo "Running in dir \"$PWD\"" +echo "ERA5 grid generation input file is \"$ifile\"" +echo "Temporary output mesh is $ofile" +echo "Will deliver to \"$mesh_atm\"" + +# since the mesh is created drectly from the data file (lat is already +# reversed), there is no need to reverse in here again by providing +# --latrev +$APRUNS $USHhafs/hafs_datm_mesh.py --ifile "$ifile" --ofile "$ofile" \ + --overwrite --latvar latitude --lonvar longitude --double +test -s "$ofile" + +$USHhafs/produtil_deliver.py -m "$ofile" "$mesh_atm" +test -s "$mesh_atm" + +ls -l "$mesh_atm" + +echo "DATM mesh was successfully generated." +echo "Enjoy your mesh and have a nice day." diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index b22684d2a..624163cf8 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -558,6 +558,7 @@ def launch(file_list,cycle,stid,moreopt,case_root,init_dirs=True, %(section,option,repr(value))) conf.set(section,option,value) conf.guess_default_values() + conf.set_data_model_variables() cycling_interval=conf.getfloat('config','cycling_interval',6.0) cycling_interval=-abs(cycling_interval*3600.0) if cycle is not None: @@ -788,6 +789,33 @@ def read_tcvitals_and_messages(self,vitdir=None,vitpattern=None, revital.readfiles(inputs,raise_all=False) return revital + def sanity_check_data_models(self,logger): + """!In the hafs_launcher job, this checks the data model variables and + files for obvious errors before starting the rest of the workflow.""" + run_datm=self.getbool('config','run_datm',False) + if run_datm: + make_mesh_atm=self.getbool('config','make_mesh_datm',True) + if not make_mesh_atm: + mesh_atm=timestrinterp('forecast','mesh_atm') + if not os.path.exists(mesh_atm): + msg='%s: mesh_atm file does not exist'%(mesh_atm,) + logger.error(msg) + raise msg + else: + logger.info("%s: will use this pre-made datm esmf mesh (mesh_atm)."%(mesh_atm,)) + + def set_data_model_variables(self): + """!Sets conf variables for the data models.""" + run_datm=self.getbool('config','run_datm',False) + if run_datm: + make_mesh_atm=self.getbool('config','make_mesh_datm',True) + if make_mesh_atm: + self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_gen')) + else: + self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_in')) + else: + self.set('forecast','mesh_atm','dummy.nc') + def set_storm(self,syndat,oldsyndat): """!Sets the storm that is to be run. @@ -1204,6 +1232,7 @@ def sanity_check(self): %(repr(case_root),)) self.sanity_check_archive(logger) + self.sanity_check_data_models(logger) def guess_default_values(self): """!Tries to guess default values for many configuration settings. diff --git a/ush/hafs_datm_mesh.py b/ush/hafs_datm_mesh.py new file mode 100755 index 000000000..35a3ef07b --- /dev/null +++ b/ush/hafs_datm_mesh.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python + +import os, sys, getopt +import argparse +try: + import numpy as np + import xarray as xr + import dask.array as da + import dask.dataframe as dd + from dask.diagnostics import ProgressBar + from datetime import datetime + import pandas as pd +except ImportError as ie: + sys.stderr.write("""You are missing some modules! +The following commands can be used to install required Python modules to run this script + + pip install xarray --user + pip install dask --user + pip install "dask[array]" --upgrade —user + pip install "dask[dataframe]" --upgrade --user +""") + exit(2) + + +def calculate_corners(center_lat, center_lon): + """Calculate corner coordinates by averaging neighbor cells + """ + + # get rank + rank = len(center_lat.dims) + + if rank == 1: + # get dimensions + nlon = center_lon.size + nlat = center_lat.size + + # convert center points from 1d to 2d + center_lat2d = da.broadcast_to(center_lat.values[None,:], (nlon, nlat)) + center_lon2d = da.broadcast_to(center_lon.values[:,None], (nlon, nlat)) + elif rank == 2: + # get dimensions + dims = center_lon.shape + nlon = dims[0] + nlat = dims[1] + + # just rename and convert to dask array + center_lat2d = da.from_array(center_lat) + center_lon2d = da.from_array(center_lon) + else: + print('Unrecognized grid! The rank of coordinate variables can be 1 or 2 but it is {}.'.format(rank)) + sys.exit(2) + + # calculate corner coordinates for latitude, counterclockwise order, imposing Fortran ordering + center_lat2d_ext = da.from_array(np.pad(center_lat2d.compute(), (1,1), mode='reflect', reflect_type='odd')) + + ur = (center_lat2d_ext[1:-1,1:-1]+ + center_lat2d_ext[0:-2,1:-1]+ + center_lat2d_ext[1:-1,2:]+ + center_lat2d_ext[0:-2,2:])/4.0 + ul = (center_lat2d_ext[1:-1,1:-1]+ + center_lat2d_ext[0:-2,1:-1]+ + center_lat2d_ext[1:-1,0:-2]+ + center_lat2d_ext[0:-2,0:-2])/4.0 + ll = (center_lat2d_ext[1:-1,1:-1]+ + center_lat2d_ext[1:-1,0:-2]+ + center_lat2d_ext[2:,1:-1]+ + center_lat2d_ext[2:,0:-2])/4.0 + lr = (center_lat2d_ext[1:-1,1:-1]+ + center_lat2d_ext[1:-1,2:]+ + center_lat2d_ext[2:,1:-1]+ + center_lat2d_ext[2:,2:])/4.0 + + # this looks clockwise ordering but it is transposed and becomes counterclockwise, bit-to-bit with NCL + corner_lat = da.stack([ul.T.reshape((-1,)).T, ll.T.reshape((-1,)).T, lr.T.reshape((-1,)).T, ur.T.reshape((-1,)).T], axis=1) + + # calculate corner coordinates for longitude, counterclockwise order, imposing Fortran ordering + center_lon2d_ext = da.from_array(np.pad(center_lon2d.compute(), (1,1), mode='reflect', reflect_type='odd')) + + ur = (center_lon2d_ext[1:-1,1:-1]+ + center_lon2d_ext[0:-2,1:-1]+ + center_lon2d_ext[1:-1,2:]+ + center_lon2d_ext[0:-2,2:])/4.0 + ul = (center_lon2d_ext[1:-1,1:-1]+ + center_lon2d_ext[0:-2,1:-1]+ + center_lon2d_ext[1:-1,0:-2]+ + center_lon2d_ext[0:-2,0:-2])/4.0 + ll = (center_lon2d_ext[1:-1,1:-1]+ + center_lon2d_ext[1:-1,0:-2]+ + center_lon2d_ext[2:,1:-1]+ + center_lon2d_ext[2:,0:-2])/4.0 + lr = (center_lon2d_ext[1:-1,1:-1]+ + center_lon2d_ext[1:-1,2:]+ + center_lon2d_ext[2:,1:-1]+ + center_lon2d_ext[2:,2:])/4.0 + + # this looks clockwise ordering but it is transposed and becomes counterclockwise, bit-to-bit with NCL + corner_lon = da.stack([ul.T.reshape((-1,)).T, ll.T.reshape((-1,)).T, lr.T.reshape((-1,)).T, ur.T.reshape((-1,)).T], axis=1) + + return center_lat2d, center_lon2d, corner_lat, corner_lon + +def write_to_esmf_mesh(filename, center_lat, center_lon, corner_lat, corner_lon, mask, area=None): + """ + """ + # create array with unique coordinate pairs + # remove coordinates that are shared between the elements + corner_pair = da.stack([corner_lon.T.reshape((-1,)).T, corner_lat.T.reshape((-1,)).T], axis=1) + + # REPLACED: corner_pair_uniq = dd.from_dask_array(corner_pair).drop_duplicates().to_dask_array(lengths=True) + # following reduces memory by %17 + corner_pair_uniq = dd.from_dask_array(corner_pair).drop_duplicates().values + corner_pair_uniq.compute_chunk_sizes() + + # check size of unique coordinate pairs + dims = mask.shape + nlon = dims[0] + nlat = dims[1] + elem_conn_size = nlon*nlat+nlon+nlat+1 + if corner_pair_uniq.shape[0] != elem_conn_size: + print('The size of unique coordinate pairs is {} but expected size is {}!'.format(corner_pair_uniq.shape[0], elem_conn_size)) + print('Please check the input file or try to force double precision with --double option. Exiting ...') + sys.exit(2) + + # create element connections + corners = dd.concat([dd.from_dask_array(c) for c in [corner_lon.T.reshape((-1,)).T, corner_lat.T.reshape((-1,)).T]], axis=1) + corners.columns = ['lon', 'lat'] + elem_conn = corners.compute().groupby(['lon','lat'], sort=False).ngroup()+1 + elem_conn = da.from_array(elem_conn.to_numpy()) + + # create new dataset for output + out = xr.Dataset() + + out['origGridDims'] = xr.DataArray(np.array(center_lon.shape, dtype=np.int32), + dims=('origGridRank')) + + out['nodeCoords'] = xr.DataArray(corner_pair_uniq, + dims=('nodeCount', 'coordDim'), + attrs={'units': 'degrees'}) + + out['elementConn'] = xr.DataArray(elem_conn.T.reshape((4,-1)).T, + dims=('elementCount', 'maxNodePElement'), + attrs={'long_name': 'Node indices that define the element connectivity'}) + out.elementConn.encoding = {'dtype': np.int32} + + out['numElementConn'] = xr.DataArray(4*np.ones(center_lon.size, dtype=np.int32), + dims=('elementCount'), + attrs={'long_name': 'Number of nodes per element'}) + + out['centerCoords'] = xr.DataArray(da.stack([center_lon.T.reshape((-1,)).T, + center_lat.T.reshape((-1,)).T], axis=1), + dims=('elementCount', 'coordDim'), + attrs={'units': 'degrees'}) + + # add area if it is available + if area: + out['elementArea'] = xr.DataArray(area.T.reshape((-1,)).T, + dims=('elementCount'), + attrs={'units': 'radians^2', + 'long_name': 'area weights'}) + + # add mask + out['elementMask'] = xr.DataArray(mask.T.reshape((-1,)).T, + dims=('elementCount'), + attrs={'units': 'unitless'}) + #out.elementMask.encoding = {'dtype': np.int32} + + # force no '_FillValue' if not specified + for v in out.variables: + if '_FillValue' not in out[v].encoding: + out[v].encoding['_FillValue'] = None + + # add global attributes + out.attrs = {'title': 'ESMF unstructured grid file for rectangular grid with {} dimension'.format('x'.join(list(map(str,center_lat.shape)))), + 'created_by': os.path.basename(__file__), + 'date_created': '{}'.format(datetime.now()), + 'conventions': 'ESMFMESH', + } + + # write output file + if filename is not None: + print('Writing {} ...'.format(filename)) + out.to_netcdf(filename) + +def write_to_scrip(filename, center_lat, center_lon, corner_lat, corner_lon, mask, area=None): + """ + dask array doesn't support order='F' for Fortran-contiguous (row-major) order + the workaround is to arr.T.reshape.T + """ + # create new dataset for output + out = xr.Dataset() + + out['grid_dims'] = xr.DataArray(np.array(center_lat.shape, dtype=np.int32), + dims=('grid_rank',)) + out.grid_dims.encoding = {'dtype': np.int32} + + out['grid_center_lat'] = xr.DataArray(center_lat.T.reshape((-1,)).T, + dims=('grid_size'), + attrs={'units': 'degrees'}) + + out['grid_center_lon'] = xr.DataArray(center_lon.T.reshape((-1,)).T, + dims=('grid_size'), + attrs={'units': 'degrees'}) + + out['grid_corner_lat'] = xr.DataArray(corner_lat.T.reshape((4, -1)).T, + dims=('grid_size','grid_corners'), + attrs={'units': 'degrees'}) + + out['grid_corner_lon'] = xr.DataArray(corner_lon.T.reshape((4, -1)).T, + dims=('grid_size','grid_corners'), + attrs={'units': 'degrees'}) + + # include area if it is available + if area: + out['grid_area'] = xr.DataArray(area.T.reshape((-1,)).T, + dims=('grid_size'), + attrs={'units': 'radians^2', + 'long_name': 'area weights'}) + + out['grid_imask'] = xr.DataArray(mask.T.reshape((-1,)).T, + dims=('grid_size'), + attrs={'units': 'unitless'}) + out.grid_imask.encoding = {'dtype': np.int32} + + # force no '_FillValue' if not specified + for v in out.variables: + if '_FillValue' not in out[v].encoding: + out[v].encoding['_FillValue'] = None + + # add global attributes + out.attrs = {'title': 'Rectangular grid with {} dimension'.format('x'.join(list(map(str,center_lat.shape)))), + 'created_by': os.path.basename(__file__), + 'date_created': '{}'.format(datetime.now()), + 'conventions': 'SCRIP', + } + + # write output file + if filename is not None: + print('Writing {} ...'.format(filename)) + out.to_netcdf(filename) + + +def file_type(x): + if x.lower() == 'scrip' or x.lower() == 'esmf': + return x + else: + raise argparse.ArgumentTypeError('SCRIP or ESMF value expected for output type.') + +#@profile +def main(argv): + """Main driver to write SCRIP and ESMF formatted grid represenation + """ + # set defaults for command line arguments + ifile = '' + ofile = '' + oformat = 'ESMF' + overwrite = False + flip = False + latrev = False + latvar = 'lat' + lonvar = 'lon' + maskvar = 'mask' + addarea = False + double = False + + # read command line arguments + parser = argparse.ArgumentParser() + parser.add_argument('--ifile' , help='Input grid file name', required=True) + parser.add_argument('--ofile' , help='Output file name', required=True) + parser.add_argument('--oformat' , help='Output data format [SCRIP, ESMF], defaults to ESMF', required=False, type=file_type, nargs='?', const='ESMF') + parser.add_argument('--overwrite', help='Overwrites output file, defaults to not', required=False, action='store_true') + parser.add_argument('--flip' , help='Flip mask values. SCRIP requires 0 for land and 1 for ocean', required=False, action='store_true') + parser.add_argument('--latrev' , help='Reverse latitude axis', required=False, action='store_true') + parser.add_argument('--latvar' , help='Name of latitude variable, defults to ''lat''', required=False, nargs='?', const='lat') + parser.add_argument('--lonvar' , help='Name of longitude variable, defaults to ''lon''', nargs='?', const='lon') + parser.add_argument('--maskvar' , help='Name of mask variable, defaults to ''mask''', nargs='?', const='mask') + parser.add_argument('--addarea' , help='Add area field to output file, defaults to not', required=False, action='store_true') + parser.add_argument('--double' , help='Double precision output, defaults to float', required=False, action='store_true') + args = parser.parse_args() + + if args.ifile: + ifile = args.ifile + if args.ofile: + ofile = args.ofile + if args.oformat: + oformat = args.oformat + if args.overwrite: + overwrite = args.overwrite + if args.flip: + flip = args.flip + if args.latrev: + latrev = args.latrev + if args.latvar: + latvar = args.latvar + if args.lonvar: + lonvar = args.lonvar + if args.maskvar: + maskvar = args.maskvar + if args.addarea: + addarea = args.addarea + if args.double: + double = args.double + + # print out configuration + print("Configuration:") + print("ifile = {}".format(ifile)) + print("ofile = {}".format(ofile)) + print("oformat = {}".format(oformat)) + print("overwrite = {}".format(overwrite)) + print("flip = {}".format(flip)) + print("latrev = {}".format(latrev)) + print("latvar = {}".format(latvar)) + print("lonvar = {}".format(lonvar)) + print("maskvar = {}".format(maskvar)) + print("addarea = {}".format(addarea)) + print("double = {}".format(double)) + + # open file, transpose() fixes dimension ordering and mimic Fortran + if os.path.isfile(ifile): + ds = xr.open_dataset(ifile, mask_and_scale=False, decode_times=False).transpose() + else: + print('Input file could not find!') + sys.exit(2) + + # check output file + if overwrite: + if os.path.isfile(ofile): + print('Removing existing output file {}.'.format(ofile)) + os.remove(ofile) + else: + if os.path.isfile(ofile): + print('Output file exists. Please provide --overwrite flag.') + sys.exit(2) + + # check coordinate variables + if latvar not in ds.coords and latvar not in ds.data_vars: + print('Input file does not have variable named {}.'.format(latvar)) + print('File has following {}'.format(ds.coords)) + print('File has following {}'.format(ds.data_vars)) + sys.exit(2) + + if lonvar not in ds.coords and lonvar not in ds.data_vars: + print('Input file does not have variable named {}.'.format(latvar)) + print('File has following {}'.format(ds.coords)) + print('File has following {}'.format(ds.data_vars)) + sys.exit(2) + + # remove time dimension from coordinate variables + hasTime = 'time' in ds[latvar].dims + if hasTime: + lat = ds[latvar][:,:,0] + else: + lat = ds[latvar] + + hasTime = 'time' in ds[lonvar].dims + if hasTime: + lon = ds[lonvar][:,:,0] + else: + lon = ds[lonvar] + + # reverse latitude dimension + if latrev: + lat_name = [x for x in lat.coords.dims if 'lat' in x] + if lat_name: + lat = lat.reindex({lat_name[0]: list(reversed(lat[lat_name[0]]))}) + + # remove time dimension from mask variable and optionally flip mask values + # this will also create artifical mask variable with all ones, if it is required + if maskvar in ds.data_vars: + print('Using mask values from the file.') + + # check mask has time dimension or not + hasTime = 'time' in ds[maskvar].dims + if hasTime: + mask = da.from_array(ds[maskvar][:,:,0]) + else: + mask = da.from_array(ds[maskvar]) + else: + print('Using artifical generated mask values, that are ones in everywhere.') + if len(lat.dims) == 1: + mask = da.from_array(np.ones((next(iter(lon.sizes.values())), next(iter(lat.sizes.values()))), dtype=np.int8)) + else: + mask = da.from_array(np.ones(tuple(lat.sizes.values()), dtype=np.int8)) + + # flip mask values + if flip: + print('Flipping mask values to 0 for land and 1 for ocean') + mask = xr.where(mask > 0, 0, 1) + + # calculate corner coordinates, center coordinates are converted to 2d if it is 1d + if double: + center_lat, center_lon, corner_lat, corner_lon = calculate_corners(lat.astype(np.float64, copy=False), lon.astype(np.float64, copy=False)) + else: + center_lat, center_lon, corner_lat, corner_lon = calculate_corners(lat, lon) + + # TODO: add support to calculate area + if addarea: + print('The area calculation is not supported! --addarea is reserved for future use.') + + # create output file + if oformat.lower() == 'scrip': + write_to_scrip(ofile, center_lat, center_lon, corner_lat, corner_lon, mask) + else: + write_to_esmf_mesh(ofile, center_lat, center_lon, corner_lat, corner_lon, mask) + +if __name__== "__main__": + main(sys.argv[1:]) From 37fdf222aafd4493bb5f6231e4fb4a01c157c801 Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 22 Mar 2021 16:09:09 -0500 Subject: [PATCH 19/73] Update to the latest version of the feature/hafs_couplehycom_cdeps branch of ufs-weather-model --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index ee0f23fb9..48b24db38 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit ee0f23fb93b601bc58c66e92982317a8c00f4a7d +Subproject commit 48b24db383f68edc8cd78afa6b0ac11aeab54306 From f95c961c59edb660d16fddb8140a6cbc1abcfdca Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 22 Mar 2021 16:34:37 -0500 Subject: [PATCH 20/73] Add DATM input (ERA5) download job. Rename ush/hafs_era5_downloader.py -> ush/hafs_datm_download.py --- jobs/JHAFS_DATM_DOWNLOAD | 78 +++++++++++++++++++ modulefiles/modulefile.hafs.run.orion | 2 + parm/hafs.conf | 4 + parm/hafs_basic.conf | 1 + parm/hafs_cdeps.conf | 5 +- parm/hafs_holdvars.txt | 3 + rocoto/hafs_workflow.xml.in | 46 ++++++++++- scripts/exhafs_datm_download.sh | 44 +++++++++++ scripts/exhafs_datm_mesh.sh | 4 +- scripts/exhafs_forecast.sh | 13 +++- sorc/build_forecast.sh | 2 +- sorc/hafs_forecast.fd | 2 +- ush/hafs/launcher.py | 9 +++ ...a5_downloader.py => hafs_datm_download.py} | 14 +++- 14 files changed, 215 insertions(+), 12 deletions(-) create mode 100755 jobs/JHAFS_DATM_DOWNLOAD create mode 100755 scripts/exhafs_datm_download.sh rename ush/{hafs_era5_downloader.py => hafs_datm_download.py} (96%) diff --git a/jobs/JHAFS_DATM_DOWNLOAD b/jobs/JHAFS_DATM_DOWNLOAD new file mode 100755 index 000000000..1d1a251be --- /dev/null +++ b/jobs/JHAFS_DATM_DOWNLOAD @@ -0,0 +1,78 @@ +#!/bin/sh + +set -xe + +date +source ${USHhafs}/hafs_pre_job.sh.inc +HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} +source ${HOLDVARS} + +export EXPT=${EXPT:-HAFS} +export SUBEXPT=${SUBEXPT:-${EXPT}} + +export CDATE=${CDATE:-${YMDH}} +export cyc=${cyc:-00} +export STORM=${STORM:-FAKE} +export STORMID=${STORMID:-00L} + +#################################### +# Specify Execution Areas +#################################### +export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +export USHhafs=${USHhafs:-${HOMEhafs}/ush} + +#################################### +# Load Modules if Needed +#################################### +export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet +#export RUN_ENVIR=${RUN_ENVIR:-prod} +export RUN_ENVIR=${envir:-prod} +if [ "$RUN_ENVIR" = prod ]; then + #module use ${HOMEhafs}/modulefiles + #module load modulefile.hafs.run.${machine} + if [ "$machine" = wcoss_dell_p3 ]; then + module unload NetCDF + module unload HDF5-serial + fi + #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} + module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles + module load build.${machine}.intel + module list +fi + +##################################### +# Set up job node/core/run environment variables +##################################### +source ${USHhafs}/hafs_runcmd.sh.inc + +#################################### +# Run setpdy and initialize PDY variables +#################################### +#setpdy.sh +#. ./PDY + +PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) +export PDY=${PDY:-${PDYtmp}} + +##################################### +# Working directories +##################################### +export DATA=${WORKhafs}/datm_mesh + +export SCRUBDATA=${SCRUBDATA:-YES} + +if [ "${SCRUBDATA}" = YES ]; then + rm -rf $DATA +fi + +mkdir -p $DATA +cd $DATA + +##################################### +# Execute the script. +${HOMEhafs}/scripts/exhafs_datm_download.sh +export err=$? +##################################### + +exit $err diff --git a/modulefiles/modulefile.hafs.run.orion b/modulefiles/modulefile.hafs.run.orion index ab4bc3c36..ce8a340b7 100644 --- a/modulefiles/modulefile.hafs.run.orion +++ b/modulefiles/modulefile.hafs.run.orion @@ -62,3 +62,5 @@ setenv CMAKE_Platform orion.intel module load rocoto/1.3.3 module load intelpython3/2020 + +module load cdo \ No newline at end of file diff --git a/parm/hafs.conf b/parm/hafs.conf index 6301c065c..99ff337af 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -286,6 +286,9 @@ COMgfs={dir/COMgfs} ;; input GFS com directory COMrtofs={dir/COMrtofs} ;; input RTOFS com directory gtype={grid/gtype} ;; grid type: uniform, stretch, nest, or regional (currently only nest and regional have been tested and supported) GFSVER={config/GFSVER} ;; Version of GFS input data, e.g., PROD2019, PROD2021 +DOWNLOAD_DATM_LIST={download_datm_list} ;; List of task ids for era5 downloader +DOWNLOAD_DATM_THRESHOLD={download_datm_threshold} ;; 1.0/(length of download_datm_list) + # Specify the forecast job resources. Only a few combinations are provided. If # needed, you may add other options in the site entity files under rocoto/sites. #FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2 @@ -305,3 +308,4 @@ RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? +DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM ERA5 files? \ No newline at end of file diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index b00d335bf..011e15c19 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -38,6 +38,7 @@ run_dwav=no ;; Data waves using CDEPS run_hrdgraphics=no ;; Run HRD graphics run_emcgraphics=no ;; Run EMC graphics make_mesh_atm=no ;; Generate DATM mesh in workflow (only for run_datm=yes) +datm_download_jobs=0 ;; Number of download_datm jobs (0=disable) # Scrub options scrub=no ;; do we delete temporary files? (placeholder) diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index 840299923..4c3efa123 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -3,12 +3,13 @@ run_datm=yes ;; Data atmosphere using CDEPS run_docn=no ;; Data ocean using CDEPS run_dwav=no ;; Data waves using CDEPS make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen +datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing [dir] -DATMdir=/work/noaa/dtc-hwrf/ekalina/RUN_test/input +DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM_ERA5 [forecast] -mesh_atm_in={DATMdir}/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change # mesh_atm=... ;; This variable is now set automatically by ush/hafs/launcher.py diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 67b4d069d..200808f52 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -3,6 +3,8 @@ # can also be used for debugging: simply source the # storm*.holdvars.txt in a ksh/sh/bash shell. +export TZ=UTC # Orion workaround + export envir={ENV[envir|-prod]} export storm_num={storm_num} @@ -37,6 +39,7 @@ export run_datm={run_datm} export run_docn={run_docn} export run_dwav={run_dwav} +export datm_download_jobs={datm_download_jobs} export make_mesh_atm={make_mesh_atm} export mesh_atm={mesh_atm} diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 4faa5702a..e14c57846 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -70,7 +70,11 @@ - + + + + + @@ -189,6 +193,38 @@ +@** if RUN_DATM==YES + + &DOWNLOAD_DATM_LIST; + + &JOBhafs;/JHAFS_DATM_DOWNLOAD + hafs_input_&SID;_@Y@m@d@H_#idx# + &WORKhafs;/hafs_download_datm_#idx#.log + &ACCOUNT; + &RESERVATION; + &QUEUE_SERVICE; + &SERVICE_EXTRA; + 1 + &CORES_EXTRA; + TOTAL_TASKS1 + 06:00:00 + &MEMORY; + &ENV_VARS; + + + + + &DOWNLOAD_DATM;YES + &RUN_DATM;YES + + + + + + + +@** endif + &JOBhafs;/JHAFS_GRID hafs_grid_&SID;_@Y@m@d@H @@ -383,6 +419,10 @@ &MAKE_MESH_ATM;YES + + &DOWNLOAD_DATM;YES + + @@ -420,6 +460,10 @@ @** if RUN_DATM==YES + + &DOWNLOAD_DATM;YES + + &MAKE_MESH_ATM;YES diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh new file mode 100755 index 000000000..04cf0a19e --- /dev/null +++ b/scripts/exhafs_datm_download.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +if [[ "$run_datm" != yes || ! "$datm_download_jobs" -gt 0 ]] ; then + echo "This job should only be run if \$run_datm=yes and \$datm_download_jobs>0" + echo " \$datm_download_jobs=\"$datm_download_jobs\"" + echo " \$run_datm=\"$run_datm\"" + echo "Billions of electrons have whizzed by, the universe's entropy has increased, and yet nothing has been accomplished." + echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " + exit 0 +fi + +if ( ! which cdo ) ; then + echo "The \"cdo\" program is not in your \$PATH. This script uses cdo to" 1>&2 + echo "reverse latitudes. Go get cdo." 1>&2 + echo " -> SCRIPT IS EXITING BECAUSE cdo IS NOT IN \$PATH <- " 1>&2 + exit 1 +fi + +set -uxe + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} +NHRS=${NHRS:?NHRS is not set} +DATMdir=${DATMdir:?DATMdir is not set} + +export TZ=UTC # Orion workaround + +EDATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d%H ) + +[ -d "$DATMdir" ] || mkdir "$DATMdir" +test -d "$DATMdir" +cd "$DATMdir" + +echo "Downloading ERA5 files and reversing latitudes." +echo "Start date: ${CDATE:0:8}" +echo "End date: ${EDATE:0:8}" +echo "Destination dir: \"$DATMdir\"" + +"$USHhafs/hafs_datm_download.py" ${CDATE:0:8}-${EDATE:0:8} + +echo "Successfully downloaded all ERA5 files and reversed their latitudes." +echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_datm_mesh.sh b/scripts/exhafs_datm_mesh.sh index d0fda6ced..f976a9f12 100755 --- a/scripts/exhafs_datm_mesh.sh +++ b/scripts/exhafs_datm_mesh.sh @@ -24,9 +24,9 @@ mesh_dir=$( dirname "$mesh_atm" ) # Pick an input file. This will get the newest one, if the names are # identical except for YYYYMMDD date string: test -d "$DATMdir" -ifile=$( ls -1 "$DATMdir"/*nc | sort -ur | head -1 ) +#ifile=$( ls -1 "$DATMdir"/*nc | sort -ur | head -1 ) +ifile="$DATMdir/ERA5_${CDATE:0:8}.nc" -test -n "$ifile" test -s "$ifile" test -r "$ifile" diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 81bd100cb..20e09af36 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -447,7 +447,15 @@ cat temp diag_table.tmp > diag_table else - ln -sf ${DATMdir}/*.nc INPUT/ + enddate=`${NDATE} +${NHRS} $CDATE` + endyr=`echo $enddate | cut -c1-4` + + nowdate=$CDATE + while (( nowdate <= enddate )) ; do + era5_name=ERA5_${nowdate:0:8}.nc + ln -sf $DATMdir/$era5_name INPUT/$era5_name + nowdate=`${NDATE} +6 $nowdate` + done #---------------------------------------------- # Copy CDEPS parm files if required. @@ -460,9 +468,6 @@ else sed -i "s/_yearFirst_/$yr/g" datm.streams.xml - enddate=`${NDATE} +${NHRS} $CDATE` - endyr=`echo $enddate | cut -c1-4` - sed -i "s/_yearLast_/$endyr/g" datm.streams.xml sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams.xml diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index dee5441a8..bafa9c8d2 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -23,5 +23,5 @@ if [ $target = jet ]; then target=jet.intel ; fi cd hafs_forecast.fd/ cd tests/ -./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y" 32bit YES NO +./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y CDEPS_DATM=Y CDEPS_DOCN=Y CDEPS=Y" 32bit YES NO diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 48b24db38..ee0f23fb9 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 48b24db383f68edc8cd78afa6b0ac11aeab54306 +Subproject commit ee0f23fb93b601bc58c66e92982317a8c00f4a7d diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 624163cf8..34e6e27a9 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -813,8 +813,17 @@ def set_data_model_variables(self): self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_gen')) else: self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_in')) + datm_download_jobs=max(0,self.getint('config','datm_download_jobs',0)) + download_datm_list=" ".join(["%02d"%x for x in range(max(1,datm_download_jobs))]) + download_datm_threshold="%7.5f"%(1.0/(max(1,datm_download_jobs)),) + self.set('config','download_datm',datm_download_jobs>0) + self.set('config','download_datm_list',download_datm_list) + self.set('config','download_datm_threshold',download_datm_threshold) else: self.set('forecast','mesh_atm','dummy.nc') + self.set('config','download_datm',False) + self.set('config','download_datm_list','0') + self.set('config','download_datm_threshold','1.0') def set_storm(self,syndat,oldsyndat): """!Sets the storm that is to be run. diff --git a/ush/hafs_era5_downloader.py b/ush/hafs_datm_download.py similarity index 96% rename from ush/hafs_era5_downloader.py rename to ush/hafs_datm_download.py index 4ce88efd0..dc9a0dd45 100755 --- a/ush/hafs_era5_downloader.py +++ b/ush/hafs_datm_download.py @@ -13,7 +13,19 @@ import logging import datetime import sys -import cdsapi + +try: + import cdsapi +except ImportError as ie: + sys.stderr.write("""You are missing the cdsapi module! +You must install it to run this script. + + pip install cdsapi --user + +You will also need to register on the cdsapi website, sign the ERA5 +license agreement, get a key, and put the key in your ~/.cdsapi file. +""") + import produtil.setup, produtil.fileop, produtil.locking # Constants From 37516b38f8662125b727ea27a90deb69821287d4 Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 23 Mar 2021 16:26:21 -0500 Subject: [PATCH 21/73] Sync with the latest ufs-weather-model feature/hafs_couplehycom_cdeps branch. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index ee0f23fb9..9ff89f518 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit ee0f23fb93b601bc58c66e92982317a8c00f4a7d +Subproject commit 9ff89f518b1cf35df1f87ddecda621434a45e32e From 3e21e034984250449e11182d7cd28908ab41c463 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 26 Mar 2021 09:34:34 -0500 Subject: [PATCH 22/73] Add the capability to run CDEPS DOCN to the workflow. --- parm/cdeps/docn.streams.xml | 24 ++++ parm/cdeps/docn_in | 10 ++ .../regional/nems.configure.cdeps.tmp | 111 ++++++++++++++++++ .../regional/nems.configure.datm_ocn.tmp | 109 ----------------- parm/hafs.conf | 3 +- parm/hafs_cdeps.conf | 37 ++++-- parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 5 + scripts/exhafs_forecast.sh | 84 ++++++++++--- ush/hafs/launcher.py | 44 ++++++- 10 files changed, 286 insertions(+), 142 deletions(-) create mode 100644 parm/cdeps/docn.streams.xml create mode 100644 parm/cdeps/docn_in create mode 100644 parm/forecast/regional/nems.configure.cdeps.tmp delete mode 100755 parm/forecast/regional/nems.configure.datm_ocn.tmp diff --git a/parm/cdeps/docn.streams.xml b/parm/cdeps/docn.streams.xml new file mode 100644 index 000000000..c832fd0ca --- /dev/null +++ b/parm/cdeps/docn.streams.xml @@ -0,0 +1,24 @@ + + + + + cycle + linear + single + bilinear + 1.5 + _yearFirst_ + _yearLast_ + 1 + null + _mesh_ocn_ + null + + + + sst So_t + + 0 + + + diff --git a/parm/cdeps/docn_in b/parm/cdeps/docn_in new file mode 100644 index 000000000..0572fe2d4 --- /dev/null +++ b/parm/cdeps/docn_in @@ -0,0 +1,10 @@ +&docn_nml + datamode = "sstdata" + model_createmesh_fromfile = "null" + model_maskfile = "_mesh_ocn_" + model_meshfile = "_mesh_ocn_" + nx_global = 1440 + ny_global = 720 + restfilm = "null" + sst_constant_value = -1.0 +/ diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/nems.configure.cdeps.tmp new file mode 100644 index 000000000..cbe0ab82f --- /dev/null +++ b/parm/forecast/regional/nems.configure.cdeps.tmp @@ -0,0 +1,111 @@ +############################################## +##### NEMS Run-Time Configuration File ##### +############################################## + +# EARTH # +EARTH_component_list: MED ATM OCN +EARTH_attributes:: + Verbosity = 0 +:: + +# MED # +MED_model: nems +_MED_petlist_bounds_ +:: + +# ATM # +ATM_model: _atm_model_ +_ATM_petlist_bounds_ +ATM_attributes:: + mesh_atm = _mesh_atm_ + Verbosity = 1 + Diagnostic = 0 +:: + +# OCN # +OCN_model: _ocn_model_ +_OCN_petlist_bounds_ +OCN_attributes:: + mesh_ocn = _mesh_ocn_ + Verbosity = 1 + Diagnostic = 0 + cdf_impexp_freq = 3 + cpl_hour = 0 + cpl_min = 0 + cpl_sec = _cpl_dt_ + base_dtg = _base_dtg_ + merge_import = _merge_import_ + skip_first_import = .true. + hycom_arche_output = .true. + hyc_esmf_exp_output = .true. + hyc_esmf_imp_output = .true. + hyc_impexp_file = dummy_file + espc_show_impexp_minmax = .true. + ocean_start_dtg = _ocean_start_dtg_ + start_hour = 0 + start_min = 0 + start_sec = 0 + end_hour = _end_hour_ + end_min = 0 + end_sec = 0 +:: + +# Run Sequence # +runSeq:: +@_cpl_dt_ +_runSeq_ALL_ +@ +:: + +# Other Attributes # +DRIVER_attributes:: + mediator_read_restart = false +:: + +MED_attributes:: + ATM_model = _atm_model_ + OCN_model = _ocn_model_ + Verbosity = 1 + Diagnostic = 0 + history_n = 1 + history_option = nhours + history_ymd = -999 + coupling_mode = hafs + system_type = ufs +:: + +ALLCOMP_attributes:: + ScalarFieldCount = 3 + ScalarFieldIdxGridNX = 1 + ScalarFieldIdxGridNY = 2 + ScalarFieldIdxNextSwCday = 3 + ScalarFieldName = cpl_scalars + start_type = startup + case_name = ufs.hafs + restart_n = 24 + restart_option = nhours + restart_ymd = -999 + normalization = none + merge_type = copy + orb_eccen = 1.e36 + orb_iyear = 2000 + orb_iyear_align = 2000 + orb_mode = fixed_year + orb_mvelp = 1.e36 + orb_obliq = 1.e36 +:: + +ATM_modelio:: + diro = . + logfile = atm.log +:: + +OCN_modelio:: + diro = . + logfile = ocn.log +:: + +MED_modelio:: + diro = . + logfile = nems.log +:: diff --git a/parm/forecast/regional/nems.configure.datm_ocn.tmp b/parm/forecast/regional/nems.configure.datm_ocn.tmp deleted file mode 100755 index 206c6504e..000000000 --- a/parm/forecast/regional/nems.configure.datm_ocn.tmp +++ /dev/null @@ -1,109 +0,0 @@ -############################################## -##### NEMS Run-Time Configuration File ##### -############################################## - -# EARTH # -EARTH_component_list: MED ATM OCN -EARTH_attributes:: - Verbosity = 0 -:: - -# MED # -MED_model: nems -_MED_petlist_bounds_ -:: - -# ATM # -ATM_model: datm -_ATM_petlist_bounds_ -ATM_attributes:: - Verbosity = low - mesh_atm = _mesh_atm_ -:: - -# OCN # -OCN_model: hycom -_OCN_petlist_bounds_ -OCN_attributes:: - Verbosity = 1 - Diagnostic = 0 - cdf_impexp_freq = 3 - cpl_hour = 0 - cpl_min = 0 - cpl_sec = _cpl_dt_ - base_dtg = _base_dtg_ - merge_import = _merge_import_ - skip_first_import = .true. - hycom_arche_output = .true. - hyc_esmf_exp_output = .true. - hyc_esmf_imp_output = .true. - hyc_impexp_file = dummy_file - espc_show_impexp_minmax = .true. - ocean_start_dtg = _ocean_start_dtg_ - start_hour = 0 - start_min = 0 - start_sec = 0 - end_hour = _end_hour_ - end_min = 0 - end_sec = 0 -:: - -# Run Sequence # -runSeq:: -@_cpl_dt_ - MED med_phases_prep_ocn_avg - MED -> OCN :remapMethod=redist - OCN - ATM - ATM -> MED :remapMethod=redist - MED med_phases_post_atm - MED med_phases_aofluxes_run - MED med_phases_prep_ocn_accum - OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true - MED med_phases_post_ocn - MED med_phases_history_write - MED med_phases_restart_write -@ -:: - -# Other Attributes # -DRIVER_attributes:: - mediator_read_restart = false -:: - -MED_attributes:: - ATM_model = datm - OCN_model = hycom - history_n = 1 - history_option = nhours - history_ymd = -999 - coupling_mode = hafs -:: - -ALLCOMP_attributes:: - ATM_model = datm - OCN_model = hycom - ScalarFieldCount = 3 - ScalarFieldIdxGridNX = 1 - ScalarFieldIdxGridNY = 2 - ScalarFieldIdxNextSwCday = 3 - ScalarFieldName = cpl_scalars - start_type = startup - case_name = ufs.hafs - restart_n = 48 - restart_option = nhours - restart_ymd = -999 - dbug_flag = 0 - use_coldstart = true - orb_eccen = 1.e36 - orb_iyear = 2000 - orb_iyear_align = 2000 - orb_mode = fixed_year - orb_mvelp = 1.e36 - orb_obliq = 1.e36 -:: - -ATM_modelio:: - diro = ./ - logfile = atm.log -:: diff --git a/parm/hafs.conf b/parm/hafs.conf index 99ff337af..8ffff4a50 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -40,6 +40,7 @@ oldsid={oldvit[stormid3]} oldcom={CDSCRUB}/{RUNhafs}/com/{oldvit[YMDH]}/{oldvit[stormid3]} intercom={WORKhafs}/intercom ;; dir for communicating data files between jobs DATMdir=dummy ;; input dir for cdeps datm +DOCNdir=dummy ;; input dir for cdeps docn outatcf={CDNOSCRUB}/{SUBEXPT} ;; Delivery location for ATCF files outdiag={CDNOSCRUB}/diagtrak/{SUBEXPT} ;; Delivery location for wrfdiag files outstatus={CDNOSCRUB}/cycstatus/{SUBEXPT} ;; Delivery location for status files @@ -308,4 +309,4 @@ RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? -DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM ERA5 files? \ No newline at end of file +DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM ERA5 files? diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index 4c3efa123..c5cc80bda 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -1,23 +1,40 @@ [config] -run_datm=yes ;; Data atmosphere using CDEPS -run_docn=no ;; Data ocean using CDEPS -run_dwav=no ;; Data waves using CDEPS +run_datm=no ;; Data atmosphere using CDEPS +run_docn=yes ;; Data ocean using CDEPS +run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. +run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen +make_mesh_ocn=no ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing +docn_download_jobs=0 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing + +scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn +scrub_work=no [dir] DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM_ERA5 +DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN [forecast] mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change -# mesh_atm=... ;; This variable is now set automatically by ush/hafs/launcher.py +mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change + +# DATM +#layoutx=10 +#layouty=8 +#write_groups=1 +#write_tasks_per_group=40 +#ocean_tasks=120 -layoutx=10 -layouty=8 -write_groups=1 -write_tasks_per_group=40 -ocean_tasks=120 +# DOCN +ocean_tasks=60 [rocotostr] -FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 + +# DATM +#FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 + +# DOCN +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp2 diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 7a3e4e47f..1a4c2ba58 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -91,3 +91,4 @@ ocean_start_dtg={forecast/ocean_start_dtg} merge_import={forecast/merge_import} mesh_atm={forecast/mesh_atm} +mesh_ocn={forecast/mesh_ocn} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 200808f52..37305e897 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -34,6 +34,7 @@ export COMIN={COMIN} export COMOUT={COMOUT} export COMgfs={COMgfs} export DATMdir={DATMdir} +export DOCNdir={DOCNdir} export run_datm={run_datm} export run_docn={run_docn} @@ -43,6 +44,10 @@ export datm_download_jobs={datm_download_jobs} export make_mesh_atm={make_mesh_atm} export mesh_atm={mesh_atm} +export docn_download_jobs={docn_download_jobs} +export make_mesh_ocn={make_mesh_ocn} +export mesh_ocn={mesh_ocn} + export SYNDAThafs={syndat} export ADECKhafs={ADECKhafs} export BDECKhafs={BDECKhafs} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 20e09af36..1cc0ef243 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -56,7 +56,9 @@ export ccpp_suite_glob=${ccpp_suite_glob:-HAFS_v0_gfdlmp} export ccpp_suite_nest=${ccpp_suite_nest:-HAFS_v0_gfdlmp_nocp} export run_datm=${run_datm:-no} +export run_docn=${run_docn:-no} export mesh_atm=${mesh_atm:-''} +export mesh_ocn=${mesh_ocn:-''} export run_ocean=${run_ocean:-no} export ocean_model=${ocean_model:-hycom} @@ -112,11 +114,17 @@ if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ] && [ ${run_datm} = no ]; then export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) export runSeq_ALL="ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n OCN -> MED :remapMethod=redist\n MED med_phases_post_ocn\n MED med_phases_prep_atm\n MED med_phases_prep_ocn_accum\n MED med_phases_prep_ocn_avg\n MED -> ATM :remapMethod=redist\n MED -> OCN :remapMethod=redist\n ATM\n OCN" fi -# CMEPS based coupling between HYCOM and DATM through the bilinear regridding method -if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ] && [ ${run_datm} = yes ]; then +# CDEPS data models +if [ ${run_datm} == yes ]; then export cplflx=.true. export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" 0 $(($ATM_tasks-1))) + export runSeq_ALL="MED med_phases_prep_ocn_avg\n MED -> OCN :remapMethod=redist\n OCN\n ATM\n ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n MED med_phases_aofluxes_run\n MED med_phases_prep_ocn_accum\n OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true\n MED med_phases_post_ocn\n MED med_phases_history_write\n MED med_phases_restart_write" +elif [ ${run_docn} = yes ]; then + export cplflx=.true. + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export runSeq_ALL="MED med_phases_prep_atm\n MED -> ATM :remapMethod=redist\n ATM\n OCN\n ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n OCN -> MED :remapMethod=redist\n MED med_phases_post_ocn\n MED med_phases_restart_write\n MED med_phases_history_write" fi export ocean_start_dtg=${ocean_start_dtg:-43340.00000} @@ -432,7 +440,7 @@ fi fi #------------------------------------------------------------------- -# Generate diag_table, model_configure from their tempelates +# Generate diag_table, model_configure from their templates #------------------------------------------------------------------- echo ${yr}${mn}${dy}.${cyc}Z.${CASE}.32bit.non-hydro echo $yr $mn $dy $cyc 0 0 @@ -441,29 +449,30 @@ ${yr}${mn}${dy}.${cyc}Z.${CASE}.32bit.non-hydro $yr $mn $dy $cyc 0 0 EOF -if [ ${run_datm} = no ]; then +enddate=`${NDATE} +${NHRS} $CDATE` +endyr=`echo $enddate | cut -c1-4` +if [ ${run_datm} = no ]; then cat temp diag_table.tmp > diag_table +fi -else +if [ ${run_datm} = yes ]; then - enddate=`${NDATE} +${NHRS} $CDATE` - endyr=`echo $enddate | cut -c1-4` +#---------------------------------------------- +# Copy CDEPS parm files if required. +#---------------------------------------------- + cp ${PARMforecast}/model_configure.tmp . + cp ${PARMhafs}/cdeps/datm_in . + cp ${PARMhafs}/cdeps/datm.streams.xml . nowdate=$CDATE while (( nowdate <= enddate )) ; do era5_name=ERA5_${nowdate:0:8}.nc ln -sf $DATMdir/$era5_name INPUT/$era5_name + sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml nowdate=`${NDATE} +6 $nowdate` done -#---------------------------------------------- -# Copy CDEPS parm files if required. -#---------------------------------------------- - cp ${PARMforecast}/model_configure.tmp . - cp ${PARMhafs}/cdeps/datm_in . - cp ${PARMhafs}/cdeps/datm.streams.xml . - sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in sed -i "s/_yearFirst_/$yr/g" datm.streams.xml @@ -471,23 +480,60 @@ else sed -i "s/_yearLast_/$endyr/g" datm.streams.xml sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams.xml - for file in `ls INPUT/*.nc ` ; do - sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" datm.streams.xml - done ln -sf ${mesh_atm} INPUT/ - cp ${PARMforecast}/nems.configure.datm_ocn.tmp ./ + cp ${PARMforecast}/nems.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ -e "s/_cpl_dt_/${cpl_dt}/g" \ + -e "s/_runSeq_ALL_/${runSeq_ALL}/g" \ -e "s/_base_dtg_/${CDATE}/g" \ -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.true.}/g" \ -e "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" \ - nems.configure.datm_ocn.tmp > nems.configure + -e "/_mesh_ocn_/d" \ + -e "s/_atm_model_/datm/g" \ + -e "s/_ocn_model_/hycom/g" \ + nems.configure.cdeps.tmp > nems.configure + +elif [ ${run_docn} = yes ]; then + + ln -sf $DOCNdir/*.nc INPUT/ + + cp ${PARMhafs}/cdeps/docn_in . + cp ${PARMhafs}/cdeps/docn.streams.xml . + + sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn_in + + sed -i "s/_yearFirst_/$yr/g" docn.streams.xml + + sed -i "s/_yearLast_/$endyr/g" docn.streams.xml + + sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams.xml + for file in `ls INPUT/sst*.nc ` ; do + sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" docn.streams.xml + done + + ln -sf ${mesh_ocn} INPUT/ + + cp ${PARMforecast}/nems.configure.cdeps.tmp ./ + sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ + -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ + -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ + -e "s/_cpl_dt_/${cpl_dt}/g" \ + -e "s/_runSeq_ALL_/${runSeq_ALL}/g" \ + -e "s/_base_dtg_/${CDATE}/g" \ + -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ + -e "s/_end_hour_/${NHRS}/g" \ + -e "s/_merge_import_/${merge_import:-.true.}/g" \ + -e "/_mesh_atm_/d" \ + -e "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" \ + -e "s/_atm_model_/fv3/g" \ + -e "s/_ocn_model_/docn/g" \ + nems.configure.cdeps.tmp > nems.configure fi diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 34e6e27a9..4e33d0a0a 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -793,22 +793,43 @@ def sanity_check_data_models(self,logger): """!In the hafs_launcher job, this checks the data model variables and files for obvious errors before starting the rest of the workflow.""" run_datm=self.getbool('config','run_datm',False) + run_docn=self.getbool('config','run_docn',False) + run_ocean=self.getbool('config','run_ocean',False) if run_datm: - make_mesh_atm=self.getbool('config','make_mesh_datm',True) + if run_docn: + msg='run_datm and run_docn cannot both be set to yes' + logger.error(msg) + raise msg + make_mesh_atm=self.getbool('config','make_mesh_atm',True) if not make_mesh_atm: - mesh_atm=timestrinterp('forecast','mesh_atm') + mesh_atm=self.getstr('forecast','mesh_atm') if not os.path.exists(mesh_atm): msg='%s: mesh_atm file does not exist'%(mesh_atm,) logger.error(msg) raise msg else: logger.info("%s: will use this pre-made datm esmf mesh (mesh_atm)."%(mesh_atm,)) + if run_docn: + make_mesh_ocn=self.getbool('config','make_mesh_ocn',True) + if not make_mesh_ocn: + mesh_ocn=self.getstr('forecast','mesh_ocn') + if not os.path.exists(mesh_ocn): + msg='%s: mesh_ocn file does not exist'%(mesh_ocn,) + logger.error(msg) + raise msg + else: + logger.info("%s: will use this pre-made docn esmf mesh (mesh_ocn)."%(mesh_ocn,)) + if run_ocean: + msg='run_ocean=yes and run_docn=yes are incompatible.' + logger.error(msg) + raise msg def set_data_model_variables(self): """!Sets conf variables for the data models.""" run_datm=self.getbool('config','run_datm',False) + run_docn=self.getbool('config','run_docn',False) if run_datm: - make_mesh_atm=self.getbool('config','make_mesh_datm',True) + make_mesh_atm=self.getbool('config','make_mesh_atm',True) if make_mesh_atm: self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_gen')) else: @@ -824,6 +845,23 @@ def set_data_model_variables(self): self.set('config','download_datm',False) self.set('config','download_datm_list','0') self.set('config','download_datm_threshold','1.0') + if run_docn: + make_mesh_ocn=self.getbool('config','make_mesh_ocn',True) + if make_mesh_ocn: + self.set('forecast','mesh_ocn',self.getraw('forecast','mesh_ocn_gen')) + else: + self.set('forecast','mesh_ocn',self.getraw('forecast','mesh_ocn_in')) + docn_download_jobs=max(0,self.getint('config','docn_download_jobs',0)) + download_docn_list=" ".join(["%02d"%x for x in range(max(1,docn_download_jobs))]) + download_docn_threshold="%7.5f"%(1.0/(max(1,docn_download_jobs)),) + self.set('config','download_docn',docn_download_jobs>0) + self.set('config','download_docn_list',download_docn_list) + self.set('config','download_docn_threshold',download_docn_threshold) + else: + self.set('forecast','mesh_ocn','dummy.nc') + self.set('config','download_docn',False) + self.set('config','download_docn_list','0') + self.set('config','download_docn_threshold','1.0') def set_storm(self,syndat,oldsyndat): """!Sets the storm that is to be run. From 04a93ced7ed083f6a647d2000c8bad07152ead79 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 26 Mar 2021 17:02:44 -0500 Subject: [PATCH 23/73] Add comment to nems.configure for CDEPS so that the user isn't confused by the additional options there. --- parm/forecast/regional/nems.configure.cdeps.tmp | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/nems.configure.cdeps.tmp index cbe0ab82f..fc25e2278 100644 --- a/parm/forecast/regional/nems.configure.cdeps.tmp +++ b/parm/forecast/regional/nems.configure.cdeps.tmp @@ -29,6 +29,7 @@ OCN_attributes:: mesh_ocn = _mesh_ocn_ Verbosity = 1 Diagnostic = 0 +# The following are only used by the hycom ocean model. # cdf_impexp_freq = 3 cpl_hour = 0 cpl_min = 0 From 49ecbcb01911929d9763310e0cbbeeb7378b769a Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 30 Mar 2021 16:39:13 -0500 Subject: [PATCH 24/73] .gitignore Rocoto db files and Emacs backup files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 01d0ab170..547a69bba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ *.exe *.x *.log +*~ +*.db +*.db.bak From 0105eecbe4e0c8c592474e0725334e2d10caed7b Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 30 Mar 2021 16:40:20 -0500 Subject: [PATCH 25/73] Download DOCN files and generate their mesh. Fix a bug in hafs launcher data model checks. --- jobs/JHAFS_DOCN_DOWNLOAD | 78 +++++++ jobs/JHAFS_DOCN_MESH | 78 +++++++ modulefiles/modulefile.hafs.run.orion | 4 +- parm/hafs.conf | 4 + parm/hafs_cdeps.conf | 4 +- rocoto/hafs_workflow.xml.in | 108 +++++++-- rocoto/run_hafs.py | 8 +- rocoto/sites/hera.ent | 2 +- rocoto/sites/kjet.ent | 2 +- rocoto/sites/orion.ent | 2 +- rocoto/sites/wcoss_cray.ent | 2 +- rocoto/sites/wcoss_dell_p3.ent | 2 +- rocoto/sites/xjet.ent | 2 +- rocoto/sites/xjet_hafsv0p1a.ent | 2 +- rocoto/sites/xjet_hafsv0p2a.ent | 2 +- scripts/exhafs_datm_mesh.sh | 11 +- scripts/exhafs_docn_download.sh | 37 ++++ scripts/exhafs_docn_mesh.sh | 100 +++++++++ scripts/exhafs_forecast.sh | 15 +- sorc/build_forecast.sh | 2 +- ush/hafs/exceptions.py | 3 + ush/hafs/launcher.py | 28 ++- ush/hafs_datm_download.py | 7 +- ush/hafs_docn_download.py | 217 +++++++++++++++++++ ush/{hafs_datm_mesh.py => hafs_esmf_mesh.py} | 34 ++- 25 files changed, 705 insertions(+), 49 deletions(-) create mode 100755 jobs/JHAFS_DOCN_DOWNLOAD create mode 100755 jobs/JHAFS_DOCN_MESH create mode 100755 scripts/exhafs_docn_download.sh create mode 100755 scripts/exhafs_docn_mesh.sh create mode 100755 ush/hafs_docn_download.py rename ush/{hafs_datm_mesh.py => hafs_esmf_mesh.py} (92%) diff --git a/jobs/JHAFS_DOCN_DOWNLOAD b/jobs/JHAFS_DOCN_DOWNLOAD new file mode 100755 index 000000000..ce6e37563 --- /dev/null +++ b/jobs/JHAFS_DOCN_DOWNLOAD @@ -0,0 +1,78 @@ +#!/bin/sh + +set -xe + +date +source ${USHhafs}/hafs_pre_job.sh.inc +HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} +source ${HOLDVARS} + +export EXPT=${EXPT:-HAFS} +export SUBEXPT=${SUBEXPT:-${EXPT}} + +export CDATE=${CDATE:-${YMDH}} +export cyc=${cyc:-00} +export STORM=${STORM:-FAKE} +export STORMID=${STORMID:-00L} + +#################################### +# Specify Execution Areas +#################################### +export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +export USHhafs=${USHhafs:-${HOMEhafs}/ush} + +#################################### +# Load Modules if Needed +#################################### +export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet +#export RUN_ENVIR=${RUN_ENVIR:-prod} +export RUN_ENVIR=${envir:-prod} +if [ "$RUN_ENVIR" = prod ]; then + #module use ${HOMEhafs}/modulefiles + #module load modulefile.hafs.run.${machine} + if [ "$machine" = wcoss_dell_p3 ]; then + module unload NetCDF + module unload HDF5-serial + fi + #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} + module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles + module load build.${machine}.intel + module list +fi + +##################################### +# Set up job node/core/run environment variables +##################################### +source ${USHhafs}/hafs_runcmd.sh.inc + +#################################### +# Run setpdy and initialize PDY variables +#################################### +#setpdy.sh +#. ./PDY + +PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) +export PDY=${PDY:-${PDYtmp}} + +##################################### +# Working directories +##################################### +export DATA=${WORKhafs}/docn_mesh + +export SCRUBDATA=${SCRUBDATA:-YES} + +if [ "${SCRUBDATA}" = YES ]; then + rm -rf $DATA +fi + +mkdir -p $DATA +cd $DATA + +##################################### +# Execute the script. +${HOMEhafs}/scripts/exhafs_docn_download.sh +export err=$? +##################################### + +exit $err diff --git a/jobs/JHAFS_DOCN_MESH b/jobs/JHAFS_DOCN_MESH new file mode 100755 index 000000000..4ae5052f2 --- /dev/null +++ b/jobs/JHAFS_DOCN_MESH @@ -0,0 +1,78 @@ +#!/bin/sh + +set -xe + +date +source ${USHhafs}/hafs_pre_job.sh.inc +HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} +source ${HOLDVARS} + +export EXPT=${EXPT:-HAFS} +export SUBEXPT=${SUBEXPT:-${EXPT}} + +export CDATE=${CDATE:-${YMDH}} +export cyc=${cyc:-00} +export STORM=${STORM:-FAKE} +export STORMID=${STORMID:-00L} + +#################################### +# Specify Execution Areas +#################################### +export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +export USHhafs=${USHhafs:-${HOMEhafs}/ush} + +#################################### +# Load Modules if Needed +#################################### +export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet +#export RUN_ENVIR=${RUN_ENVIR:-prod} +export RUN_ENVIR=${envir:-prod} +if [ "$RUN_ENVIR" = prod ]; then + #module use ${HOMEhafs}/modulefiles + #module load modulefile.hafs.run.${machine} + if [ "$machine" = wcoss_dell_p3 ]; then + module unload NetCDF + module unload HDF5-serial + fi + #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} + module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles + module load build.${machine}.intel + module list +fi + +##################################### +# Set up job node/core/run environment variables +##################################### +source ${USHhafs}/hafs_runcmd.sh.inc + +#################################### +# Run setpdy and initialize PDY variables +#################################### +#setpdy.sh +#. ./PDY + +PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) +export PDY=${PDY:-${PDYtmp}} + +##################################### +# Working directories +##################################### +export DATA=${WORKhafs}/docn_mesh + +export SCRUBDATA=${SCRUBDATA:-YES} + +if [ "${SCRUBDATA}" = YES ]; then + rm -rf $DATA +fi + +mkdir -p $DATA +cd $DATA + +##################################### +# Execute the script. +${HOMEhafs}/scripts/exhafs_docn_mesh.sh +export err=$? +##################################### + +exit $err diff --git a/modulefiles/modulefile.hafs.run.orion b/modulefiles/modulefile.hafs.run.orion index ce8a340b7..77618350a 100644 --- a/modulefiles/modulefile.hafs.run.orion +++ b/modulefiles/modulefile.hafs.run.orion @@ -63,4 +63,6 @@ module load rocoto/1.3.3 module load intelpython3/2020 -module load cdo \ No newline at end of file +# For DATM and DOCN: +module load cdo +module load nco \ No newline at end of file diff --git a/parm/hafs.conf b/parm/hafs.conf index 8ffff4a50..43ab906e6 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -289,6 +289,8 @@ gtype={grid/gtype} ;; grid type: uniform, stretch, nest, or reg GFSVER={config/GFSVER} ;; Version of GFS input data, e.g., PROD2019, PROD2021 DOWNLOAD_DATM_LIST={download_datm_list} ;; List of task ids for era5 downloader DOWNLOAD_DATM_THRESHOLD={download_datm_threshold} ;; 1.0/(length of download_datm_list) +DOWNLOAD_DOCN_LIST={download_docn_list} ;; List of task ids for era5 downloader +DOWNLOAD_DOCN_THRESHOLD={download_docn_threshold} ;; 1.0/(length of download_docn_list) # Specify the forecast job resources. Only a few combinations are provided. If # needed, you may add other options in the site entity files under rocoto/sites. @@ -310,3 +312,5 @@ SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM ERA5 files? +MAKE_MESH_OCN={make_mesh_ocn} ;; Should the DOCN mesh be generated by the workflow? +DOWNLOAD_DOCN={download_docn} ;; Should the workflow download DOCN OISST files? diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index c5cc80bda..e675d2fdc 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -4,9 +4,9 @@ run_docn=yes ;; Data ocean using CDEPS run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen -make_mesh_ocn=no ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen +make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing -docn_download_jobs=0 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing +docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index e14c57846..1ee2cde27 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -65,17 +65,23 @@ + + + + + + @@ -225,6 +231,39 @@ @** endif + +@** if RUN_DOCN==YES + + &DOWNLOAD_DOCN_LIST; + + &JOBhafs;/JHAFS_DOCN_DOWNLOAD + hafs_input_&SID;_@Y@m@d@H_#idx# + &WORKhafs;/hafs_download_docn_#idx#.log + &ACCOUNT; + &RESERVATION; + &QUEUE_SERVICE; + &SERVICE_EXTRA; + 1 + &CORES_EXTRA; + TOTAL_TASKS1 + 06:00:00 + &MEMORY; + &ENV_VARS; + + + + + &DOWNLOAD_DOCN;YES + &RUN_DOCN;YES + + + + + + + +@** endif + &JOBhafs;/JHAFS_GRID hafs_grid_&SID;_@Y@m@d@H @@ -307,21 +346,46 @@ @Y@m@d@H@M@S -@** if GFSVER==PROD2019 +@** if GFSVER==PROD2019 &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf126.nemsio &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.pgrb2.0p25.f126 -@** endif -@** if GFSVER==PROD2021 +@** endif +@** if GFSVER==PROD2021 &COMgfs;/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f126 &COMgfs;/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.atmf0126.nc -@** endif +@** endif -@** if RUN_OCEAN==YES +@** if RUN_DOCN==YES + + &JOBhafs;/JHAFS_DOCN_MESH + hafs_docn_mesh_&SID;_@Y@m@d@H + &WORKhafs;/hafs_docn_mesh.log + &ACCOUNT; + &RESERVATION; + &QUEUE_PE; + &PE_EXTRA; + &MAKE_ESMF_MESH_RESOURCES; + &ENV_VARS; + + + + + &MAKE_MESH_OCN;YES + + &DOWNLOAD_DOCN;YES + + + + + +@** endif + +@** if RUN_OCEAN==YES &PRE; &EXhafs;/exhafs_hycominit1.py @@ -391,18 +455,18 @@ @Y@m@d@H@M@S -@** if GFSVER==PROD2019 +@** if GFSVER==PROD2019 &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.pgrb2.0p25.f129 -@** endif -@** if GFSVER==PROD2021 +@** endif +@** if GFSVER==PROD2021 &COMgfs;/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f129 -@** endif +@** endif -@** if RUN_DATM==YES +@** if RUN_DATM==YES &JOBhafs;/JHAFS_DATM_MESH @@ -412,7 +476,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; - &MESH_ATM_RESOURCES; + &MAKE_ESMF_MESH_RESOURCES; &ENV_VARS; @@ -427,9 +491,9 @@ -@** endif +@** endif -@** endif +@** endif @** endif @@ -456,10 +520,11 @@ @** if gtype==regional -@** if RUN_OCEAN==YES +@** if RUN_OCEAN==YES -@** if RUN_DATM==YES +@** endif +@** if RUN_DATM==YES &DOWNLOAD_DATM;YES @@ -468,8 +533,17 @@ &MAKE_MESH_ATM;YES -@** endif -@** endif +@** endif +@** if RUN_DOCN==YES + + &DOWNLOAD_DOCN;YES + + + + &MAKE_MESH_OCN;YES + + +@** endif @** endif diff --git a/rocoto/run_hafs.py b/rocoto/run_hafs.py index 65fcdb159..d78528689 100755 --- a/rocoto/run_hafs.py +++ b/rocoto/run_hafs.py @@ -29,7 +29,7 @@ # * -n --- disable renumbering of invests into non-invests # * -W N --- discard invests weaker than N m/s before renumbering # -# Conf opitons: +# Conf options: # * ../parm/hafs_more.conf --- read this configuration file # * config.run_gsi=yes --- specify the value of one configuration option @@ -418,9 +418,9 @@ def fullify(s): conf.timeless_sanity_check(enset,logger) except Exception as e: tcutil.rocoto.sanity_check_failed(logger,e) - logger.critical("HAFS Sanity Checker Designation: INSANE!") - logger.critical("Check your configuration for errors.") - logger.critical("See earlier messages for clues.") + logger.error("HAFS Sanity Checker Designation: INSANE!") + logger.error("Check your configuration for errors.") + logger.error("See earlier messages for clues.") sys.exit(1) logger.info("I think I'm sane.") diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 09dc9de8f..47fd84899 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 8cbaf77f9..2065191b1 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 03fcdd903..1cc36e5ff 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS101:00:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index 8864b779b..e9e3794c8 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 321ea7ff8..52a6e4612 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -15,7 +15,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 1:ppn=6:tpp=1TOTAL_TASKS6NCTSK6OMP_THREADS4-R affinity[core\(4\):distribute=balance]00:30:00"> 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK24OMP_THREADS100:30:00"> 2:ppn=24:tpp=1TOTAL_TASKS48NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index c28e7e0a0..6a0ab5e42 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p1a.ent b/rocoto/sites/xjet_hafsv0p1a.ent index 2977e0173..f3b486575 100644 --- a/rocoto/sites/xjet_hafsv0p1a.ent +++ b/rocoto/sites/xjet_hafsv0p1a.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 6:ppn=24:tpp=1TOTAL_TASKS144NCTSK24OMP_THREADS100:30:00"> 6:ppn=24:tpp=1TOTAL_TASKS144NCTSK24OMP_THREADS103:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 0dbbf2c50..977ed2213 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -14,7 +14,7 @@ 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 30:ppn=24:tpp=1TOTAL_TASKS720NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> diff --git a/scripts/exhafs_datm_mesh.sh b/scripts/exhafs_datm_mesh.sh index f976a9f12..e0525b3c7 100755 --- a/scripts/exhafs_datm_mesh.sh +++ b/scripts/exhafs_datm_mesh.sh @@ -34,23 +34,26 @@ test -r "$ifile" rm -f "$mesh_atm" [ -e "$ofile" -o -L "$ofile" ]] && rm -f "$ofile" +set +x echo "Generating ESMF mesh from ERA5 files." echo "Running in dir \"$PWD\"" echo "ERA5 grid generation input file is \"$ifile\"" echo "Temporary output mesh is $ofile" echo "Will deliver to \"$mesh_atm\"" +set -x -# since the mesh is created drectly from the data file (lat is already -# reversed), there is no need to reverse in here again by providing -# --latrev -$APRUNS $USHhafs/hafs_datm_mesh.py --ifile "$ifile" --ofile "$ofile" \ +# Generate the mesh from the ERA5 file: +$APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ --overwrite --latvar latitude --lonvar longitude --double test -s "$ofile" +# Copy mesh to final destination. $USHhafs/produtil_deliver.py -m "$ofile" "$mesh_atm" test -s "$mesh_atm" ls -l "$mesh_atm" +# Rejoice. +set +x echo "DATM mesh was successfully generated." echo "Enjoy your mesh and have a nice day." diff --git a/scripts/exhafs_docn_download.sh b/scripts/exhafs_docn_download.sh new file mode 100755 index 000000000..2bffa82f2 --- /dev/null +++ b/scripts/exhafs_docn_download.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [[ "$run_docn" != yes || ! "$docn_download_jobs" -gt 0 ]] ; then + echo "This job should only be run if \$run_docn=yes and \$docn_download_jobs>0" + echo " \$docn_download_jobs=\"$docn_download_jobs\"" + echo " \$run_docn=\"$run_docn\"" + echo "Beware! You may anger Poseidon by misusing this script. Avoid coastlines." + echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " + exit 0 +fi + +set -uxe + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} +NHRS=${NHRS:?NHRS is not set} +DOCNdir=${DOCNdir:?DOCNdir is not set} + +export TZ=UTC # Orion workaround + +EDATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d%H ) + +[ -d "$DOCNdir" ] || mkdir "$DOCNdir" +test -d "$DOCNdir" +cd "$DOCNdir" + +echo "Downloading OISST files." +echo "Start date: ${CDATE:0:8}" +echo "End date: ${EDATE:0:8}" +echo "Destination dir: \"$DOCNdir\"" + +"$USHhafs/hafs_docn_download.py" ${CDATE:0:8}-${EDATE:0:8} + +echo "Successfully downloaded all OISST files." +echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh new file mode 100755 index 000000000..ebceb696c --- /dev/null +++ b/scripts/exhafs_docn_mesh.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +if [[ "$make_mesh_ocn" != yes || "$run_docn" != yes ]] ; then + echo "This job should only be run if \$run_docn and \$make_mesh_ocn are both yes." + echo " \$make_mesh_ocn=\"$make_mesh_ocn\"" + echo " \$run_docn=\"$run_docn\"" + echo "Beware! You may anger Poseidon by misusing this script. Avoid coastlines." + echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " + exit 0 +fi + +if ( ! which cdo ) ; then + echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 + exit 1 +fi + +if ( ! which ncwa ) ; then + echo "The \"ncwa\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 + exit 1 +fi + +set -xe + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} +APRUNS=${APRUNS:-"aprun -b -j1 -n1 -N1 -d1 -cc depth"} + +merged=oisst-avhrr-v02r01.merged.nc +nozlev=oisst-avhrr-v02r01.merged_nozlev.nc +ofile=ofile.nc +mesh_ocn="$mesh_ocn" +mesh_dir=$( dirname "$mesh_ocn" ) + +# Start & end times are at day precision, not hour +now=${CDATE:0:8} +end=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d ) + +set +x +echo "Generating ESMF mesh from OISST files." +echo "Running in dir \"$PWD\"" +echo "OISST Date range is $now to $end" +set -x + +# Generate the filenames. +mergefiles='' +itry=0 +itrytoohard=99 # infinite loop guard +while (( now <= end && itry <= itrytoohard )) ; do + mergefiles="$mergefiles $DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" + now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) + itry=$(( itry+1 )) +done +if (( itry > itrytoohard )) ; then + echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 + exit 1 +fi + +set +x +echo "OISST input files are:" +for f in $mergefiles ; do + echo " - $f" +done +echo "Will merge oisst files into $merged" +echo "Will remove z levels into $nozlev" +echo "Temporary output mesh is $ofile" +echo "Will deliver to \"$mesh_ocn\"" +set -x + +# Merge all oisst files into one, as expected by hafs_esmf_mesh.py +cdo mergetime $mergefiles "$merged" +test -s "$merged" +test -r "$merged" + +# Remove z dimension +ncwa -O -a zlev "$merged" "$nozlev" +test -s "$nozlev" +test -r "$nozlev" + +[ -d "$mesh_dir" ] || mkdir "$mesh_dir" +rm -f "$mesh_ocn" +test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" + +# Generate the mesh from the merged oisst file. +$APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$nozlev" --ofile "$ofile" \ + --maskvar sst --maskcal --double --overwrite +test -s "$ofile" + +# Copy mesh to final destination. +$USHhafs/produtil_deliver.py -m "$ofile" "$mesh_ocn" +test -s "$mesh_ocn" + +ls -l "$mesh_ocn" + +# Rejoice. +set +x +echo "OISST files successfully merged." +echo "DOCN mesh was successfully generated." +echo "Enjoy your mesh and have a nice day." diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 1cc0ef243..fbcb6576a 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -501,7 +501,20 @@ if [ ${run_datm} = yes ]; then elif [ ${run_docn} = yes ]; then - ln -sf $DOCNdir/*.nc INPUT/ + now=${CDATE:0:8}00 + end=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d )00 + while (( now <= end )) ; do + infile="$DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" + linkfile="INPUT/oisst-avhrr-v02r01.${now:0:8}.nc" + if [[ ! -s "$infile" ]] ; then + echo "Error: $infile does not exist or is zero size." 1>&2 + exit 1 + fi + ln -sf "$infile" "$linkfile" + now=$( "$NDATE" +24 $now ) + done + +# ln -sf $DOCNdir/*.nc INPUT/ cp ${PARMhafs}/cdeps/docn_in . cp ${PARMhafs}/cdeps/docn.streams.xml . diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index bafa9c8d2..2f113ce63 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -23,5 +23,5 @@ if [ $target = jet ]; then target=jet.intel ; fi cd hafs_forecast.fd/ cd tests/ -./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y CDEPS_DATM=Y CDEPS_DOCN=Y CDEPS=Y" 32bit YES NO +./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=N CMEPS=Y CDEPS_DATM=N CDEPS_DOCN=Y CDEPS=Y" 32bit YES NO diff --git a/ush/hafs/exceptions.py b/ush/hafs/exceptions.py index 52ecb92e3..b56d4edcf 100644 --- a/ush/hafs/exceptions.py +++ b/ush/hafs/exceptions.py @@ -122,6 +122,9 @@ class HAFSFixInsane(HAFSSanityError): class HAFSArchiveInsane(HAFSSanityError): """!Raised when the sanity check of the HAFS archiving settings fails.""" +class HAFSDataModelInsane(HAFSSanityError): + """!Raised when the sanity check of the HAFS data model settings + fails.""" ######################################################################## # OCEAN AND WAVE EXCEPTIONS diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 4e33d0a0a..c2129aa90 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -789,6 +789,25 @@ def read_tcvitals_and_messages(self,vitdir=None,vitpattern=None, revital.readfiles(inputs,raise_all=False) return revital + def timeless_sanity_check_data_models(self,logger): + """!In the hafs_launcher job, this checks the data model variables and + files for obvious errors on the command line, before submitting jobs.""" + run_datm=self.getbool('config','run_datm',False) + run_docn=self.getbool('config','run_docn',False) + run_ocean=self.getbool('config','run_ocean',False) + if run_datm and run_docn: + msg='run_datm and run_docn cannot both be set to yes' + logger.error(msg) + raise HAFSDataModelInsane(msg) + if run_docn and run_ocean: + msg='run_docn and run_ocean cannot both be set to yes' + logger.error(msg) + raise HAFSDataModelInsane(msg) + if run_datm and not run_ocean: + msg='run_datm is useless without run_ocean' + logger.error(msg) + raise HAFSDataModelInsane(msg) + def sanity_check_data_models(self,logger): """!In the hafs_launcher job, this checks the data model variables and files for obvious errors before starting the rest of the workflow.""" @@ -799,14 +818,14 @@ def sanity_check_data_models(self,logger): if run_docn: msg='run_datm and run_docn cannot both be set to yes' logger.error(msg) - raise msg + raise HAFSDataModelInsane(msg) make_mesh_atm=self.getbool('config','make_mesh_atm',True) if not make_mesh_atm: mesh_atm=self.getstr('forecast','mesh_atm') if not os.path.exists(mesh_atm): msg='%s: mesh_atm file does not exist'%(mesh_atm,) logger.error(msg) - raise msg + raise HAFSDataModelInsane(msg) else: logger.info("%s: will use this pre-made datm esmf mesh (mesh_atm)."%(mesh_atm,)) if run_docn: @@ -816,13 +835,13 @@ def sanity_check_data_models(self,logger): if not os.path.exists(mesh_ocn): msg='%s: mesh_ocn file does not exist'%(mesh_ocn,) logger.error(msg) - raise msg + raise HAFSDataModelInsane(msg) else: logger.info("%s: will use this pre-made docn esmf mesh (mesh_ocn)."%(mesh_ocn,)) if run_ocean: msg='run_ocean=yes and run_docn=yes are incompatible.' logger.error(msg) - raise msg + raise HAFSDataModelInsane(msg) def set_data_model_variables(self): """!Sets conf variables for the data models.""" @@ -1192,6 +1211,7 @@ def timeless_sanity_check(self,enset=None,logger=None): '%s: not the same as the launcher.py that is running now ' '(%s) -- check your paths and EXPT.'%(checkme,myfile)) self.sanity_check_forecast_length(logger) + self.timeless_sanity_check_data_models(logger) def sanity_check_forecast_length(self,logger=None): """!Ensures the forecast length is valid. diff --git a/ush/hafs_datm_download.py b/ush/hafs_datm_download.py index dc9a0dd45..4ed6138ad 100755 --- a/ush/hafs_datm_download.py +++ b/ush/hafs_datm_download.py @@ -114,6 +114,9 @@ def request(when): return with produtil.locking.LockFile(filename_lock,logger): try: + if os.path.exists(filename_final): + logger.info(filename_final+': already exists (after lock). Skipping.') + return quiet_remove(filename_download) quiet_remove(filename_invert) logger.info(filename_download+': retrieve '+str(when)+'...') @@ -177,8 +180,8 @@ def request(when): elif re.match('\A\d{4}\Z',arg): logger.info('year') # Year - start=datetime.datetime(int(arg,8),1,1,0,0,0) - end=datetime.datetime(int(arg,8),12,31,23,59,0) + start=datetime.datetime(int(arg,10),1,1,0,0,0) + end=datetime.datetime(int(arg,10),12,31,23,59,0) now=start while now=len(daylist): + logger.info(f'{day}: sleep for a little while... 30 second snooze...') + time.sleep(30) + logger.info(f'{day}: done sleeping.') + iloop=0 + except Exception as ex: # Unfortunately, cdsapi raises Exception + happy = False + logger.error(f'Download failed for {day}: {ex}',exc_info=ex) + +# Exit 0 on success, 1 on failure: +exit( 0 if happy else 1 ) diff --git a/ush/hafs_datm_mesh.py b/ush/hafs_esmf_mesh.py similarity index 92% rename from ush/hafs_datm_mesh.py rename to ush/hafs_esmf_mesh.py index 35a3ef07b..4b389cc1f 100755 --- a/ush/hafs_datm_mesh.py +++ b/ush/hafs_esmf_mesh.py @@ -100,6 +100,9 @@ def calculate_corners(center_lat, center_lon): def write_to_esmf_mesh(filename, center_lat, center_lon, corner_lat, corner_lon, mask, area=None): """ + Writes ESMF Mesh to file + dask array doesn't support order='F' for Fortran-contiguous (row-major) order + the workaround is to arr.T.reshape.T """ # create array with unique coordinate pairs # remove coordinates that are shared between the elements @@ -161,7 +164,7 @@ def write_to_esmf_mesh(filename, center_lat, center_lon, corner_lat, corner_lon, out['elementMask'] = xr.DataArray(mask.T.reshape((-1,)).T, dims=('elementCount'), attrs={'units': 'unitless'}) - #out.elementMask.encoding = {'dtype': np.int32} + out.elementMask.encoding = {'dtype': np.int32} # force no '_FillValue' if not specified for v in out.variables: @@ -182,6 +185,7 @@ def write_to_esmf_mesh(filename, center_lat, center_lon, corner_lat, corner_lon, def write_to_scrip(filename, center_lat, center_lon, corner_lat, corner_lon, mask, area=None): """ + Writes SCRIP grid definition to file dask array doesn't support order='F' for Fortran-contiguous (row-major) order the workaround is to arr.T.reshape.T """ @@ -246,7 +250,8 @@ def file_type(x): #@profile def main(argv): - """Main driver to write SCRIP and ESMF formatted grid represenation + """ + Main driver to calculate and write SCRIP and ESMF formatted grid represenation """ # set defaults for command line arguments ifile = '' @@ -258,6 +263,7 @@ def main(argv): latvar = 'lat' lonvar = 'lon' maskvar = 'mask' + maskcal = False addarea = False double = False @@ -267,11 +273,12 @@ def main(argv): parser.add_argument('--ofile' , help='Output file name', required=True) parser.add_argument('--oformat' , help='Output data format [SCRIP, ESMF], defaults to ESMF', required=False, type=file_type, nargs='?', const='ESMF') parser.add_argument('--overwrite', help='Overwrites output file, defaults to not', required=False, action='store_true') - parser.add_argument('--flip' , help='Flip mask values. SCRIP requires 0 for land and 1 for ocean', required=False, action='store_true') + parser.add_argument('--flip' , help='Flip mask values. SCRIP requires 0/land and 1/ocean', required=False, action='store_true') parser.add_argument('--latrev' , help='Reverse latitude axis', required=False, action='store_true') parser.add_argument('--latvar' , help='Name of latitude variable, defults to ''lat''', required=False, nargs='?', const='lat') parser.add_argument('--lonvar' , help='Name of longitude variable, defaults to ''lon''', nargs='?', const='lon') parser.add_argument('--maskvar' , help='Name of mask variable, defaults to ''mask''', nargs='?', const='mask') + parser.add_argument('--maskcal' , help='Calculate mask using fill value from variable defined in maskvar - 0/land and 1/ocean', required=False, action='store_true') parser.add_argument('--addarea' , help='Add area field to output file, defaults to not', required=False, action='store_true') parser.add_argument('--double' , help='Double precision output, defaults to float', required=False, action='store_true') args = parser.parse_args() @@ -294,6 +301,11 @@ def main(argv): lonvar = args.lonvar if args.maskvar: maskvar = args.maskvar + if args.maskcal: + maskcal = args.maskcal + if not args.maskvar: + print('maskcal argument requires maskvar to calculate mask! exiting ...') + sys.exit() if args.addarea: addarea = args.addarea if args.double: @@ -310,6 +322,7 @@ def main(argv): print("latvar = {}".format(latvar)) print("lonvar = {}".format(lonvar)) print("maskvar = {}".format(maskvar)) + print("maskcal = {} ({})".format(maskcal, maskvar)) print("addarea = {}".format(addarea)) print("double = {}".format(double)) @@ -370,9 +383,20 @@ def main(argv): # check mask has time dimension or not hasTime = 'time' in ds[maskvar].dims if hasTime: - mask = da.from_array(ds[maskvar][:,:,0]) + mask = ds[maskvar][:,:,0] else: - mask = da.from_array(ds[maskvar]) + mask = ds[maskvar][:] + + # use variable to construct mask information + if maskcal: + fill_value = None + if '_FillValue' in mask.attrs: + fill_value = mask._FillValue + elif 'missing_value' in mask.attrs: + fill_value = mask.missing_value + + if fill_value: + mask = da.from_array(xr.where(mask == fill_value, 0, 1).astype(dtype=np.int8)) else: print('Using artifical generated mask values, that are ones in everywhere.') if len(lat.dims) == 1: From af0ed54fce0a027c1dfcc5a7ecfb2d2d55b4ab77 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 30 Mar 2021 16:42:02 -0500 Subject: [PATCH 26/73] .gitignore system.conf, fix directory, and generated rocoto xml files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 547a69bba..3f772d1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ *~ *.db *.db.bak +system.conf +fix/ +rocoto/*.xml From 6ebf79e448734567c7cdc8a7603b8bb5b208a0ae Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 2 Apr 2021 08:48:14 -0500 Subject: [PATCH 27/73] Add QUEUE_FORECAST for forecast job. Orion defaults to "urgent" --- rocoto/hafs_workflow.xml.in | 2 +- rocoto/sites/hera.ent | 1 + rocoto/sites/kjet.ent | 1 + rocoto/sites/orion.ent | 1 + rocoto/sites/wcoss_cray.ent | 1 + rocoto/sites/wcoss_dell_p3.ent | 1 + rocoto/sites/xjet.ent | 1 + rocoto/sites/xjet_hafsv0p1a.ent | 1 + rocoto/sites/xjet_hafsv0p2a.ent | 1 + 9 files changed, 9 insertions(+), 1 deletion(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 1ee2cde27..a7f467feb 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -503,7 +503,7 @@ &WORKhafs;/hafs_forecast.log &ACCOUNT; &RESERVATION; - &QUEUE_PE; + &QUEUE_FORECAST; &PE_EXTRA; &FORECAST_RESOURCES; &ENV_VARS; diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 47fd84899..e93e83494 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -2,6 +2,7 @@ + diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 2065191b1..0004306a6 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -2,6 +2,7 @@ + --partition=kjet"> --partition=kjet"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 1cc36e5ff..d43d187b5 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -2,6 +2,7 @@ + --partition=orion"> --partition=orion"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index e9e3794c8..b0c0cc823 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -2,6 +2,7 @@ + diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 52a6e4612..0acc5689d 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -2,6 +2,7 @@ + diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 6a0ab5e42..f55504377 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -2,6 +2,7 @@ + --partition=xjet"> --partition=xjet"> diff --git a/rocoto/sites/xjet_hafsv0p1a.ent b/rocoto/sites/xjet_hafsv0p1a.ent index f3b486575..161379357 100644 --- a/rocoto/sites/xjet_hafsv0p1a.ent +++ b/rocoto/sites/xjet_hafsv0p1a.ent @@ -2,6 +2,7 @@ + --partition=xjet"> --partition=xjet"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 977ed2213..7a1e9ee49 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -2,6 +2,7 @@ + --partition=xjet"> --partition=xjet"> From 77b65055c6322d5f3a76c2769215ee08c921fab9 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 2 Apr 2021 08:50:02 -0500 Subject: [PATCH 28/73] Bug fixes to get DOCN+FV3 to run. --- parm/hafs_cdeps.conf | 1 + parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 1 + scripts/exhafs_docn_mesh.sh | 3 ++- scripts/exhafs_forecast.sh | 29 +++++++++++------------------ 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf index e675d2fdc..88f77e96b 100644 --- a/parm/hafs_cdeps.conf +++ b/parm/hafs_cdeps.conf @@ -20,6 +20,7 @@ mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFm mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change +merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc # DATM #layoutx=10 diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 1a4c2ba58..e5c3bae9e 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -92,3 +92,4 @@ merge_import={forecast/merge_import} mesh_atm={forecast/mesh_atm} mesh_ocn={forecast/mesh_ocn} +merged_docn_input={forecast/merged_docn_input} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 37305e897..b8316cb04 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -47,6 +47,7 @@ export mesh_atm={mesh_atm} export docn_download_jobs={docn_download_jobs} export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} +export merged_docn_input={merged_docn_input} export SYNDAThafs={syndat} export ADECKhafs={ADECKhafs} diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh index ebceb696c..3a47b3fa4 100755 --- a/scripts/exhafs_docn_mesh.sh +++ b/scripts/exhafs_docn_mesh.sh @@ -87,8 +87,9 @@ $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$nozlev" --ofile "$ofile" \ --maskvar sst --maskcal --double --overwrite test -s "$ofile" -# Copy mesh to final destination. +# Copy mesh and merged file to final destinations. $USHhafs/produtil_deliver.py -m "$ofile" "$mesh_ocn" +$USHhafs/produtil_deliver.py -m "$nozlev" "$merged_docn_input" test -s "$mesh_ocn" ls -l "$mesh_ocn" diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index fbcb6576a..cf4c5793d 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -500,21 +500,12 @@ if [ ${run_datm} = yes ]; then nems.configure.cdeps.tmp > nems.configure elif [ ${run_docn} = yes ]; then - - now=${CDATE:0:8}00 - end=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d )00 - while (( now <= end )) ; do - infile="$DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" - linkfile="INPUT/oisst-avhrr-v02r01.${now:0:8}.nc" - if [[ ! -s "$infile" ]] ; then - echo "Error: $infile does not exist or is zero size." 1>&2 - exit 1 - fi - ln -sf "$infile" "$linkfile" - now=$( "$NDATE" +24 $now ) - done - -# ln -sf $DOCNdir/*.nc INPUT/ + MAKE_MESH_OCN=$( echo "${make_mesh_ocn:-no}" | tr a-z A-Z ) + if [[ "$MAKE_MESH_OCN" == YES ]] ; then + ln -sf "$merged_docn_input" INPUT/ + else + ln -sf "$DOCNdir"/*.nc INPUT/ + fi cp ${PARMhafs}/cdeps/docn_in . cp ${PARMhafs}/cdeps/docn.streams.xml . @@ -526,11 +517,13 @@ elif [ ${run_docn} = yes ]; then sed -i "s/_yearLast_/$endyr/g" docn.streams.xml sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams.xml - for file in `ls INPUT/sst*.nc ` ; do - sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" docn.streams.xml + for file in INPUT/oisst*.nc INPUT/sst*.nc ; do + if [[ -s "$file" ]] ; then + sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" docn.streams.xml + fi done - ln -sf ${mesh_ocn} INPUT/ + ln -sf "${mesh_ocn}" INPUT/ cp ${PARMforecast}/nems.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ From 935089dfda4ef9dec5306f8ad33ef767bcabdb48 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 2 Apr 2021 19:34:07 -0500 Subject: [PATCH 29/73] Changes to get DATM+HYCOM working. --- .../regional/nems.configure.cdeps.tmp | 2 +- parm/hafs.conf | 12 ++++++ parm/hafs_basic.conf | 2 + parm/hafs_cdeps.conf | 41 ------------------- scripts/exhafs_forecast.sh | 15 ++++--- 5 files changed, 24 insertions(+), 48 deletions(-) delete mode 100644 parm/hafs_cdeps.conf diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/nems.configure.cdeps.tmp index fc25e2278..92612956d 100644 --- a/parm/forecast/regional/nems.configure.cdeps.tmp +++ b/parm/forecast/regional/nems.configure.cdeps.tmp @@ -72,7 +72,7 @@ MED_attributes:: history_option = nhours history_ymd = -999 coupling_mode = hafs - system_type = ufs + system_type = cdeps :: ALLCOMP_attributes:: diff --git a/parm/hafs.conf b/parm/hafs.conf index 43ab906e6..ca9ca03bc 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -62,6 +62,10 @@ FIXcrtm={FIXhafs}/hafs-crtm-2.2.3/ ;; GSI CRTM input data utilexec={HOMEhafs}/exec ;; utility exe location (placeholder) +# Data model locations +DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN +DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM_ERA5 + ## Executable program locations # Currently not used in the workflow script system [exe] @@ -236,6 +240,14 @@ merge_import=.true. domain_atm=dummy.nc # mesh_atm=dummy.nc ;; This variable is now set by ush/hafs/launcher.py +# Data model defaults +merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc +mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no +mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change +mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change + + [post] # Grid definition for post and tracker, used by wgrib2 # Example: diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index 011e15c19..c85239183 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -38,7 +38,9 @@ run_dwav=no ;; Data waves using CDEPS run_hrdgraphics=no ;; Run HRD graphics run_emcgraphics=no ;; Run EMC graphics make_mesh_atm=no ;; Generate DATM mesh in workflow (only for run_datm=yes) +make_mesh_ocn=no ;; Generate DOCN mesh in workflow (only for run_docn=yes) datm_download_jobs=0 ;; Number of download_datm jobs (0=disable) +docn_download_jobs=0 ;; Number of download_docn jobs (0=disable) # Scrub options scrub=no ;; do we delete temporary files? (placeholder) diff --git a/parm/hafs_cdeps.conf b/parm/hafs_cdeps.conf deleted file mode 100644 index 88f77e96b..000000000 --- a/parm/hafs_cdeps.conf +++ /dev/null @@ -1,41 +0,0 @@ -[config] -run_datm=no ;; Data atmosphere using CDEPS -run_docn=yes ;; Data ocean using CDEPS -run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. -run_dwav=no ;; Data waves using CDEPS. Not implemented. -make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen -make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen -datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing -docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing - -scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn -scrub_work=no - -[dir] -DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM_ERA5 -DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN - -[forecast] -mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no -mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no -mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change -mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change -merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc - -# DATM -#layoutx=10 -#layouty=8 -#write_groups=1 -#write_tasks_per_group=40 -#ocean_tasks=120 - -# DOCN -ocean_tasks=60 - -[rocotostr] - -# DATM -#FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 - -# DOCN -FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp2 diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index cf4c5793d..a61078c14 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -465,12 +465,15 @@ if [ ${run_datm} = yes ]; then cp ${PARMhafs}/cdeps/datm_in . cp ${PARMhafs}/cdeps/datm.streams.xml . - nowdate=$CDATE - while (( nowdate <= enddate )) ; do - era5_name=ERA5_${nowdate:0:8}.nc - ln -sf $DATMdir/$era5_name INPUT/$era5_name - sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml - nowdate=`${NDATE} +6 $nowdate` + now=${CDATE:0:8}00 + end=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d )00 + while (( now <= end )) ; do + if [[ -s $DATMdir/$era5_name ]] ; then + era5_name=ERA5_${now:0:8}.nc + ln -sf $DATMdir/$era5_name INPUT/$era5_name + sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml + fi + now=`${NDATE} +24 $now` done sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in From dd50964f322d876dbded9cd2bdfbdaf973037ba4 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 2 Apr 2021 19:41:14 -0500 Subject: [PATCH 30/73] Two files missing from prior commit: split hafs_cdeps.conf into hafs_datm.conf and hafs_docn.conf --- parm/hafs_datm.conf | 20 ++++++++++++++++++++ parm/hafs_docn.conf | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 parm/hafs_datm.conf create mode 100644 parm/hafs_docn.conf diff --git a/parm/hafs_datm.conf b/parm/hafs_datm.conf new file mode 100644 index 000000000..6ef4d05ef --- /dev/null +++ b/parm/hafs_datm.conf @@ -0,0 +1,20 @@ +[config] +run_datm=yes ;; Data atmosphere using CDEPS +run_docn=no ;; Data ocean using CDEPS +run_ocean=yes ;; Whether to run the ocean model. Must be no if run_docn=yes. +run_dwav=no ;; Data waves using CDEPS. Not implemented. +make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen +datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing + +scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn +scrub_work=no + +[forecast] +layoutx=10 +layouty=8 +write_groups=1 +write_tasks_per_group=40 +ocean_tasks=120 + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 diff --git a/parm/hafs_docn.conf b/parm/hafs_docn.conf new file mode 100644 index 000000000..f4661b36b --- /dev/null +++ b/parm/hafs_docn.conf @@ -0,0 +1,18 @@ +[config] +run_datm=no ;; Data atmosphere using CDEPS +run_docn=yes ;; Data ocean using CDEPS +run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. +run_dwav=no ;; Data waves using CDEPS. Not implemented. +make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen +docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing + +scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn +scrub_work=no + +[forecast] +ocean_tasks=60 + +[rocotostr] + +# DOCN +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp2 From f0f58e926ca33f7c3a1b3266e6a0928f0bd87432 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 5 Apr 2021 11:11:51 -0500 Subject: [PATCH 31/73] Download an extra day of data before and after storm. Without this, DOCN fails. --- scripts/exhafs_docn_download.sh | 9 +++++---- scripts/exhafs_docn_mesh.sh | 6 ++++-- scripts/exhafs_forecast.sh | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/exhafs_docn_download.sh b/scripts/exhafs_docn_download.sh index 2bffa82f2..5a61fa549 100755 --- a/scripts/exhafs_docn_download.sh +++ b/scripts/exhafs_docn_download.sh @@ -20,18 +20,19 @@ DOCNdir=${DOCNdir:?DOCNdir is not set} export TZ=UTC # Orion workaround -EDATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d%H ) +M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) +P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) [ -d "$DOCNdir" ] || mkdir "$DOCNdir" test -d "$DOCNdir" cd "$DOCNdir" echo "Downloading OISST files." -echo "Start date: ${CDATE:0:8}" -echo "End date: ${EDATE:0:8}" +echo "Start date: ${M1DATE:0:8}" +echo "End date: ${P1DATE:0:8}" echo "Destination dir: \"$DOCNdir\"" -"$USHhafs/hafs_docn_download.py" ${CDATE:0:8}-${EDATE:0:8} +"$USHhafs/hafs_docn_download.py" ${M1DATE:0:8}-${P1DATE:0:8} echo "Successfully downloaded all OISST files." echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh index 3a47b3fa4..460301884 100755 --- a/scripts/exhafs_docn_mesh.sh +++ b/scripts/exhafs_docn_mesh.sh @@ -34,8 +34,10 @@ mesh_ocn="$mesh_ocn" mesh_dir=$( dirname "$mesh_ocn" ) # Start & end times are at day precision, not hour -now=${CDATE:0:8} -end=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d ) +m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) +p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) +now=$m1date +end=$p1date set +x echo "Generating ESMF mesh from OISST files." diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index cf4c5793d..a190c80f8 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -465,8 +465,9 @@ if [ ${run_datm} = yes ]; then cp ${PARMhafs}/cdeps/datm_in . cp ${PARMhafs}/cdeps/datm.streams.xml . + p1date=`${NDATE} +$(( NHRS+6 )) $CDATE` nowdate=$CDATE - while (( nowdate <= enddate )) ; do + while (( nowdate <= p1date )) ; do era5_name=ERA5_${nowdate:0:8}.nc ln -sf $DATMdir/$era5_name INPUT/$era5_name sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml From 90c7ffeb70d9de25cd9cc6e8a2434e82d744e521 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 5 Apr 2021 11:37:09 -0500 Subject: [PATCH 32/73] Download one extra day of DATM data --- scripts/exhafs_datm_download.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh index 04cf0a19e..8a5759f85 100755 --- a/scripts/exhafs_datm_download.sh +++ b/scripts/exhafs_datm_download.sh @@ -27,7 +27,7 @@ DATMdir=${DATMdir:?DATMdir is not set} export TZ=UTC # Orion workaround -EDATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$NHRS hours" +%Y%m%d%H ) +P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) [ -d "$DATMdir" ] || mkdir "$DATMdir" test -d "$DATMdir" @@ -35,10 +35,10 @@ cd "$DATMdir" echo "Downloading ERA5 files and reversing latitudes." echo "Start date: ${CDATE:0:8}" -echo "End date: ${EDATE:0:8}" +echo "End date: ${P1DATE:0:8}" echo "Destination dir: \"$DATMdir\"" -"$USHhafs/hafs_datm_download.py" ${CDATE:0:8}-${EDATE:0:8} +"$USHhafs/hafs_datm_download.py" ${CDATE:0:8}-${P1DATE:0:8} echo "Successfully downloaded all ERA5 files and reversed their latitudes." echo "Enjoy your files and have a nice day." From e51140219542bb336d127c6086d81423418c06f6 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 6 Apr 2021 12:50:40 -0500 Subject: [PATCH 33/73] Give DATM data 1 day before and after simulation to avoid future problems. --- scripts/exhafs_datm_download.sh | 5 +++-- scripts/exhafs_forecast.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh index 8a5759f85..a3a17455d 100755 --- a/scripts/exhafs_datm_download.sh +++ b/scripts/exhafs_datm_download.sh @@ -27,6 +27,7 @@ DATMdir=${DATMdir:?DATMdir is not set} export TZ=UTC # Orion workaround +M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) [ -d "$DATMdir" ] || mkdir "$DATMdir" @@ -34,11 +35,11 @@ test -d "$DATMdir" cd "$DATMdir" echo "Downloading ERA5 files and reversing latitudes." -echo "Start date: ${CDATE:0:8}" +echo "Start date: ${M1DATE:0:8}" echo "End date: ${P1DATE:0:8}" echo "Destination dir: \"$DATMdir\"" -"$USHhafs/hafs_datm_download.py" ${CDATE:0:8}-${P1DATE:0:8} +"$USHhafs/hafs_datm_download.py" ${M1DATE:0:8}-${P1DATE:0:8} echo "Successfully downloaded all ERA5 files and reversed their latitudes." echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 35b302d04..b125bd04b 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -465,15 +465,16 @@ if [ ${run_datm} = yes ]; then cp ${PARMhafs}/cdeps/datm_in . cp ${PARMhafs}/cdeps/datm.streams.xml . + m1date=`${NDATE} -24 $CDATE` p1date=`${NDATE} +$(( NHRS+6 )) $CDATE` - nowdate=$CDATE + nowdate=$m1date while (( nowdate <= p1date )) ; do era5_name=ERA5_${nowdate:0:8}.nc if [[ -s $DATMdir/$era5_name ]] ; then ln -sf $DATMdir/$era5_name INPUT/$era5_name sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml fi - nowdate=`${NDATE} +6 $nowdate` + nowdate=`${NDATE} +24 $nowdate` done sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in From 8d72be170890b35002cef5ce7ea3bc76cffa5d33 Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 3 May 2021 10:44:36 -0500 Subject: [PATCH 34/73] Small refinements to the implementation of CDEPS in the HAFS workflow. Includes setting taxMode and yearAlign correctly and making system_type a dynamic variable in the workflow. --- parm/cdeps/datm.streams.xml | 4 ++-- parm/cdeps/docn.streams.xml | 4 ++-- parm/forecast/regional/nems.configure.cdeps.tmp | 2 +- parm/hafs.conf | 5 +---- scripts/exhafs_forecast.sh | 2 ++ ush/hafs_esmf_mesh.py | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/parm/cdeps/datm.streams.xml b/parm/cdeps/datm.streams.xml index 27369ca29..6d71b4486 100755 --- a/parm/cdeps/datm.streams.xml +++ b/parm/cdeps/datm.streams.xml @@ -2,14 +2,14 @@ - cycle + limit linear single bilinear 1.5 _yearFirst_ _yearLast_ - 1900 + _yearFirst_ u:v _mesh_atm_ null diff --git a/parm/cdeps/docn.streams.xml b/parm/cdeps/docn.streams.xml index c832fd0ca..8e327da87 100644 --- a/parm/cdeps/docn.streams.xml +++ b/parm/cdeps/docn.streams.xml @@ -2,14 +2,14 @@ - cycle + limit linear single bilinear 1.5 _yearFirst_ _yearLast_ - 1 + _yearFirst_ null _mesh_ocn_ null diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/nems.configure.cdeps.tmp index 92612956d..c1a0cdc1b 100644 --- a/parm/forecast/regional/nems.configure.cdeps.tmp +++ b/parm/forecast/regional/nems.configure.cdeps.tmp @@ -72,7 +72,7 @@ MED_attributes:: history_option = nhours history_ymd = -999 coupling_mode = hafs - system_type = cdeps + system_type = _system_type_ :: ALLCOMP_attributes:: diff --git a/parm/hafs.conf b/parm/hafs.conf index ca9ca03bc..0fb7dd2c4 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -39,8 +39,6 @@ realstormwork={CDSCRUB}/{RUNhafs}/{YMDH}/{realstorm} oldsid={oldvit[stormid3]} oldcom={CDSCRUB}/{RUNhafs}/com/{oldvit[YMDH]}/{oldvit[stormid3]} intercom={WORKhafs}/intercom ;; dir for communicating data files between jobs -DATMdir=dummy ;; input dir for cdeps datm -DOCNdir=dummy ;; input dir for cdeps docn outatcf={CDNOSCRUB}/{SUBEXPT} ;; Delivery location for ATCF files outdiag={CDNOSCRUB}/diagtrak/{SUBEXPT} ;; Delivery location for wrfdiag files outstatus={CDNOSCRUB}/cycstatus/{SUBEXPT} ;; Delivery location for status files @@ -64,7 +62,7 @@ utilexec={HOMEhafs}/exec ;; utility exe location (placeholder # Data model locations DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN -DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM_ERA5 +DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM ## Executable program locations # Currently not used in the workflow script system @@ -238,7 +236,6 @@ ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31 merge_import=.true. domain_atm=dummy.nc -# mesh_atm=dummy.nc ;; This variable is now set by ush/hafs/launcher.py # Data model defaults merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index b125bd04b..4654d5cfe 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -499,6 +499,7 @@ if [ ${run_datm} = yes ]; then -e "s/_merge_import_/${merge_import:-.true.}/g" \ -e "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" \ -e "/_mesh_ocn_/d" \ + -e "/_system_type_/d" \ -e "s/_atm_model_/datm/g" \ -e "s/_ocn_model_/hycom/g" \ nems.configure.cdeps.tmp > nems.configure @@ -541,6 +542,7 @@ elif [ ${run_docn} = yes ]; then -e "s/_merge_import_/${merge_import:-.true.}/g" \ -e "/_mesh_atm_/d" \ -e "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" \ + -e "s/_system_type_/ufs/g" \ -e "s/_atm_model_/fv3/g" \ -e "s/_ocn_model_/docn/g" \ nems.configure.cdeps.tmp > nems.configure diff --git a/ush/hafs_esmf_mesh.py b/ush/hafs_esmf_mesh.py index 4b389cc1f..338c6377b 100755 --- a/ush/hafs_esmf_mesh.py +++ b/ush/hafs_esmf_mesh.py @@ -16,7 +16,7 @@ pip install xarray --user pip install dask --user - pip install "dask[array]" --upgrade —user + pip install "dask[array]" --upgrade --user pip install "dask[dataframe]" --upgrade --user """) exit(2) From cab1b0efa2bdbccd021d30e18d791e69941ae5b9 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 7 May 2021 15:35:51 -0500 Subject: [PATCH 35/73] Make sure that the atmospheric initialization system is disabled if the hafs_datm.conf is used. --- parm/hafs_datm.conf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/parm/hafs_datm.conf b/parm/hafs_datm.conf index 6ef4d05ef..8c4c8d4d1 100644 --- a/parm/hafs_datm.conf +++ b/parm/hafs_datm.conf @@ -1,7 +1,7 @@ [config] run_datm=yes ;; Data atmosphere using CDEPS run_docn=no ;; Data ocean using CDEPS -run_ocean=yes ;; Whether to run the ocean model. Must be no if run_docn=yes. +run_ocean=yes ;; Whether to run the ocean model. run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing @@ -9,6 +9,17 @@ datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no +;; Make sure the atmospheric initialization system is disabled +run_vortexinit=no ;; vortex initialization +run_gsi_vr=no ;; GSI based vortex relocation +run_gsi_vr_fgat=no ;; GSI based vortex relocation for FGAT +run_gsi_vr_ens=no ;; GSI based vortex relocation for ensda members +run_gsi=no ;; GSI and FGAT initialization +run_fgat=no ;; Enable FGAT in DA +run_envar=no ;; Run GSI with hybrid EnVar with either GDAS ensembles or regional ensembles +run_ensda=no ;; Run ensemble data assimilation system +run_enkf=no ;; Run self-cycled ensemble data assimilation system with EnKF analysis + [forecast] layoutx=10 layouty=8 From ef1dac0fda9498fc0bdf85d5ccc65e85cbb0b433 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 7 May 2021 18:04:54 -0500 Subject: [PATCH 36/73] Bug fix: When run_datm=yes, make sure the forecast job does not start before ocn_prep finishes. --- rocoto/hafs_workflow.xml.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 1b4f1c425..05a247328 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -638,9 +638,6 @@ @** if gtype==regional -@** if RUN_OCEAN==YES - -@** endif @** if RUN_GSI==YES @** endif @@ -649,6 +646,9 @@ @** endif +@** if RUN_OCEAN==YES + +@** endif @** endif From 81f91265106497669989b6ae880bbe7610e81176 Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 7 May 2021 20:00:22 -0500 Subject: [PATCH 37/73] Bug fix: Make sure INPUT/ directory is created before attempting to copy files to it. Clean up hafs.conf. --- parm/hafs.conf | 2 +- scripts/exhafs_forecast.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index 6d6890f1f..c25ce8098 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -269,7 +269,7 @@ merge_import=.true. merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change -mesh_atm_in=/work/noaa/dtc-hwrf/ekalina/RUN_test/input/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_in=/work/noaa/{disk_project}/{ENV[USER]}/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change [forecast_ens] diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 36242bb25..1db0fad2a 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -294,6 +294,8 @@ else mkdir -p ${RESTARTout} fi +mkdir -p INPUT + if [ ${run_datm} = no ]; then # Link the input IC and/or LBC files into the INPUT dir @@ -302,7 +304,6 @@ if [ ! -d $INPdir ]; then exit 9 fi -mkdir -p INPUT ${NLN} ${INPdir}/*.nc INPUT/ # Copy fix files @@ -622,6 +623,9 @@ ${yr}${mn}${dy}.${cyc}Z.${CASE}.32bit.non-hydro $yr $mn $dy $cyc 0 0 EOF +enddate=`${NDATE} +${NHRS} $CDATE` +endyr=`echo $enddate | cut -c1-4` + if [ ${run_datm} = no ]; then cat temp diag_table.tmp > diag_table fi From 6eed2ad15eef557e83e06b97a5c525dc5e7088a2 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 28 May 2021 14:28:11 -0500 Subject: [PATCH 38/73] Corrections to hafs_esmf_mesh.py: print reason for import failure and request python3 instead of python --- ush/hafs_esmf_mesh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/hafs_esmf_mesh.py b/ush/hafs_esmf_mesh.py index 338c6377b..8cabf9f98 100755 --- a/ush/hafs_esmf_mesh.py +++ b/ush/hafs_esmf_mesh.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os, sys, getopt import argparse @@ -19,6 +19,7 @@ pip install "dask[array]" --upgrade --user pip install "dask[dataframe]" --upgrade --user """) + sys.stderr.write(str(ie)) exit(2) From a5629dd4decafd47a611db88db2b016914c0ce27 Mon Sep 17 00:00:00 2001 From: uturuncoglu Date: Thu, 3 Jun 2021 10:44:43 -0600 Subject: [PATCH 39/73] add cheyenne support to build --- sorc/build_forecast.sh | 1 + sorc/machine-setup.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index eba30d59a..d2e390fdf 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -6,6 +6,7 @@ cwd=`pwd` if [ $target = hera ]; then target=hera.intel ; fi if [ $target = orion ]; then target=orion.intel ; fi if [ $target = jet ]; then target=jet.intel ; fi +if [ $target = cheyenne ]; then target=cheyenne.intel ; fi cd hafs_forecast.fd/tests diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 9fbb4ee73..786175330 100755 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -90,12 +90,12 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then module purge source /usrx/local/prod/lmod/lmod/init/$__ms_shell elif [[ -d /glade ]] ; then - # We are on NCAR Yellowstone + # We are on NCAR Cheyenne if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 . /usr/share/Modules/init/$__ms_shell fi - target=yellowstone + target=cheyenne module purge elif [[ -d /lustre && -d /ncrc ]] ; then # We are on GAEA. From 20d0a1e96df44d2699b91448b7a2ea060d31744d Mon Sep 17 00:00:00 2001 From: evankalina Date: Fri, 2 Jul 2021 14:43:37 -0500 Subject: [PATCH 40/73] Update submodule pointers to the latest hashes of feature/hafs_couplehycom_cdeps (ufs-weather-model) and support/HAFS (GSI, post, and utils). --- sorc/hafs_forecast.fd | 2 +- sorc/hafs_gsi.fd | 2 +- sorc/hafs_post.fd | 2 +- sorc/hafs_utils.fd | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 9ff89f518..1da5ad71a 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 9ff89f518b1cf35df1f87ddecda621434a45e32e +Subproject commit 1da5ad71aede7c7cddf4cecd929c670edbf0b38b diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 498e58477..af7b97860 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 498e5847763a3b691d395f9db6656df90ddbc16c +Subproject commit af7b9786015a7a0c56deb8d668aca834faefe4e5 diff --git a/sorc/hafs_post.fd b/sorc/hafs_post.fd index 227e6e67c..ce67fb9f5 160000 --- a/sorc/hafs_post.fd +++ b/sorc/hafs_post.fd @@ -1 +1 @@ -Subproject commit 227e6e67ca74ae5ab19637114ac80b65b184c7f8 +Subproject commit ce67fb9f5dc738e9ddd118a626b45419f359a70f diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index a428bc26b..3c3126535 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit a428bc26b4aba2213aa43220a000c5fcbe4bd688 +Subproject commit 3c31265354b34ae1b61c09a02225e320c913395f From ecb2f91c8a3947256197485f433cb583e197a6ac Mon Sep 17 00:00:00 2001 From: evankalina Date: Thu, 8 Jul 2021 10:02:28 -0500 Subject: [PATCH 41/73] Update the HAFS-CDEPS workflow to use dxxx.streams files in text format, instead of XML. This change is required by recent updates to the CDEPS implementation in UFS weather model. --- parm/cdeps/datm.streams | 15 ++++ parm/cdeps/datm.streams.xml | 47 ----------- parm/cdeps/datm_in | 1 - parm/cdeps/docn.streams | 15 ++++ parm/cdeps/docn.streams.xml | 24 ------ parm/cdeps/docn_in | 1 - .../regional/nems.configure.cdeps.tmp | 84 ++++++++++++++----- scripts/exhafs_forecast.sh | 28 +++---- 8 files changed, 103 insertions(+), 112 deletions(-) create mode 100644 parm/cdeps/datm.streams delete mode 100755 parm/cdeps/datm.streams.xml create mode 100644 parm/cdeps/docn.streams delete mode 100644 parm/cdeps/docn.streams.xml diff --git a/parm/cdeps/datm.streams b/parm/cdeps/datm.streams new file mode 100644 index 000000000..b8e3a91e3 --- /dev/null +++ b/parm/cdeps/datm.streams @@ -0,0 +1,15 @@ +stream_info: ERA5_HOURLY01 +taxmode01: limit +mapalgo01: bilinear +tInterpAlgo01: linear +readMode01: single +dtlimit01: 1.5 +stream_offset01: 0 +yearFirst01: _yearFirst_ +yearLast01: _yearLast_ +yearAlign01: _yearFirst_ +stream_vectors01: "u:v" +stream_mesh_file01: _mesh_atm_ +stream_lev_dimname01: null +stream_data_files01: +stream_data_variables01: "u10 Sa_u10m" "v10 Sa_v10m" "t2m Sa_t2m" "skt Sa_tskn" "d2m Sa_tdew" "msl Sa_pslv" "tp Faxa_rain" "cp Faxa_rainc" "lsp Faxa_rainl" "csf Faxa_snowc" "lsf Faxa_snowl" "ssrd Faxa_swdn" "ssr Faxa_swnet" "strd Faxa_lwdn" "str Faxa_lwnet" "aluvp Faxa_swvdr" "aluvd Faxa_swvdf" "alnip Faxa_swndr" "alnid Faxa_swndf" "sshf Faxa_sen" "slhf Faxa_lat" "ewss Faxa_taux" "nsss Faxa_tauy" diff --git a/parm/cdeps/datm.streams.xml b/parm/cdeps/datm.streams.xml deleted file mode 100755 index 6d71b4486..000000000 --- a/parm/cdeps/datm.streams.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - limit - linear - single - bilinear - 1.5 - _yearFirst_ - _yearLast_ - _yearFirst_ - u:v - _mesh_atm_ - null - - - - u10 Sa_u - v10 Sa_v - t2m Sa_tbot - skt Sa_tskn - d2m Sa_tdew - msl Sa_pslv - sp Sa_pbot - tp Faxa_rain - cp Faxa_rainc - lsp Faxa_rainl - csf Faxa_snowc - lsf Faxa_snowl - ssrd Faxa_swdn - ssr Faxa_swnet - strd Faxa_lwdn - str Faxa_lwnet - aluvp Faxa_swvdr - aluvd Faxa_swvdf - alnip Faxa_swndr - alnid Faxa_swndf - sshf Faxa_sen - slhf Faxa_lat - ewss Faxa_taux - nsss Faxa_tauy - - 0 - - - diff --git a/parm/cdeps/datm_in b/parm/cdeps/datm_in index 3c481f34e..62b1ab0cf 100755 --- a/parm/cdeps/datm_in +++ b/parm/cdeps/datm_in @@ -6,7 +6,6 @@ flds_presaero = .false. flds_wiso = .false. iradsw = 1 - model_createmesh_fromfile = "null" model_maskfile = "_mesh_atm_" model_meshfile = "_mesh_atm_" nx_global = 1440 diff --git a/parm/cdeps/docn.streams b/parm/cdeps/docn.streams new file mode 100644 index 000000000..68501c732 --- /dev/null +++ b/parm/cdeps/docn.streams @@ -0,0 +1,15 @@ +stream_info: PRESCRIBED01 +taxmode01: limit +mapalgo01: bilinear +tInterpAlgo01: linear +readMode01: single +dtlimit01: 1.5 +stream_offset01: 0 +yearFirst01: _yearFirst_ +yearLast01: _yearLast_ +yearAlign01: _yearFirst_ +stream_vectors01: "null" +stream_mesh_file01: _mesh_ocn_ +stream_lev_dimname01: null +stream_data_files01: +stream_data_variables01: "sst So_t" diff --git a/parm/cdeps/docn.streams.xml b/parm/cdeps/docn.streams.xml deleted file mode 100644 index 8e327da87..000000000 --- a/parm/cdeps/docn.streams.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - limit - linear - single - bilinear - 1.5 - _yearFirst_ - _yearLast_ - _yearFirst_ - null - _mesh_ocn_ - null - - - - sst So_t - - 0 - - - diff --git a/parm/cdeps/docn_in b/parm/cdeps/docn_in index 0572fe2d4..b562382de 100644 --- a/parm/cdeps/docn_in +++ b/parm/cdeps/docn_in @@ -1,6 +1,5 @@ &docn_nml datamode = "sstdata" - model_createmesh_fromfile = "null" model_maskfile = "_mesh_ocn_" model_meshfile = "_mesh_ocn_" nx_global = 1440 diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/nems.configure.cdeps.tmp index c1a0cdc1b..ecb7b77e2 100644 --- a/parm/forecast/regional/nems.configure.cdeps.tmp +++ b/parm/forecast/regional/nems.configure.cdeps.tmp @@ -9,26 +9,38 @@ EARTH_attributes:: :: # MED # -MED_model: nems +MED_model: cmeps _MED_petlist_bounds_ +MED_attributes:: + Verbosity = 1 + Diagnostic = 0 + ATM_model = _atm_model_ + OCN_model = _ocn_model_ + MED_model = cmeps + history_n = 1 + history_option = ndays + history_ymd = -999 + coupling_mode = hafs + normalization = none + merge_type = copy :: # ATM # ATM_model: _atm_model_ _ATM_petlist_bounds_ ATM_attributes:: - mesh_atm = _mesh_atm_ Verbosity = 1 Diagnostic = 0 + mesh_atm = _mesh_atm_ :: # OCN # OCN_model: _ocn_model_ _OCN_petlist_bounds_ OCN_attributes:: - mesh_ocn = _mesh_ocn_ Verbosity = 1 Diagnostic = 0 + mesh_ocn = _mesh_ocn_ # The following are only used by the hycom ocean model. # cdf_impexp_freq = 3 cpl_hour = 0 @@ -37,10 +49,12 @@ OCN_attributes:: base_dtg = _base_dtg_ merge_import = _merge_import_ skip_first_import = .true. - hycom_arche_output = .true. + hycom_arche_output = .false. hyc_esmf_exp_output = .true. hyc_esmf_imp_output = .true. - hyc_impexp_file = dummy_file + import_diagnostics = .false. + import_setting = flexible + hyc_impexp_file = nems.configure espc_show_impexp_minmax = .true. ocean_start_dtg = _ocean_start_dtg_ start_hour = 0 @@ -54,28 +68,31 @@ OCN_attributes:: # Run Sequence # runSeq:: @_cpl_dt_ -_runSeq_ALL_ + ATM -> MED :remapMethod=redist + MED med_phases_post_atm + OCN -> MED :remapMethod=redist + MED med_phases_post_ocn + MED med_phases_prep_atm + MED med_phases_prep_ocn_accum + MED med_phases_prep_ocn_avg + MED -> ATM :remapMethod=redist + MED -> OCN :remapMethod=redist + ATM + OCN + MED med_phases_restart_write + MED med_phases_history_write @ :: # Other Attributes # DRIVER_attributes:: - mediator_read_restart = false + start_type = startup :: -MED_attributes:: +ALLCOMP_attributes:: ATM_model = _atm_model_ OCN_model = _ocn_model_ - Verbosity = 1 - Diagnostic = 0 - history_n = 1 - history_option = nhours - history_ymd = -999 - coupling_mode = hafs - system_type = _system_type_ -:: - -ALLCOMP_attributes:: + MED_model = cmeps ScalarFieldCount = 3 ScalarFieldIdxGridNX = 1 ScalarFieldIdxGridNY = 2 @@ -83,17 +100,19 @@ ALLCOMP_attributes:: ScalarFieldName = cpl_scalars start_type = startup case_name = ufs.hafs - restart_n = 24 - restart_option = nhours + restart_n = 1 + restart_option = ndays restart_ymd = -999 - normalization = none - merge_type = copy + dbug_flag = 20 + use_coldstart = true orb_eccen = 1.e36 orb_iyear = 2000 orb_iyear_align = 2000 orb_mode = fixed_year orb_mvelp = 1.e36 orb_obliq = 1.e36 + mediator_read_restart = false + mediator_present = true :: ATM_modelio:: @@ -108,5 +127,24 @@ OCN_modelio:: MED_modelio:: diro = . - logfile = nems.log + logfile = med.log +:: + +# The following are only used by the hycom ocean model. # +ocn_export_fields:: + 'sst' 'sea_surface_temperature' 'K' + 'mask' 'ocean_mask' '1' + 'cpl_scalars' 'cpl_scalars' '1' +:: + +# The following are only used by the hycom ocean model. # +ocn_import_fields:: + 'taux10' 'mean_zonal_moment_flx_atm' 'N_m-2' + 'tauy10' 'mean_merid_moment_flx_atm' 'N_m-2' + 'prcp' 'mean_prec_rate' 'kg_m-2_s-1' + 'swflxd' 'mean_net_sw_flx' 'W_m-2' + 'lwflxd' 'mean_net_lw_flx' 'W_m-2' + 'mslprs' 'inst_pres_height_surface' 'Pa' + 'sensflx' 'mean_sensi_heat_flx' 'W_m-2' + 'latflx' 'mean_laten_heat_flx' 'W_m-2' :: diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index d3b7d72f9..818bd6e54 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -276,12 +276,10 @@ if [ ${run_datm} = yes ]; then cplflx=.true. OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" 0 $(($ATM_tasks-1))) - runSeq_ALL="MED med_phases_prep_ocn_avg\n MED -> OCN :remapMethod=redist\n OCN\n ATM\n ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n MED med_phases_aofluxes_run\n MED med_phases_prep_ocn_accum\n OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true\n MED med_phases_post_ocn\n MED med_phases_history_write\n MED med_phases_restart_write" elif [ ${run_docn} = yes ]; then cplflx=.true. OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) - runSeq_ALL="MED med_phases_prep_atm\n MED -> ATM :remapMethod=redist\n ATM\n OCN\n ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n OCN -> MED :remapMethod=redist\n MED med_phases_post_ocn\n MED med_phases_restart_write\n MED med_phases_history_write" fi # Prepare the output RESTART dir @@ -653,7 +651,7 @@ if [ ${run_datm} = yes ]; then #---------------------------------------------- ${NCP} ${PARMforecast}/model_configure.tmp . ${NCP} ${PARMhafs}/cdeps/datm_in . - ${NCP} ${PARMhafs}/cdeps/datm.streams.xml . + ${NCP} ${PARMhafs}/cdeps/datm.streams . m1date=`${NDATE} -24 $CDATE` p1date=`${NDATE} +$(( NHRS+6 )) $CDATE` @@ -662,18 +660,18 @@ if [ ${run_datm} = yes ]; then era5_name=ERA5_${nowdate:0:8}.nc if [[ -s $DATMdir/$era5_name ]] ; then ${NLN} $DATMdir/$era5_name INPUT/$era5_name - sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ INPUT/$era5_name<\/file>" datm.streams.xml + sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$era5_name/" datm.streams fi nowdate=`${NDATE} +24 $nowdate` done sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in - sed -i "s/_yearFirst_/$yr/g" datm.streams.xml + sed -i "s/_yearFirst_/$yr/g" datm.streams - sed -i "s/_yearLast_/$endyr/g" datm.streams.xml + sed -i "s/_yearLast_/$endyr/g" datm.streams - sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams.xml + sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams ${NLN} ${mesh_atm} INPUT/ @@ -682,7 +680,6 @@ if [ ${run_datm} = yes ]; then -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ -e "s/_cpl_dt_/${cpl_dt}/g" \ - -e "s/_runSeq_ALL_/${runSeq_ALL}/g" \ -e "s/_base_dtg_/${CDATE}/g" \ -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ @@ -703,18 +700,18 @@ elif [ ${run_docn} = yes ]; then fi ${NCP} ${PARMhafs}/cdeps/docn_in . - ${NCP} ${PARMhafs}/cdeps/docn.streams.xml . + ${NCP} ${PARMhafs}/cdeps/docn.streams . sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn_in - sed -i "s/_yearFirst_/$yr/g" docn.streams.xml + sed -i "s/_yearFirst_/$yr/g" docn.streams - sed -i "s/_yearLast_/$endyr/g" docn.streams.xml + sed -i "s/_yearLast_/$endyr/g" docn.streams - sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams.xml + sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams for file in INPUT/oisst*.nc INPUT/sst*.nc ; do if [[ -s "$file" ]] ; then - sed -i "/<\/stream_data_files>/i \ \ \ \ \ \ $file<\/file>" docn.streams.xml + sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$(basename $file)/" docn.streams fi done @@ -725,7 +722,6 @@ elif [ ${run_docn} = yes ]; then -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ -e "s/_cpl_dt_/${cpl_dt}/g" \ - -e "s/_runSeq_ALL_/${runSeq_ALL}/g" \ -e "s/_base_dtg_/${CDATE}/g" \ -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ @@ -783,7 +779,7 @@ ${APRUNC} ./hafs_forecast.x 1>out.forecast 2>err.forecast cat ./out.forecast cat ./err.forecast -if [ $gtype = regional ]; then +if [ $gtype = regional ] && [ ${run_datm} = no ]; then # Rename the restart files with a proper convention if needed cd RESTART @@ -809,6 +805,6 @@ if [ ! -s RESTART/oro_data.nc ]; then ${NCP} -pL INPUT/oro_data.nc RESTART/ fi -fi # if [ $gtype = regional ]; then +fi # if [ $gtype = regional ] && [ ${run_datm} = no ]; then exit From 431a2d6c212371abe5746073bb605dd5ab56cdd5 Mon Sep 17 00:00:00 2001 From: evankalina Date: Thu, 8 Jul 2021 17:24:53 -0500 Subject: [PATCH 42/73] Fix incomplete merge in wcoss_cray.ent. --- rocoto/sites/wcoss_cray.ent | 1 - 1 file changed, 1 deletion(-) diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index b3edc993f..9ecd37b7a 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -23,7 +23,6 @@ 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> ->>>>>>> origin/develop 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> From aa1f5127a7a35849e563581ab89384db96e4ab94 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 9 Jul 2021 09:58:00 -0500 Subject: [PATCH 43/73] split DOCN scripts into three, for GHRSST, OISST, and RTOFS --- parm/cdeps/docn_in | 4 +- parm/hafs.conf | 7 +- parm/hafs_docn.conf | 4 + parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 1 + parm/system.conf.hera | 2 + parm/system.conf.jet | 2 + parm/system.conf.kjet | 2 + parm/system.conf.orion | 2 + parm/system.conf.wcoss_cray | 2 + parm/system.conf.wcoss_dell_p3 | 2 + rocoto/hafs_workflow.xml.in | 2 +- rocoto/sites/hera.ent | 1 + rocoto/sites/kjet.ent | 1 + rocoto/sites/orion.ent | 1 + rocoto/sites/wcoss_cray.ent | 2 +- rocoto/sites/wcoss_dell_p3.ent | 1 + rocoto/sites/xjet.ent | 1 + rocoto/sites/xjet_hafsv0p2a.ent | 1 + scripts/exhafs_docn_download.sh | 16 +- scripts/exhafs_docn_mesh.sh | 94 +++---- scripts/exhafs_forecast.sh | 22 +- sorc/build_forecast.sh | 2 +- ush/hafs_ghrsst_download.py | 223 +++++++++++++++++ ...ocn_download.py => hafs_oisst_download.py} | 0 ush/hafs_oisst_prep_mesh.sh | 90 +++++++ ush/hafs_rtofs_download.py | 230 ++++++++++++++++++ ush/hafs_rtofs_prep_mesh.sh | 93 +++++++ 28 files changed, 733 insertions(+), 76 deletions(-) mode change 100755 => 100644 scripts/exhafs_docn_mesh.sh create mode 100755 ush/hafs_ghrsst_download.py rename ush/{hafs_docn_download.py => hafs_oisst_download.py} (100%) create mode 100644 ush/hafs_oisst_prep_mesh.sh create mode 100755 ush/hafs_rtofs_download.py create mode 100644 ush/hafs_rtofs_prep_mesh.sh diff --git a/parm/cdeps/docn_in b/parm/cdeps/docn_in index b562382de..d863d076e 100644 --- a/parm/cdeps/docn_in +++ b/parm/cdeps/docn_in @@ -2,8 +2,8 @@ datamode = "sstdata" model_maskfile = "_mesh_ocn_" model_meshfile = "_mesh_ocn_" - nx_global = 1440 - ny_global = 720 + nx_global = _nx_global_ + ny_global = _ny_global_ restfilm = "null" sst_constant_value = -1.0 / diff --git a/parm/hafs.conf b/parm/hafs.conf index edaae7121..99d8f9bff 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -273,11 +273,14 @@ ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31 merge_import=.true. # Data model defaults -merged_docn_input={WORKhafs}/intercom/oisst-avhrr-v02r01.{YMD}.nc +merged_docn_input={WORKhafs}/intercom/DOCN_input_merged.nc mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change -mesh_atm_in=/work/noaa/{disk_project}/{ENV[USER]}/static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_in={DATMdir}/../static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change +docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction +docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction + [forecast_ens] # ccpp suites diff --git a/parm/hafs_docn.conf b/parm/hafs_docn.conf index f4661b36b..6c476de54 100644 --- a/parm/hafs_docn.conf +++ b/parm/hafs_docn.conf @@ -6,11 +6,15 @@ run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing +docn_source=OISST ;; OISST, RTOFS, or GHRSST + scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no [forecast] ocean_tasks=60 +docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction +docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction [rocotostr] diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index b08894ee0..386df2fde 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -33,6 +33,7 @@ ENVEQUIV={WHERE_AM_I} ;; Present cluster name # SUBEXPT={ENV[SUBEXPT]} #JET_NAME={ENV[JET_NAME]} #WHERE_AM_I={ENV[ENVEQUIV]} +DOCN_SOURCE={docn_source} CASE={grid/CASE} LEVS={grid/LEVS} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 8f2e239f2..320b6a8b4 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -57,6 +57,7 @@ export docn_download_jobs={docn_download_jobs} export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} export merged_docn_input={merged_docn_input} +export DOCN_SOURCE={DOCN_SOURCE} export SYNDAThafs={syndat} export ADECKhafs={ADECKhafs} diff --git a/parm/system.conf.hera b/parm/system.conf.hera index e388d25b4..1c923ea6a 100644 --- a/parm/system.conf.hera +++ b/parm/system.conf.hera @@ -22,6 +22,8 @@ inputroot=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16 [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/scratch1/NCEPDEV/{disk_project}/noscrub/{ENV[USER]}/hafstrak +DOCNdir=/scratch1/NCEPDEV/{disk_project}/noscrub/{ENV[USER]}/DOCN +DATMdir=/scratch1/NCEPDEV/{disk_project}/noscrub/{ENV[USER]}/DATM ## Save directory. Make sure you edit this. CDSAVE=/scratch1/NCEPDEV/{disk_project}/save/{ENV[USER]} ## Scrubbed directory for large work files. Make sure you edit this. diff --git a/parm/system.conf.jet b/parm/system.conf.jet index 4b0cef88e..8601297db 100644 --- a/parm/system.conf.jet +++ b/parm/system.conf.jet @@ -23,6 +23,8 @@ inputroot=/lfs4/HFIP/hwrf-data/hafs-input/COMGFSv16 [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/hafstrak +DATMdir=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/DATM +DOCNdir=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/DOCN ## Scrubbed directory for large work files. Make sure you edit this. CDSCRUB=/lfs4/HFIP/{disk_project}/{ENV[USER]}/hafstmp ## Save directory. Make sure you edit this. diff --git a/parm/system.conf.kjet b/parm/system.conf.kjet index a117a0c8a..2b4dd7d54 100644 --- a/parm/system.conf.kjet +++ b/parm/system.conf.kjet @@ -23,6 +23,8 @@ inputroot=/lfs4/HFIP/hwrf-data/hafs-input/COMGFSv16 [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/hafstrak +DATMdir=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/DATM +DOCNdir=/lfs4/HFIP/{disk_project}/{ENV[USER]}/noscrub/DOCN ## Scrubbed directory for large work files. Make sure you edit this. CDSCRUB=/lfs4/HFIP/{disk_project}/{ENV[USER]}/hafstmp ## Save directory. Make sure you edit this. diff --git a/parm/system.conf.orion b/parm/system.conf.orion index 3f419c5db..702fc9550 100644 --- a/parm/system.conf.orion +++ b/parm/system.conf.orion @@ -22,6 +22,8 @@ inputroot=/work/noaa/hwrf/noscrub/hafs-input/COMGFSv16 [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/work/noaa/{disk_project}/noscrub/{ENV[USER]}/hafstrak +DATMdir=/work/noaa/{disk_project}/noscrub/{ENV[USER]}/DATM +DOCNdir=/work/noaa/{disk_project}/noscrub/{ENV[USER]}/DOCN ## Save directory. Make sure you edit this. CDSAVE=/work/noaa/{disk_project}/save/{ENV[USER]} ## Scrubbed directory for large work files. Make sure you edit this. diff --git a/parm/system.conf.wcoss_cray b/parm/system.conf.wcoss_cray index 3e419b26c..8c502befe 100644 --- a/parm/system.conf.wcoss_cray +++ b/parm/system.conf.wcoss_cray @@ -23,6 +23,8 @@ inputroot=/gpfs/dell1/nco/ops/com/gfs/prod [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/gpfs/hps3/emc/{disk_project}/noscrub/{ENV[USER]}/hafstrak +DATMdir=/gpfs/hps3/emc/{disk_project}/noscrub/{ENV[USER]}/DATM +DOCNdir=/gpfs/hps3/emc/{disk_project}/noscrub/{ENV[USER]}/DOCN ## Scrubbed directory for large work files. Make sure you edit this. CDSCRUB=/gpfs/hps3/ptmp/{ENV[USER]} ## Save directory. Make sure you edit this. diff --git a/parm/system.conf.wcoss_dell_p3 b/parm/system.conf.wcoss_dell_p3 index 47573253f..1cb462243 100644 --- a/parm/system.conf.wcoss_dell_p3 +++ b/parm/system.conf.wcoss_dell_p3 @@ -22,6 +22,8 @@ inputroot=/gpfs/dell1/nco/ops/com/gfs/prod [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. CDNOSCRUB=/gpfs/dell2/emc/{disk_project}/noscrub/{ENV[USER]}/hafstrak +DATMdir=/gpfs/dell2/emc/{disk_project}/noscrub/{ENV[USER]}/DATM +DOCNdir=/gpfs/dell2/emc/{disk_project}/noscrub/{ENV[USER]}/DOCN ## Scrubbed directory for large work files. Make sure you edit this. CDSCRUB=/gpfs/dell2/ptmp/{ENV[USER]} ## Save directory. Make sure you edit this. diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 10611dfd2..2b293fe69 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -257,7 +257,7 @@ &CORES_EXTRA; TOTAL_TASKS1 06:00:00 - &MEMORY; + &MORE_MEMORY; &ENV_VARS; diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 65835b62c..84a503dbf 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -14,6 +14,7 @@ 40"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 7ce849dc3..2f01b5da7 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -14,6 +14,7 @@ 40"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 5d8748b67..ad3dd2e58 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -14,6 +14,7 @@ 40"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index b3edc993f..e7839ad23 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -14,6 +14,7 @@ 24"> 1G"> + 1G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> @@ -23,7 +24,6 @@ 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> ->>>>>>> origin/develop 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 3bd93001f..b45225767 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -14,6 +14,7 @@ 24"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index e46df76af..83557074d 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -14,6 +14,7 @@ 24"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 9caa186c7..f4c1c0ae3 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -14,6 +14,7 @@ 24"> 1G"> + 5G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> diff --git a/scripts/exhafs_docn_download.sh b/scripts/exhafs_docn_download.sh index 5a61fa549..a38c9255b 100755 --- a/scripts/exhafs_docn_download.sh +++ b/scripts/exhafs_docn_download.sh @@ -32,7 +32,19 @@ echo "Start date: ${M1DATE:0:8}" echo "End date: ${P1DATE:0:8}" echo "Destination dir: \"$DOCNdir\"" -"$USHhafs/hafs_docn_download.py" ${M1DATE:0:8}-${P1DATE:0:8} +docn_source=${DOCN_SOURCE:-OISST} + +if [[ "$docn_source" == OISST ]] ; then + "$USHhafs/hafs_oisst_download.py" ${M1DATE:0:8}-${P1DATE:0:8} +elif [[ "${docn_source}" == RTOFS ]] ; then + "$USHhafs/hafs_rtofs_download.py" ${CDATE:0:8} +elif [[ "${docn_source}" == GHRSST ]] ; then + "$USHhafs/hafs_ghrsst_download.py" ${M1DATE:0:8}-${P1DATE:0:8} +else + echo "ERROR: Unknown data ocean source $docn_source. Giving up." 2>&1 + echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DOCN_SOURCE VALUE <- " + exit 1 +fi -echo "Successfully downloaded all OISST files." +echo "Successfully downloaded all DOCN $docn_source files." echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh old mode 100755 new mode 100644 index 460301884..7146432e7 --- a/scripts/exhafs_docn_mesh.sh +++ b/scripts/exhafs_docn_mesh.sh @@ -7,17 +7,7 @@ if [[ "$make_mesh_ocn" != yes || "$run_docn" != yes ]] ; then echo "Beware! You may anger Poseidon by misusing this script. Avoid coastlines." echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " exit 0 -fi - -if ( ! which cdo ) ; then - echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 - exit 1 -fi - -if ( ! which ncwa ) ; then - echo "The \"ncwa\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 - exit 1 -fi +fi set -xe @@ -27,77 +17,59 @@ USHhafs=${USHhafs:-${HOMEhafs}/ush} CDATE=${CDATE:-${YMDH}} APRUNS=${APRUNS:-"aprun -b -j1 -n1 -N1 -d1 -cc depth"} -merged=oisst-avhrr-v02r01.merged.nc -nozlev=oisst-avhrr-v02r01.merged_nozlev.nc +merged=merged.nc ofile=ofile.nc mesh_ocn="$mesh_ocn" mesh_dir=$( dirname "$mesh_ocn" ) +docn_source=${DOCN_SOURCE:-OISST} -# Start & end times are at day precision, not hour -m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) -p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) -now=$m1date -end=$p1date - -set +x -echo "Generating ESMF mesh from OISST files." -echo "Running in dir \"$PWD\"" -echo "OISST Date range is $now to $end" -set -x +[ -d "$mesh_dir" ] || mkdir "$mesh_dir" +rm -f "$mesh_ocn" +test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" -# Generate the filenames. -mergefiles='' -itry=0 -itrytoohard=99 # infinite loop guard -while (( now <= end && itry <= itrytoohard )) ; do - mergefiles="$mergefiles $DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" - now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) - itry=$(( itry+1 )) -done -if (( itry > itrytoohard )) ; then - echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 +if [[ "$docn_source" == OISST ]] ; then + "$USHhafs/hafs_oisst_prep_mesh.sh" "$merged" +elif [[ "${docn_source}" == RTOFS ]] ; then + "$USHhafs/hafs_rtofs_prep_mesh.sh" "$merged" +elif [[ "${docn_source}" == GHRSST ]] ; then + "$USHhafs/hafs_ghrsst_prep_mesh.sh" "$merged" +else + echo "ERROR: Unknown data ocean source $docn_source. Giving up." 2>&1 + echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DOCN_SOURCE VALUE <- " exit 1 fi -set +x -echo "OISST input files are:" -for f in $mergefiles ; do - echo " - $f" -done -echo "Will merge oisst files into $merged" -echo "Will remove z levels into $nozlev" -echo "Temporary output mesh is $ofile" -echo "Will deliver to \"$mesh_ocn\"" -set -x - -# Merge all oisst files into one, as expected by hafs_esmf_mesh.py -cdo mergetime $mergefiles "$merged" test -s "$merged" test -r "$merged" -# Remove z dimension -ncwa -O -a zlev "$merged" "$nozlev" -test -s "$nozlev" -test -r "$nozlev" - -[ -d "$mesh_dir" ] || mkdir "$mesh_dir" -rm -f "$mesh_ocn" -test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" +set +x +echo "Subprocess successfully merged ocean input files." +echo "Will now generate mesh in \"$ofile\"" +echo "Will deliver to \"$mesh_ocn\"" +set -x -# Generate the mesh from the merged oisst file. -$APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$nozlev" --ofile "$ofile" \ - --maskvar sst --maskcal --double --overwrite +# Generate the mesh from the merged file. +if [[ "$docn_source" == OISST ]] ; then + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + --maskvar sst --maskcal --double --overwrite +elif [[ "${docn_source}" == RTOFS ]] ; then + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + --overwrite --latvar Latitude --lonvar Longitude \ + --maskvar sst --maskcal —double +elif [[ "${docn_source}" == GHRSST ]] ; then + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + --maskvar analysed_sst --maskcal --overwrite --double +fi test -s "$ofile" # Copy mesh and merged file to final destinations. $USHhafs/produtil_deliver.py -m "$ofile" "$mesh_ocn" -$USHhafs/produtil_deliver.py -m "$nozlev" "$merged_docn_input" +$USHhafs/produtil_deliver.py -m "$merged" "$merged_docn_input" test -s "$mesh_ocn" ls -l "$mesh_ocn" # Rejoice. set +x -echo "OISST files successfully merged." echo "DOCN mesh was successfully generated." echo "Enjoy your mesh and have a nice day." diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 818bd6e54..89761f527 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -699,15 +699,25 @@ elif [ ${run_docn} = yes ]; then ${NLN} "$DOCNdir"/*.nc INPUT/ fi - ${NCP} ${PARMhafs}/cdeps/docn_in . - ${NCP} ${PARMhafs}/cdeps/docn.streams . - - sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn_in + #${NCP} ${PARMhafs}/cdeps/docn_in . + #${NCP} ${PARMhafs}/cdeps/docn.streams . + docn_source=${DOCN_SOURCE:-OISST} + + # Generate docn_in from template: + ${NCP} ${PARMhafs}/cdeps/docn_in docn_in_template + sed -e "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" \ + -e "s/_nx_global_/$docn_mesh_nx_global/g" \ + -e "s/_ny_global_/$docn_mesh_ny_global/g" \ + < docn_in_template > docn_in + + if [ "$docn_source" == GHRSST ] ; then + ${NLN} "$merged_docn_input" docn/ghrsst_v1.nc + fi + # Generate docn_streams from template specific to the model: + ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn_streams sed -i "s/_yearFirst_/$yr/g" docn.streams - sed -i "s/_yearLast_/$endyr/g" docn.streams - sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams for file in INPUT/oisst*.nc INPUT/sst*.nc ; do if [[ -s "$file" ]] ; then diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index d2e390fdf..8341787cf 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -10,6 +10,6 @@ if [ $target = cheyenne ]; then target=cheyenne.intel ; fi cd hafs_forecast.fd/tests -./compile.sh "$target" "APP=HAFS CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf 32BIT=Y" 32bit YES NO +./compile.sh "$target" "APP=HAFS_DOCN CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf 32BIT=Y" 32bit YES NO exit diff --git a/ush/hafs_ghrsst_download.py b/ush/hafs_ghrsst_download.py new file mode 100755 index 000000000..70dacb359 --- /dev/null +++ b/ush/hafs_ghrsst_download.py @@ -0,0 +1,223 @@ +#! /usr/bin/env python3 + +# This next line will abort in any version earlier than Python 3.6: +f'This script requires Python 3.6 or newer.' + +import time +import subprocess +import contextlib +import os +import tempfile +import getopt +import re +import logging +import datetime +import sys +import random + +try: + import requests +except ImportError as ie: + sys.stderr.write("""You are missing the request module! +You must install it to run this script. + + pip install request --user +""") + +import produtil.setup, produtil.fileop, produtil.locking + +# Constants +UTILITY_NAME = 'ghrsst_downloader' +VERSION_STRING = '0.0.1' +LOGGING_DOMAIN = UTILITY_NAME +CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) +EPSILON = datetime.timedelta(seconds=5) # epsilon for time comparison: five seconds + +# Non-constant globals: +filename_format="JPL-L4_GHRSST-SSTfnd-MUR-GLOB-%Y%m%d" +dayset=set() # list of YYYYMMDD strings +happy=True # False = something failed +url_after_change='https://www.ncei.noaa.gov/data/oceans/ghrsst/L4/GLOB/JPL/MUR25/%Y/%j/%Y%m%d090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc' +url_before_change='https://www.ncei.noaa.gov/data/oceans/ghrsst/L4/GLOB/JPL/MUR25/%Y/%j/%Y%m%d-JPL-L4UHfnd-GLOB-v01-fv04-MUR.nc.bz2' +date_url_changed=datetime.datetime.strptime("2009309","%Y%j") +block_size=65536 + +def usage(why=None): + print(f'''Synopsis: {UTILITY_NAME} [options] day [day [...]] + +Downloads the listed days of data. Days can be specified as: + 20210815 = specify one day: August 15, 2021 + 20210815-20210819 = specify a range of days: August 15th to 19th, 2021 + 2018 = specify an entire year (2018) + +Options: + -q | --quiet = log only warnings and errors + -v | --verbose = log all messages + -F format | --format format = filename format as in strftime(3) + -b N | --block-size N = bytes to download in each block (default {block_size}) + --version = print {UTILITY_NAME} {VERSION_STRING} + --help = this message + +Format example: stuffnthings_%Y%m%d = stuffnthings_20210815 +Script will automatically append ".nc" or ".nc.bz2" +''') + if why: + sys.stderr.write(f'SCRIPT IS ABORTING BECAUSE: {why}\n') + return 1 + return 0 + +def quiet_remove(filename): + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + +class RequestFailed(Exception): + def __init__(self,url,code): + self.url=str(url) + self.code=code + def __str__(self): + return f'requests.get("{self.url!s}") failed with code {self.code!s}' + def __repr__(self): + return f'RequestFailed({self.url!r},{self.code!r})' + +# The meat of the program: retrieve a file +def download_one_day(when): + filename_base = when.strftime(filename_format) + filename_lock = filename_base+'.lock' + + if when>=date_url_changed: + url=when.strftime(url_after_change) + else: + url=when.strftime(url_before_before) + + if url.endswith('.bz2'): + filename_download = filename_base+'.download.nc.bz2' + filename_final = filename_base+'.nc.bz2' + else: + filename_download = filename_base+'.download.nc' + filename_final = filename_base+'.nc' + + if os.path.exists(filename_final): + logger.info(filename_final+': already exists. Skipping.') + return True + + request = None + + with produtil.locking.LockFile(filename_lock,logger): + try: + if os.path.exists(filename_final): + logger.info(filename_final+': already exists (after lock). Skipping. Will pick a random date to avoid lock contention..') + return False + with open(filename_download,'wb') as downloaded: + logger.info(filename_download+' <-- '+str(url)) + request = requests.get(url) + if request.status_code!=200: + raise RequestFailed(url,request.status_code) + for chunk in request.iter_content(block_size): + downloaded.write(chunk) + request.close() + produtil.fileop.deliver_file(filename_download,filename_final,logger=logger, + keep=False,verify=False,moveok=True,force=True) + quiet_remove(filename_download) + quiet_remove(filename_lock) + except Exception as e: + quiet_remove(filename_download) + if request is not None: + request.close() + raise e + return True + +# Parse arguments and initialize logging: +log_level = logging.INFO +optlist,args = getopt.getopt(sys.argv[1:],'qvb:F:',[ + 'version','help','verbose','quiet','block-size','format']) +if len(args)<1: + exit(usage("No arguments provided!")) +for optarg in optlist: + if optarg[0] in ['-q', '--quiet']: + log_level = logging.WARNING + elif optarg[0] in ['-v', '--verbose']: + log_level = logging.DEBUG + elif optarg[0] in ['-F', '--format']: + filename_format = optarg[1] + elif optarg[0] in ['-b', '--block-size' ]: + block_size=max(1,int(optarg[1])) + elif optarg[0]=='--help': + exit(usage()) + elif optarg[0]=='--version': + print(UTILITY_NAME+' '+VERSION_STRING) + exit(0) +logger = logging.getLogger(LOGGING_DOMAIN) +produtil.setup.setup(level=log_level,send_dbn=False) + +# Parse the days. This loop was modified from run_hafs.py: +for arg in args: + if re.match('\A\d{8}\Z',arg): + logger.info('single date/time') + # Single date/time + dayset.add(arg) + elif re.match('\A\d{4}\Z',arg): + logger.info('year') + # Year + start=datetime.datetime(int(arg,10),1,1,0,0,0) + end=datetime.datetime(int(arg,10),12,31,23,59,0) + now=start + while now=len(daylist): + logger.info(f'{day}: sleep for a little while... 30 second snooze...') + time.sleep(30) + logger.info(f'{day}: done sleeping.') + iloop=0 + except Exception as ex: # Unfortunately, cdsapi raises Exception + happy = False + logger.error(f'Download failed for {day}: {ex}',exc_info=ex) + +# Exit 0 on success, 1 on failure: +exit( 0 if happy else 1 ) diff --git a/ush/hafs_docn_download.py b/ush/hafs_oisst_download.py similarity index 100% rename from ush/hafs_docn_download.py rename to ush/hafs_oisst_download.py diff --git a/ush/hafs_oisst_prep_mesh.sh b/ush/hafs_oisst_prep_mesh.sh new file mode 100644 index 000000000..575e9e459 --- /dev/null +++ b/ush/hafs_oisst_prep_mesh.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -xe + +nozlev="${1:-oisst-avhrr-v02r01.merged_nozlev.nc}" + +if ( ! which cdo ) ; then + echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 + exit 1 +fi + +if ( ! which ncwa ) ; then + echo "The \"ncwa\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 + exit 1 +fi + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} + +merged=oisst-avhrr-v02r01.merged.nc +mesh_ocn="$mesh_ocn" +mesh_dir=$( dirname "$mesh_ocn" ) + +# Start & end times are at day precision, not hour +m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) +p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) +now=$m1date +end=$p1date + +set +x +echo "Generating ESMF mesh from OISST files." +echo "Running in dir \"$PWD\"" +echo "OISST Date range is $now to $end" +set -x + +# Generate the filenames. +mergefiles='' +missing='' +itry=0 +itrytoohard=99 # infinite loop guard +while (( now <= end && itry <= itrytoohard )) ; do + infile="$DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" + if [[ ! -s "$infile" || ! -r "$infile" ]] ; then + echo "OISST input file is missing: $infile" 2>&1 + missing="$missing $infile" + else + mergefiles="$mergefiles $infile" + fi + now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) + itry=$(( itry+1 )) +done +if (( itry > itrytoohard )) ; then + echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 + exit 1 +fi + +if [[ "${missing:-}Q" != Q ]] ; then + set +x + echo "You are missing some GHRSST input files!" + for infile in $missing ; do + echo " missing: $infile" + done + echo " -> SCRIPT IS ABORTING BECAUSE INPUT FILES ARE MISSING <- " + exit 1 +fi + +set +x +echo "OISST input files are:" +for f in $mergefiles ; do + echo " - $f" +done +echo "Will merge oisst files into $merged" +echo "Will remove z levels into $nozlev" +set -x + +# Merge all oisst files into one, as expected by hafs_esmf_mesh.py +cdo mergetime $mergefiles "$merged" +test -s "$merged" +test -r "$merged" + +# Remove z dimension +ncwa -O -a zlev "$merged" "$nozlev" +test -s "$nozlev" +test -r "$nozlev" + +# Rejoice. +set +x +echo "Successfully merged OISST files and removed z dimension." diff --git a/ush/hafs_rtofs_download.py b/ush/hafs_rtofs_download.py new file mode 100755 index 000000000..f0678dc54 --- /dev/null +++ b/ush/hafs_rtofs_download.py @@ -0,0 +1,230 @@ +#! /usr/bin/env python3 + +# This next line will abort in any version earlier than Python 3.6: +f'This script requires Python 3.6 or newer.' + +import time +import subprocess +import contextlib +import os +import tempfile +import getopt +import re +import logging +import datetime +import sys +import random + +try: + import requests +except ImportError as ie: + sys.stderr.write("""You are missing the request module! +You must install it to run this script. + + pip install request --user +""") + +import produtil.setup, produtil.fileop, produtil.locking + +# Constants +UTILITY_NAME = 'rtofs_downloader' +VERSION_STRING = '0.0.1' +LOGGING_DOMAIN = UTILITY_NAME +CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) +EPSILON = datetime.timedelta(seconds=5) # epsilon for time comparison: five seconds + +# Non-constant globals: +dayset=set() # list of YYYYMMDD strings +happy=True # False = something failed +filename_format = 'rtofs_glo_2ds_%Y%m%d_f{fhour:03d}' +url_format='https://nomads.ncep.noaa.gov/pub/data/nccf/com/rtofs/prod/rtofs.%Y%m%d/rtofs_glo_2ds_f{fhour:03d}_prog.nc' +block_size=65536 +last_fhour=126 +fhour_interval=3 + +def usage(why=None): + print(f'''Synopsis: {UTILITY_NAME} [options] day [day [...]] + +Downloads the listed days of data. Days can be specified as: + 20210815 = specify one day: August 15, 2021 + 20210815-20210819 = specify a range of days: August 15th to 19th, 2021 + 2018 = specify an entire year (2018) + +Options: + -q | --quiet = log only warnings and errors + -v | --verbose = log all messages + -u https:... | --url https:... = base url with no ending / + default: {base_url} + -F format | --format format = filename format as in strftime(3) + -b N | --block-size N = bytes to download in each block (default {block_size}) + --version = print {UTILITY_NAME} {VERSION_STRING} + --help = this message + +Format example: stuffnthings_%Y%m%d = stuffnthings_20210815 +Script will automatically append ".nc" +''') + + if why: + sys.stderr.write(f'SCRIPT IS ABORTING BECAUSE: {why}\n') + return 1 + return 0 + +def quiet_remove(filename): + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + +class RequestFailed(Exception): + def __init__(self,url,code): + self.url=str(url) + self.code=code + def __str__(self): + return f'requests.get("{self.url!s}") failed with code {self.code!s}' + def __repr__(self): + return f'RequestFailed({self.url!r},{self.code!r})' + +# The meat of the program: retrieve a file +def download_one_hour(when,fhour): + filename_base = when.strftime(filename_format.format(fhour=fhour)) + url = when.strftime(url_format.format(fhour=fhour)) + filename_final = filename_base+'.nc' + if os.path.exists(filename_final): + logger.info(filename_final+': already exists. Skipping.') + return True + + filename_download = filename_base+'_download.nc' + filename_lock = filename_base+'.lock' + request = None + with produtil.locking.LockFile(filename_lock,logger): + try: + if os.path.exists(filename_final): + logger.info(filename_final+': already exists (after lock). Skipping. Will pick a random date to avoid lock contention..') + return False + with open(filename_download,'wb') as downloaded: + logger.info(filename_download+' <-- '+str(url)) + request = requests.get(url) + if request.status_code!=200: + raise RequestFailed(url,request.status_code) + for chunk in request.iter_content(block_size): + downloaded.write(chunk) + request.close() + produtil.fileop.deliver_file(filename_download,filename_final,logger=logger, + keep=False,verify=False,moveok=True,force=True) + quiet_remove(filename_download) + quiet_remove(filename_lock) + except Exception as e: + quiet_remove(filename_download) + if request is not None: + request.close() + raise e + return True +# Parse arguments and initialize logging: +log_level = logging.INFO +optlist,args = getopt.getopt(sys.argv[1:],'qveniu:b:F:',[ + 'version','help','verbose','quiet','block-size','url','format']) +if len(args)<1: + exit(usage("No arguments provided!")) +for optarg in optlist: + if optarg[0] in ['-q', '--quiet']: + log_level = logging.WARNING + elif optarg[0] in ['-v', '--verbose']: + log_level = logging.DEBUG + elif optarg[0] in ['-F', '--format']: + filename_format = optarg[1] + elif optarg[0] in ['-u', '--url' ]: + base_url=optarg[1] + elif optarg[0] in ['-b', '--block-size' ]: + block_size=max(1,int(optarg[1])) + elif optarg[0]=='--help': + exit(usage()) + elif optarg[0]=='--version': + print(UTILITY_NAME+' '+VERSION_STRING) + exit(0) +logger = logging.getLogger(LOGGING_DOMAIN) + +produtil.setup.setup(level=log_level,send_dbn=False) + +# Parse the days. This loop was modified from run_hafs.py: +for arg in args: + if re.match('\A\d{8}\Z',arg): + logger.info('single date/time') + # Single date/time + dayset.add(arg) + elif re.match('\A\d{4}\Z',arg): + logger.info('year') + # Year + start=datetime.datetime(int(arg,10),1,1,0,0,0) + end=datetime.datetime(int(arg,10),12,31,23,59,0) + now=start + while now=len(hourlist): + logger.info(f'{day}: sleep for a little while... 30 second snooze...') + time.sleep(30) + logger.info(f'{day}: done sleeping.') + iloop=0 + except Exception as ex: # Unfortunately, cdsapi raises Exception + happy = False + logger.error(f'Download failed for {day}: {ex}',exc_info=ex) + +# Exit 0 on success, 1 on failure: +exit( 0 if happy else 1 ) diff --git a/ush/hafs_rtofs_prep_mesh.sh b/ush/hafs_rtofs_prep_mesh.sh new file mode 100644 index 000000000..e31515af0 --- /dev/null +++ b/ush/hafs_rtofs_prep_mesh.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -xe + +merged="${1:-merged.nc}" + +for exe in cdo ncks ncrename ncap2 ; do + if ( ! which "$exe" ) ; then + echo "The \"$exe\" command isn't in your path! Go find it and rerun this job." 1>&2 + exit 1 + fi +done + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} + +mesh_ocn="$mesh_ocn" +mesh_dir=$( dirname "$mesh_ocn" ) + +# Start & end times are at day precision, not hour +m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) +p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) +now=$m1date +end=$p1date + +set +x +echo "Generating ESMF mesh from RTOFS files." +echo "Running in dir \"$PWD\"" +echo "Using files from \"$DOCNdir\"" +echo "Running in dir \"$PWD\"" +set -x + +files_to_merge='' +missing='' + +for fhour in $( seq 0 3 $NHRS ) ; do + infile="$DOCNdir/rtofs_glo_2ds_${CDATE:0:8}_f"$( printf %03d $fhour ).nc + if [[ ! -s "$infile" || ! -r "$infile" ]] ; then + echo "RTOFS input file is missing: $infile" 2>&1 + missing="$missing $infile" + else + files_to_merge="$files_to_merge $infile" + fi +done + +if [[ "${missing:-}Q" != Q ]] ; then + set +x + echo "You are missing some RTOFS input files!" + for infile in $missing ; do + echo " missing: $infile" + done + echo " -> SCRIPT IS ABORTING BECAUSE INPUT FILES ARE MISSING <- " + exit 1 +fi + +set +x +echo "RTOFS input files are:" +for infile in $files_to_merge ; do + echo " - $infile" +done +echo "Will merge RTOFS files into $merged" +set -x + +# Merge files +cdo mergetime $files_to_merge rtofs_glo_2ds_prog.nc + +# Subset data over HAFS region (lat, 250 to 355.0732 and lon, 0 to 50.0108) +ncks -O -d X,2198,3511 -d Y,1504,2228 rtofs_glo_2ds_prog.nc rtofs_glo_2ds_prog_v1.nc + +# Keep only required variables +ncks -O -v sst,Latitude,Longitude rtofs_glo_2ds_prog_v1.nc rtofs_glo_2ds_prog_v2.nc +rm -f rtofs_glo_2ds_prog_v1.nc +ncks -O -C -x -v Date rtofs_glo_2ds_prog_v2.nc rtofs_glo_2ds_prog_v3.nc +rm -f rtofs_glo_2ds_prog_v2.nc + +# Rename variables to make them CDEPS compatible. +# We have a workaround in here because I once tried to change a +# variable name "MT" to "time" in the data, and it was set to the +# missing value. This could be a bug in the NetCDF Operators, but the +# following code produces a correct file: +ncrename -d MT,time rtofs_glo_2ds_prog_v3.nc +ncap2 -O -v -s 'time=MT' rtofs_glo_2ds_prog_v3.nc rtofs_glo_2ds_prog_v4.nc +rm -f rtofs_glo_2ds_prog_v3.nc +ncks -A -C -v time rtofs_glo_2ds_prog_v4.nc rtofs_glo_2ds_prog_v5.nc +rm -f rtofs_glo_2ds_prog_v4.nc +ncks -x -v MT rtofs_glo_2ds_prog_v5.nc "$merged" +rm -f rtofs_glo_2ds_prog_v5.nc + +# Rejoice. +set +x +echo "RTOFS files successfully merged." From d8a2c4af5b9ac46c510dd04f62ef730114ad200b Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 14 Jul 2021 12:32:12 -0500 Subject: [PATCH 44/73] Allow the choice between several DOCN model sources. Presently only OISST is confirmed to work though. --- parm/cdeps/docn_ghrsst.streams | 15 ++++ .../{docn.streams => docn_oisst.streams} | 0 parm/hafs_docn_ghrsst.conf | 7 ++ parm/hafs_docn_oisst.conf | 7 ++ parm/hafs_holdvars.conf | 3 + parm/hafs_holdvars.txt | 2 + rocoto/cronjob_hafs_docn.sh | 63 +++++++++++++ rocoto/sites/hera.ent | 2 +- rocoto/sites/kjet.ent | 2 +- rocoto/sites/orion.ent | 2 +- rocoto/sites/wcoss_cray.ent | 2 +- rocoto/sites/wcoss_dell_p3.ent | 2 +- rocoto/sites/xjet.ent | 2 +- rocoto/sites/xjet_hafsv0p2a.ent | 2 +- scripts/exhafs_docn_mesh.sh | 0 scripts/exhafs_forecast.sh | 5 +- ush/hafs_ghrsst_prep_mesh.sh | 88 +++++++++++++++++++ ush/hafs_oisst_prep_mesh.sh | 0 18 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 parm/cdeps/docn_ghrsst.streams rename parm/cdeps/{docn.streams => docn_oisst.streams} (100%) create mode 100644 parm/hafs_docn_ghrsst.conf create mode 100644 parm/hafs_docn_oisst.conf create mode 100755 rocoto/cronjob_hafs_docn.sh mode change 100644 => 100755 scripts/exhafs_docn_mesh.sh create mode 100755 ush/hafs_ghrsst_prep_mesh.sh mode change 100644 => 100755 ush/hafs_oisst_prep_mesh.sh diff --git a/parm/cdeps/docn_ghrsst.streams b/parm/cdeps/docn_ghrsst.streams new file mode 100644 index 000000000..cb8492ce0 --- /dev/null +++ b/parm/cdeps/docn_ghrsst.streams @@ -0,0 +1,15 @@ +stream_info: PRESCRIBED01 +taxmode01: limit +mapalgo01: bilinear +tInterpAlgo01: linear +readMode01: single +dtlimit01: 1.5 +stream_offset01: 0 +yearFirst01: _yearFirst_ +yearLast01: _yearLast_ +yearAlign01: 1 +stream_vectors01: "null" +stream_mesh_file01: "INPUT/DOCN_ESMF_mesh.nc" +stream_lev_dimname01: null +stream_data_files01: "ghrsst_v1.nc" +stream_data_variables01: "analysed_sst So_t" diff --git a/parm/cdeps/docn.streams b/parm/cdeps/docn_oisst.streams similarity index 100% rename from parm/cdeps/docn.streams rename to parm/cdeps/docn_oisst.streams diff --git a/parm/hafs_docn_ghrsst.conf b/parm/hafs_docn_ghrsst.conf new file mode 100644 index 000000000..a15d5d3b0 --- /dev/null +++ b/parm/hafs_docn_ghrsst.conf @@ -0,0 +1,7 @@ +[config] +docn_source=GHRSST + +[forecast] +docn_mesh_nx_global=10501 +docn_mesh_ny_global=5001 +mesh_ocn_in={DOCNdir}/mesh/GHRSST_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no diff --git a/parm/hafs_docn_oisst.conf b/parm/hafs_docn_oisst.conf new file mode 100644 index 000000000..24155a833 --- /dev/null +++ b/parm/hafs_docn_oisst.conf @@ -0,0 +1,7 @@ +[config] +docn_source=OISST + +[forecast] +docn_mesh_nx_global=1440 +docn_mesh_ny_global=720 +mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 386df2fde..f94ba1702 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -35,6 +35,9 @@ ENVEQUIV={WHERE_AM_I} ;; Present cluster name #WHERE_AM_I={ENV[ENVEQUIV]} DOCN_SOURCE={docn_source} +docn_mesh_nx_global={forecast/docn_mesh_nx_global} +docn_mesh_ny_global={forecast/docn_mesh_ny_global} + CASE={grid/CASE} LEVS={grid/LEVS} gtype={grid/gtype} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 320b6a8b4..8d508c19b 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -53,6 +53,8 @@ export datm_download_jobs={datm_download_jobs} export make_mesh_atm={make_mesh_atm} export mesh_atm={mesh_atm} +export docn_mesh_nx_global={docn_mesh_nx_global} +export docn_mesh_ny_global={docn_mesh_ny_global} export docn_download_jobs={docn_download_jobs} export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh new file mode 100755 index 000000000..8c777822e --- /dev/null +++ b/rocoto/cronjob_hafs_docn.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/zrtrr/strahan/HAFSTwoDOCN + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== +# Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh + + # ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + # config.SUBEXPT=${EXPT}_ghrsst \ + # forecast.output_history=.true. \ + # ../parm/hafs_regional_static.conf \ + # ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf + + ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + config.SUBEXPT=${EXPT}_oisst \ + forecast.output_history=.true. \ + ../parm/hafs_regional_static.conf \ + ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf + +# Run all cycles of a storm +#${PYTHON3} ./run_hafs.py ${dev} 2020 13L HISTORY config.EXPT=${EXPT} # Laura + +# Run specified cycles of a storm +#${PYTHON3} ./run_hafs.py ${dev} 2020082506-2020082512 13L HISTORY \ +# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} # Laura + +# Run one cycle of a storm +# ${PYTHON3} ./run_hafs.py -t ${dev} 2020082512 13L HISTORY config.EXPT=${EXPT} + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 84a503dbf..951febd49 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -16,7 +16,7 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 2f01b5da7..e7b23144d 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -16,7 +16,7 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index ad3dd2e58..0d63ed065 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -16,7 +16,7 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index e7839ad23..7e9ecda78 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -16,7 +16,7 @@ 1G"> 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=12:tpp=1TOTAL_TASKS360NCTSK12OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index b45225767..e4dee142d 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -17,7 +17,7 @@ 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 1:ppn=6:tpp=1TOTAL_TASKS6NCTSK6OMP_THREADS4-R affinity[core\(4\):distribute=balance]00:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 83557074d..4cae457a0 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -16,7 +16,7 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index f4c1c0ae3..95491c529 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -16,7 +16,7 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=24:tpp=1TOTAL_TASKS720NCTSK24OMP_THREADS101:30:00"> diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh old mode 100644 new mode 100755 diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 89761f527..1ee7f71be 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -711,15 +711,16 @@ elif [ ${run_docn} = yes ]; then < docn_in_template > docn_in if [ "$docn_source" == GHRSST ] ; then + [ -d docn ] || mkdir docn ${NLN} "$merged_docn_input" docn/ghrsst_v1.nc fi # Generate docn_streams from template specific to the model: - ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn_streams + ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn.streams sed -i "s/_yearFirst_/$yr/g" docn.streams sed -i "s/_yearLast_/$endyr/g" docn.streams sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams - for file in INPUT/oisst*.nc INPUT/sst*.nc ; do + for file in INPUT/oisst*.nc INPUT/sst*.nc INPUT/DOCN_input_merged.nc ; do if [[ -s "$file" ]] ; then sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$(basename $file)/" docn.streams fi diff --git a/ush/hafs_ghrsst_prep_mesh.sh b/ush/hafs_ghrsst_prep_mesh.sh new file mode 100755 index 000000000..ec0236e77 --- /dev/null +++ b/ush/hafs_ghrsst_prep_mesh.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -xe + +subset="${1:-ghrsst_subset.nc}" +merged=ghrsst_merged.nc + +if ( ! which cdo ) ; then + echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 + exit 1 +fi + +if ( ! which ncks ) ; then + echo "The \"ncks\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 + exit 1 +fi + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} + +# Start & end times are at day precision, not hour +m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) +p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) +now=$m1date +end=$p1date + +set +x +echo "Generating ESMF mesh from GHRSST files." +echo "Running in dir \"$PWD\"" +echo "Using files from \"$DOCNdir\"" +echo "GHRSST Date range is $now to $end" +set -x + +# Generate the filenames. +missing='' +mergefiles='' +itry=0 +itrytoohard=99 # infinite loop guard +while (( now <= end && itry <= itrytoohard )) ; do + infile="$DOCNdir/JPL-L4_GHRSST-SSTfnd-MUR-GLOB-${now:0:8}.nc" + if [[ ! -s "$infile" || ! -r "$infile" ]] ; then + echo "GHRSST input file is missing: $infile" 2>&1 + missing="$missing $infile" + else + mergefiles="$mergefiles $infile" + fi + now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) + itry=$(( itry+1 )) +done +if (( itry > itrytoohard )) ; then + echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 + exit 1 +fi + +if [[ "${missing:-}Q" != Q ]] ; then + set +x + echo "You are missing some GHRSST input files!" + for infile in $missing ; do + echo " missing: $infile" + done + echo " -> SCRIPT IS ABORTING BECAUSE INPUT FILES ARE MISSING <- " + exit 1 +fi + +set +x +echo "GHRSST input files are:" +for f in $mergefiles ; do + echo " - $f" +done +echo "Will merge ghrsst files into $merged" + echo "Will subset data over HAFS region into $subset" +set -x + +# Merge all ghrsst files into one, as expected by hafs_esmf_mesh.py +cdo mergetime $mergefiles "$merged" +test -s "$merged" +test -r "$merged" + +# Subset data over HAFS region (lat, 250 to 355 and lon, 0 to 50) +ncks -O -d lon,6999,17499 -d lat,8999,13999 "$merged" "$subset" +test -s "$subset" +test -r "$subset" + +# Rejoice. +set +x +echo "Successfully merged GHRSST files and subsetted area." diff --git a/ush/hafs_oisst_prep_mesh.sh b/ush/hafs_oisst_prep_mesh.sh old mode 100644 new mode 100755 From 292c573379c213d44c8cb057b7880946f1012654 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 16 Jul 2021 10:36:16 -0500 Subject: [PATCH 45/73] Make DATM configurable via datm_source. Fix scripts so one can run without docn. Correct some comments and usage messages. --- parm/hafs.conf | 3 + parm/hafs_datm.conf | 2 + parm/hafs_datm_era5.conf | 13 ++++ parm/hafs_docn.conf | 2 +- parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 1 + rocoto/cronjob_hafs_datm.sh | 63 +++++++++++++++++++ rocoto/hafs_workflow.xml.in | 4 ++ scripts/exhafs_datm_download.sh | 13 +++- scripts/exhafs_datm_mesh.sh | 17 +++-- sorc/build_forecast.sh | 7 +++ ...datm_download.py => hafs_era5_download.py} | 2 +- ush/hafs_ghrsst_download.py | 2 +- ush/hafs_oisst_download.py | 2 +- ush/hafs_rtofs_download.py | 4 +- 15 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 parm/hafs_datm_era5.conf create mode 100755 rocoto/cronjob_hafs_datm.sh rename ush/{hafs_datm_download.py => hafs_era5_download.py} (99%) diff --git a/parm/hafs.conf b/parm/hafs.conf index 99d8f9bff..e7946380d 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -27,6 +27,9 @@ ENS=99 ;; The ensemble number (placeholder) # Specifies a section (default: [hafsdata]) to use: hafsdata, wcoss_fcst_nco input_catalog=fcst_{GFSVER} +docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, RTOFS, or NONE) +datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE) + ## Configure file and directory paths [dir] HOMEhafs={CDSAVE}/{EXPT} diff --git a/parm/hafs_datm.conf b/parm/hafs_datm.conf index 8c4c8d4d1..a4723d406 100644 --- a/parm/hafs_datm.conf +++ b/parm/hafs_datm.conf @@ -9,6 +9,8 @@ datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no +; A second conf file sets this: datm_source=ERA5 + ;; Make sure the atmospheric initialization system is disabled run_vortexinit=no ;; vortex initialization run_gsi_vr=no ;; GSI based vortex relocation diff --git a/parm/hafs_datm_era5.conf b/parm/hafs_datm_era5.conf new file mode 100644 index 000000000..206ba78a1 --- /dev/null +++ b/parm/hafs_datm_era5.conf @@ -0,0 +1,13 @@ +[config] +datm_source=ERA5 + +[forecast] +layoutx=10 +layouty=8 +write_groups=1 +write_tasks_per_group=40 +ocean_tasks=120 + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 + diff --git a/parm/hafs_docn.conf b/parm/hafs_docn.conf index 6c476de54..e754d2668 100644 --- a/parm/hafs_docn.conf +++ b/parm/hafs_docn.conf @@ -6,7 +6,7 @@ run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing -docn_source=OISST ;; OISST, RTOFS, or GHRSST +# A second file sets this option: docn_source=OISST ;; OISST, RTOFS, or GHRSST scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index f94ba1702..ef3a5749f 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -34,6 +34,7 @@ ENVEQUIV={WHERE_AM_I} ;; Present cluster name #JET_NAME={ENV[JET_NAME]} #WHERE_AM_I={ENV[ENVEQUIV]} DOCN_SOURCE={docn_source} +DATM_SOURCE={datm_source} docn_mesh_nx_global={forecast/docn_mesh_nx_global} docn_mesh_ny_global={forecast/docn_mesh_ny_global} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 8d508c19b..8c4f68557 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -60,6 +60,7 @@ export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} export merged_docn_input={merged_docn_input} export DOCN_SOURCE={DOCN_SOURCE} +export DATM_SOURCE={DATM_SOURCE} export SYNDAThafs={syndat} export ADECKhafs={ADECKhafs} diff --git a/rocoto/cronjob_hafs_datm.sh b/rocoto/cronjob_hafs_datm.sh new file mode 100755 index 000000000..9c8f0ffbf --- /dev/null +++ b/rocoto/cronjob_hafs_datm.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/zrtrr/strahan/HAFSTwoDOCN + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== +# Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh + + # ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + # config.SUBEXPT=${EXPT}_ghrsst \ + # forecast.output_history=.true. \ + # ../parm/hafs_regional_static.conf \ + # ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf + + ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + config.SUBEXPT=${EXPT}_era5 \ + forecast.output_history=.true. \ + ../parm/hafs_regional_static.conf ../parm/hafs_hycom.conf \ + ../parm/hafs_datm.conf ../parm/hafs_datm_era5.conf + +# Run all cycles of a storm +#${PYTHON3} ./run_hafs.py ${dev} 2020 13L HISTORY config.EXPT=${EXPT} # Laura + +# Run specified cycles of a storm +#${PYTHON3} ./run_hafs.py ${dev} 2020082506-2020082512 13L HISTORY \ +# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} # Laura + +# Run one cycle of a storm +# ${PYTHON3} ./run_hafs.py -t ${dev} 2020082512 13L HISTORY config.EXPT=${EXPT} + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 2b293fe69..412b10e42 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -603,6 +603,10 @@ &JOBhafs;/JHAFS_DATM_MESH hafs_datm_mesh_&SID;_@Y@m@d@H &WORKhafs;/hafs_datm_mesh.log + &ACCOUNT; + &RESERVATION; + &QUEUE_PE; + &PE_EXTRA; &MAKE_ESMF_MESH_RESOURCES; &ENV_VARS; diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh index a3a17455d..f74ca49fe 100755 --- a/scripts/exhafs_datm_download.sh +++ b/scripts/exhafs_datm_download.sh @@ -39,7 +39,16 @@ echo "Start date: ${M1DATE:0:8}" echo "End date: ${P1DATE:0:8}" echo "Destination dir: \"$DATMdir\"" -"$USHhafs/hafs_datm_download.py" ${M1DATE:0:8}-${P1DATE:0:8} +datm_source=${DATM_SOURCE:-ERA5} -echo "Successfully downloaded all ERA5 files and reversed their latitudes." +if [[ "$datm_source" == ERA5 ]] ; then + "$USHhafs/hafs_era5_download.py" ${M1DATE:0:8}-${P1DATE:0:8} +else + echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 + echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DATM_SOURCE VALUE <- " + exit 1 +fi + + +echo "Successfully downloaded all DATM $datm_source files." echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_datm_mesh.sh b/scripts/exhafs_datm_mesh.sh index e0525b3c7..e186fb188 100755 --- a/scripts/exhafs_datm_mesh.sh +++ b/scripts/exhafs_datm_mesh.sh @@ -35,16 +35,23 @@ rm -f "$mesh_atm" [ -e "$ofile" -o -L "$ofile" ]] && rm -f "$ofile" set +x -echo "Generating ESMF mesh from ERA5 files." +echo "Generating ESMF mesh from $datm_source files." echo "Running in dir \"$PWD\"" echo "ERA5 grid generation input file is \"$ifile\"" echo "Temporary output mesh is $ofile" echo "Will deliver to \"$mesh_atm\"" set -x -# Generate the mesh from the ERA5 file: -$APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ - --overwrite --latvar latitude --lonvar longitude --double +# Generate the mesh from the merged file: +datm_source=${DATM_SOURCE:-ERA5} +if [[ "$datm_source" == ERA5 ]] ; then + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ + --overwrite --latvar latitude --lonvar longitude --double +else + echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 + echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DATM_SOURCE VALUE <- " + exit 1 +fi test -s "$ofile" # Copy mesh to final destination. @@ -55,5 +62,5 @@ ls -l "$mesh_atm" # Rejoice. set +x -echo "DATM mesh was successfully generated." +echo "DATM $datm_source mesh was successfully generated." echo "Enjoy your mesh and have a nice day." diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 8341787cf..abed3b155 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -10,6 +10,13 @@ if [ $target = cheyenne ]; then target=cheyenne.intel ; fi cd hafs_forecast.fd/tests +# For Data Atmosphere (DATM): +#./compile.sh "$target" "APP=HAFS_DATM CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf 32BIT=Y" 32bit YES NO + +# For Data Ocean (DOCN): ./compile.sh "$target" "APP=HAFS_DOCN CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf 32BIT=Y" 32bit YES NO +# For normal HAFS: +#./compile.sh "$target" "APP=HAFS CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf 32BIT=Y" 32bit YES NO + exit diff --git a/ush/hafs_datm_download.py b/ush/hafs_era5_download.py similarity index 99% rename from ush/hafs_datm_download.py rename to ush/hafs_era5_download.py index 4ed6138ad..f650d1478 100755 --- a/ush/hafs_datm_download.py +++ b/ush/hafs_era5_download.py @@ -29,7 +29,7 @@ import produtil.setup, produtil.fileop, produtil.locking # Constants -UTILITY_NAME = 'era5_downloader' +UTILITY_NAME = 'hafs_era5_download' VERSION_STRING = '0.0.1' LOGGING_DOMAIN = UTILITY_NAME DATASET = 'reanalysis-era5-single-levels' diff --git a/ush/hafs_ghrsst_download.py b/ush/hafs_ghrsst_download.py index 70dacb359..9f8cded7b 100755 --- a/ush/hafs_ghrsst_download.py +++ b/ush/hafs_ghrsst_download.py @@ -27,7 +27,7 @@ import produtil.setup, produtil.fileop, produtil.locking # Constants -UTILITY_NAME = 'ghrsst_downloader' +UTILITY_NAME = 'hafs_ghrsst_download' VERSION_STRING = '0.0.1' LOGGING_DOMAIN = UTILITY_NAME CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) diff --git a/ush/hafs_oisst_download.py b/ush/hafs_oisst_download.py index 44d4bec71..cd1afca99 100755 --- a/ush/hafs_oisst_download.py +++ b/ush/hafs_oisst_download.py @@ -27,7 +27,7 @@ import produtil.setup, produtil.fileop, produtil.locking # Constants -UTILITY_NAME = 'oisst_downloader' +UTILITY_NAME = 'hafs_oisst_download' VERSION_STRING = '0.0.1' LOGGING_DOMAIN = UTILITY_NAME CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) diff --git a/ush/hafs_rtofs_download.py b/ush/hafs_rtofs_download.py index f0678dc54..0dde0a3e4 100755 --- a/ush/hafs_rtofs_download.py +++ b/ush/hafs_rtofs_download.py @@ -18,7 +18,7 @@ try: import requests except ImportError as ie: - sys.stderr.write("""You are missing the request module! + sys.stderr.write("""You are missing the "request" module! You must install it to run this script. pip install request --user @@ -27,7 +27,7 @@ import produtil.setup, produtil.fileop, produtil.locking # Constants -UTILITY_NAME = 'rtofs_downloader' +UTILITY_NAME = 'hafs_rtofs_download' VERSION_STRING = '0.0.1' LOGGING_DOMAIN = UTILITY_NAME CYCLING_INTERVAL = datetime.timedelta(seconds=3600*24) From 4630d3ba4e742dc3587d298a5210154201fe0546 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 16 Jul 2021 12:26:48 -0500 Subject: [PATCH 46/73] Use a python installation that already has the needed modules installed --- modulefiles/modulefile.hafs.orion | 4 +++- rocoto/cronjob_hafs_docn.sh | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modulefiles/modulefile.hafs.orion b/modulefiles/modulefile.hafs.orion index 37a1efd35..4b62b939f 100644 --- a/modulefiles/modulefile.hafs.orion +++ b/modulefiles/modulefile.hafs.orion @@ -62,7 +62,9 @@ module load nco/4.9.3 module load rocoto/1.3.3 -module load intelpython3/2020 +#module load intelpython3/2020 +module use /work/noaa/zrtrr/strahan/modulefiles +module load python/3.7.10-hafs # For CDEPS data models: module load cdo diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh index 8c777822e..9b9d69b08 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_docn.sh @@ -18,9 +18,9 @@ date #PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion - HOMEhafs=/work/noaa/zrtrr/strahan/HAFSTwoDOCN + HOMEhafs=/work/noaa/zrtrr/strahan/HAFSOtherPython dev="-s sites/orion.ent -f" - PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python # NOAA RDHPCS Hera #HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS @@ -41,7 +41,7 @@ EXPT=$(basename ${HOMEhafs}) # ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - config.SUBEXPT=${EXPT}_oisst \ + config.SUBEXPT=${EXPT}_py \ forecast.output_history=.true. \ ../parm/hafs_regional_static.conf \ ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf From 1650a740f93e0f9ceab6904d591272012174eae6 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 21 Jul 2021 10:44:30 -0500 Subject: [PATCH 47/73] DOCN GHRSST works on Orion --- parm/cdeps/docn_ghrsst.streams | 4 ++-- parm/hafs_docn_ghrsst.conf | 5 +++-- rocoto/cronjob_hafs_datm.sh | 4 ++-- rocoto/cronjob_hafs_docn.sh | 18 ++++++++++-------- rocoto/sites/hera.ent | 1 + rocoto/sites/kjet.ent | 1 + rocoto/sites/orion.ent | 2 ++ rocoto/sites/wcoss_cray.ent | 1 + rocoto/sites/wcoss_dell_p3.ent | 1 + rocoto/sites/xjet.ent | 1 + rocoto/sites/xjet_hafsv0p2a.ent | 1 + ush/hafs_ghrsst_prep_mesh.sh | 18 +++++++++++++----- 12 files changed, 38 insertions(+), 19 deletions(-) diff --git a/parm/cdeps/docn_ghrsst.streams b/parm/cdeps/docn_ghrsst.streams index cb8492ce0..afff5191c 100644 --- a/parm/cdeps/docn_ghrsst.streams +++ b/parm/cdeps/docn_ghrsst.streams @@ -7,9 +7,9 @@ dtlimit01: 1.5 stream_offset01: 0 yearFirst01: _yearFirst_ yearLast01: _yearLast_ -yearAlign01: 1 +yearAlign01: _yearFirst_ stream_vectors01: "null" stream_mesh_file01: "INPUT/DOCN_ESMF_mesh.nc" stream_lev_dimname01: null -stream_data_files01: "ghrsst_v1.nc" +stream_data_files01: stream_data_variables01: "analysed_sst So_t" diff --git a/parm/hafs_docn_ghrsst.conf b/parm/hafs_docn_ghrsst.conf index a15d5d3b0..55b9ebdac 100644 --- a/parm/hafs_docn_ghrsst.conf +++ b/parm/hafs_docn_ghrsst.conf @@ -2,6 +2,7 @@ docn_source=GHRSST [forecast] -docn_mesh_nx_global=10501 -docn_mesh_ny_global=5001 +docn_mesh_nx_global=11301 +docn_mesh_ny_global=7501 mesh_ocn_in={DOCNdir}/mesh/GHRSST_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no +ocean_tasks=80 diff --git a/rocoto/cronjob_hafs_datm.sh b/rocoto/cronjob_hafs_datm.sh index 9c8f0ffbf..6c28d6ac6 100755 --- a/rocoto/cronjob_hafs_datm.sh +++ b/rocoto/cronjob_hafs_datm.sh @@ -18,9 +18,9 @@ date #PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion - HOMEhafs=/work/noaa/zrtrr/strahan/HAFSTwoDOCN + HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS dev="-s sites/orion.ent -f" - PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python # NOAA RDHPCS Hera #HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh index 9b9d69b08..c3517ca2c 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_docn.sh @@ -18,7 +18,7 @@ date #PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion - HOMEhafs=/work/noaa/zrtrr/strahan/HAFSOtherPython + HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS dev="-s sites/orion.ent -f" PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python @@ -34,17 +34,19 @@ EXPT=$(basename ${HOMEhafs}) #=============================================================================== # Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh - # ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - # config.SUBEXPT=${EXPT}_ghrsst \ - # forecast.output_history=.true. \ - # ../parm/hafs_regional_static.conf \ - # ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf +# Run data ocean with OISST +# ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ +# config.SUBEXPT=${EXPT}_oisst \ +# forecast.output_history=.true. \ +# ../parm/hafs_regional_static.conf \ +# ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf +# Run data ocean with GHRSST ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - config.SUBEXPT=${EXPT}_py \ + config.SUBEXPT=${EXPT}_ghrsst \ forecast.output_history=.true. \ ../parm/hafs_regional_static.conf \ - ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf + ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf # Run all cycles of a storm #${PYTHON3} ./run_hafs.py ${dev} 2020 13L HISTORY config.EXPT=${EXPT} # Laura diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 5733cf4b0..681540cd8 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -63,6 +63,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index e7b23144d..080a6dea2 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -60,6 +60,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 0d63ed065..ca47a1dc7 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -60,6 +60,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> @@ -85,6 +86,7 @@ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20&FORECAST_EXTRA;"> 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index 7e9ecda78..9d4639670 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -61,6 +61,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index e4dee142d..89522cacd 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -62,6 +62,7 @@ 43:ppn=12:tpp=1TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=1TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=1TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=1TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=1TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 4cae457a0..eaa0f81f7 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -61,6 +61,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 95491c529..690bf906a 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -61,6 +61,7 @@ 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> diff --git a/ush/hafs_ghrsst_prep_mesh.sh b/ush/hafs_ghrsst_prep_mesh.sh index ec0236e77..8459346dd 100755 --- a/ush/hafs_ghrsst_prep_mesh.sh +++ b/ush/hafs_ghrsst_prep_mesh.sh @@ -3,7 +3,6 @@ set -xe subset="${1:-ghrsst_subset.nc}" -merged=ghrsst_merged.nc if ( ! which cdo ) ; then echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 @@ -74,15 +73,24 @@ echo "Will merge ghrsst files into $merged" set -x # Merge all ghrsst files into one, as expected by hafs_esmf_mesh.py -cdo mergetime $mergefiles "$merged" -test -s "$merged" -test -r "$merged" +cdo mergetime $mergefiles ghrsst_v1.nc # Subset data over HAFS region (lat, 250 to 355 and lon, 0 to 50) -ncks -O -d lon,6999,17499 -d lat,8999,13999 "$merged" "$subset" +cdo -sellonlatbox,-118,-5,-15.0,60.0 ghrsst_v1.nc ghrsst_v2.nc + +# Convert temperature units: +aos_old=`ncdump -c ghrsst_v2.nc | grep add_offset | grep analysed_sst | awk '{print $3}'` +aos_new=$(echo "scale=3; $aos_old-273.15" | bc) +ncatted -O -a add_offset,analysed_sst,o,f,"$aos_new" ghrsst_v2.nc + +# Deliver file: +rm -f "$subset" +cp -fp ghrsst_v2.nc "$subset" test -s "$subset" test -r "$subset" +#ncks -O -d lon,6999,17499 -d lat,8999,13999 "$merged" ghrsst_v1.nc + # Rejoice. set +x echo "Successfully merged GHRSST files and subsetted area." From df14e2af6e36e4d015249550e8a217cc09cbc592 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Thu, 22 Jul 2021 19:34:14 +0000 Subject: [PATCH 48/73] Missed a "-p" in mkdir for DOCNdir and DATMdir --- scripts/exhafs_datm_download.sh | 2 +- scripts/exhafs_docn_download.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh index f74ca49fe..3b7bc051a 100755 --- a/scripts/exhafs_datm_download.sh +++ b/scripts/exhafs_datm_download.sh @@ -30,7 +30,7 @@ export TZ=UTC # Orion workaround M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) -[ -d "$DATMdir" ] || mkdir "$DATMdir" +[ -d "$DATMdir" ] || mkdir -p "$DATMdir" test -d "$DATMdir" cd "$DATMdir" diff --git a/scripts/exhafs_docn_download.sh b/scripts/exhafs_docn_download.sh index a38c9255b..7bef09715 100755 --- a/scripts/exhafs_docn_download.sh +++ b/scripts/exhafs_docn_download.sh @@ -23,7 +23,7 @@ export TZ=UTC # Orion workaround M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) -[ -d "$DOCNdir" ] || mkdir "$DOCNdir" +[ -d "$DOCNdir" ] || mkdir -p "$DOCNdir" test -d "$DOCNdir" cd "$DOCNdir" From 01005cb19a22000e05afb705090c4f6e78ac018f Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Fri, 23 Jul 2021 16:07:32 +0000 Subject: [PATCH 49/73] link the new fix_cdeps directory --- sorc/link_fix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh index bbe75f642..ecc5b1a63 100755 --- a/sorc/link_fix.sh +++ b/sorc/link_fix.sh @@ -24,7 +24,7 @@ else exit 1 fi -for subdir in fix_am fix_orog fix_fv3_gmted2010 fix_sfc_climo fix_hycom hwrf-crtm-2.2.6; +for subdir in fix_am fix_orog fix_fv3_gmted2010 fix_sfc_climo fix_hycom hwrf-crtm-2.2.6 fix_cdeps; do ln -sf ${FIXROOT}/${subdir} ./ done From 8e088ba04fb1b10b7e120567d767dbe2e11f0ab5 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 27 Jul 2021 13:50:36 -0500 Subject: [PATCH 50/73] Use fix-file meshes by default for DOCN. (DATM not yet tested this way) --- parm/hafs.conf | 12 +++++++----- parm/hafs_datm.conf | 2 +- parm/hafs_datm_era5.conf | 1 + parm/hafs_docn.conf | 2 +- parm/hafs_docn_ghrsst.conf | 1 + parm/hafs_docn_oisst.conf | 1 + rocoto/hafs_workflow.xml.in | 6 +----- scripts/exhafs_docn_mesh.sh | 23 ++++++++++++++++++----- scripts/exhafs_forecast.sh | 18 +++++++----------- 9 files changed, 38 insertions(+), 28 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index 2a2036725..2b70a9bfd 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -27,7 +27,7 @@ ENS=99 ;; The ensemble number (placeholder) # Specifies a section (default: [hafsdata]) to use: hafsdata, wcoss_fcst_nco input_catalog=fcst_{GFSVER} -docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, RTOFS, or NONE) +docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, or NONE) datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE) ## Configure file and directory paths @@ -60,6 +60,8 @@ gsistatus2=gsi_status.{vit[stormname]}{vit[stnum]:02d}{vit[basin1lc]}.{cycle} PARMforecast={PARMhafs}/forecast/regional ;; The location where the forecast job will find its parm and namelist files PARMgsi={PARMhafs}/hafs-gsi/ ;; GSI input data for everything except CRTM FIXcrtm={FIXhafs}/hafs-crtm-2.2.3/ ;; GSI CRTM input data +FIXcdeps={FIXhafs}/fix_cdeps ;; CDEPS fix files +FIXmeshes={FIXcdeps}/meshes ;; premade CDEPS meshes utilexec={HOMEhafs}/exec ;; utility exe location (placeholder) @@ -281,9 +283,9 @@ merge_import=.true. # Data model defaults merged_docn_input={WORKhafs}/intercom/DOCN_input_merged.nc -mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no +mesh_ocn_in=missing ;; premade mesh to use if make_mesh_ocn=no mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change -mesh_atm_in={DATMdir}/../static/TL639_200618_ESMFmesh.nc ;; premade mesh to use if make_mesh_atm=no +mesh_atm_in=missing ;; premade mesh to use if make_mesh_atm=no mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction @@ -416,6 +418,6 @@ RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? -DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM ERA5 files? +DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM files? MAKE_MESH_OCN={make_mesh_ocn} ;; Should the DOCN mesh be generated by the workflow? -DOWNLOAD_DOCN={download_docn} ;; Should the workflow download DOCN OISST files? +DOWNLOAD_DOCN={download_docn} ;; Should the workflow download DOCN files? \ No newline at end of file diff --git a/parm/hafs_datm.conf b/parm/hafs_datm.conf index a4723d406..87acefeee 100644 --- a/parm/hafs_datm.conf +++ b/parm/hafs_datm.conf @@ -3,7 +3,7 @@ run_datm=yes ;; Data atmosphere using CDEPS run_docn=no ;; Data ocean using CDEPS run_ocean=yes ;; Whether to run the ocean model. run_dwav=no ;; Data waves using CDEPS. Not implemented. -make_mesh_atm=yes ;; yes=generate mesh_atm_gen; no=copy from mesh_atm_in to mesh_atm_gen +make_mesh_atm=no ;; yes=generate mesh_atm_gen; no=copy from FIXmeshes datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn diff --git a/parm/hafs_datm_era5.conf b/parm/hafs_datm_era5.conf index 206ba78a1..28f71bc1b 100644 --- a/parm/hafs_datm_era5.conf +++ b/parm/hafs_datm_era5.conf @@ -7,6 +7,7 @@ layouty=8 write_groups=1 write_tasks_per_group=40 ocean_tasks=120 +mesh_atm_in={FIXmeshes}/datm_era5_mesh.nc [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_ocn{forecast/ocean_tasks}_omp1 diff --git a/parm/hafs_docn.conf b/parm/hafs_docn.conf index e754d2668..10a8920a8 100644 --- a/parm/hafs_docn.conf +++ b/parm/hafs_docn.conf @@ -3,7 +3,7 @@ run_datm=no ;; Data atmosphere using CDEPS run_docn=yes ;; Data ocean using CDEPS run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. run_dwav=no ;; Data waves using CDEPS. Not implemented. -make_mesh_ocn=yes ;; yes=generate mesh_ocn_gen; no=copy from mesh_ocn_in to mesh_ocn_gen +make_mesh_ocn=no ;; yes=generate mesh_ocn_gen; no=copy from FIXmeshes docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing # A second file sets this option: docn_source=OISST ;; OISST, RTOFS, or GHRSST diff --git a/parm/hafs_docn_ghrsst.conf b/parm/hafs_docn_ghrsst.conf index 55b9ebdac..642b27109 100644 --- a/parm/hafs_docn_ghrsst.conf +++ b/parm/hafs_docn_ghrsst.conf @@ -6,3 +6,4 @@ docn_mesh_nx_global=11301 docn_mesh_ny_global=7501 mesh_ocn_in={DOCNdir}/mesh/GHRSST_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no ocean_tasks=80 +mesh_ocn_in={FIXmeshes}/docn_ghrsst_mesh.nc \ No newline at end of file diff --git a/parm/hafs_docn_oisst.conf b/parm/hafs_docn_oisst.conf index 24155a833..8bb6a0189 100644 --- a/parm/hafs_docn_oisst.conf +++ b/parm/hafs_docn_oisst.conf @@ -5,3 +5,4 @@ docn_source=OISST docn_mesh_nx_global=1440 docn_mesh_ny_global=720 mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no +mesh_ocn_in={FIXmeshes}/docn_ghrsst_mesh.nc \ No newline at end of file diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 412b10e42..4581347d7 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -385,7 +385,6 @@ - &MAKE_MESH_OCN;YES &DOWNLOAD_DOCN;YES @@ -675,10 +674,7 @@ &DOWNLOAD_DOCN;YES - - &MAKE_MESH_OCN;YES - - + @** endif &RUN_DATM;YES diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_mesh.sh index 7146432e7..8407c9a06 100755 --- a/scripts/exhafs_docn_mesh.sh +++ b/scripts/exhafs_docn_mesh.sh @@ -1,8 +1,7 @@ #!/bin/bash -if [[ "$make_mesh_ocn" != yes || "$run_docn" != yes ]] ; then - echo "This job should only be run if \$run_docn and \$make_mesh_ocn are both yes." - echo " \$make_mesh_ocn=\"$make_mesh_ocn\"" +if [[ "$run_docn" != yes ]] ; then + echo "This job should only be run if \$run_docn=yes" echo " \$run_docn=\"$run_docn\"" echo "Beware! You may anger Poseidon by misusing this script. Avoid coastlines." echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " @@ -24,9 +23,12 @@ mesh_dir=$( dirname "$mesh_ocn" ) docn_source=${DOCN_SOURCE:-OISST} [ -d "$mesh_dir" ] || mkdir "$mesh_dir" -rm -f "$mesh_ocn" test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" +if [[ "$make_mesh_ocn" == yes ]] ; then + rm -f "$mesh_ocn" +fi + if [[ "$docn_source" == OISST ]] ; then "$USHhafs/hafs_oisst_prep_mesh.sh" "$merged" elif [[ "${docn_source}" == RTOFS ]] ; then @@ -41,6 +43,18 @@ fi test -s "$merged" test -r "$merged" +$USHhafs/produtil_deliver.py -m "$merged" "$merged_docn_input" +test -s "$merged_docn_input" +test -r "$merged_docn_input" + +if [[ "$make_mesh_ocn" != yes ]] ; then + set +x + echo "Delivered merged file to $merged" + echo "Will use a premade mesh." + echo "Please enjoy your merged files and have a nice day." + set -x + exit 0 +fi set +x echo "Subprocess successfully merged ocean input files." @@ -64,7 +78,6 @@ test -s "$ofile" # Copy mesh and merged file to final destinations. $USHhafs/produtil_deliver.py -m "$ofile" "$mesh_ocn" -$USHhafs/produtil_deliver.py -m "$merged" "$merged_docn_input" test -s "$mesh_ocn" ls -l "$mesh_ocn" diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 4cff5a52f..1e45b6d7b 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -673,7 +673,7 @@ if [ ${run_datm} = yes ]; then sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams - ${NLN} ${mesh_atm} INPUT/ + ${NLN} ${mesh_atm} INPUT/DATM_ESMF_mesh.nc ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ @@ -684,7 +684,7 @@ if [ ${run_datm} = yes ]; then -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.true.}/g" \ - -e "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" \ + -e "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" \ -e "/_mesh_ocn_/d" \ -e "/_system_type_/d" \ -e "s/_atm_model_/datm/g" \ @@ -693,11 +693,7 @@ if [ ${run_datm} = yes ]; then elif [ ${run_docn} = yes ]; then MAKE_MESH_OCN=$( echo "${make_mesh_ocn:-no}" | tr a-z A-Z ) - if [[ "$MAKE_MESH_OCN" == YES ]] ; then - ${NLN} "$merged_docn_input" INPUT/ - else - ${NLN} "$DOCNdir"/*.nc INPUT/ - fi + ${NLN} "$merged_docn_input" INPUT/ #${NCP} ${PARMhafs}/cdeps/docn_in . #${NCP} ${PARMhafs}/cdeps/docn.streams . @@ -705,7 +701,7 @@ elif [ ${run_docn} = yes ]; then # Generate docn_in from template: ${NCP} ${PARMhafs}/cdeps/docn_in docn_in_template - sed -e "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" \ + sed -e "s/_mesh_ocn_/INPUT\/DOCN_ESMF_mesh.nc/g" \ -e "s/_nx_global_/$docn_mesh_nx_global/g" \ -e "s/_ny_global_/$docn_mesh_ny_global/g" \ < docn_in_template > docn_in @@ -719,14 +715,14 @@ elif [ ${run_docn} = yes ]; then ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn.streams sed -i "s/_yearFirst_/$yr/g" docn.streams sed -i "s/_yearLast_/$endyr/g" docn.streams - sed -i "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" docn.streams + sed -i "s/_mesh_ocn_/INPUT\/DOCN_ESMF_mesh.nc/g" docn.streams for file in INPUT/oisst*.nc INPUT/sst*.nc INPUT/DOCN_input_merged.nc ; do if [[ -s "$file" ]] ; then sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$(basename $file)/" docn.streams fi done - ${NLN} "${mesh_ocn}" INPUT/ + ${NLN} "${mesh_ocn}" INPUT/DOCN_ESMF_mesh.nc ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ @@ -738,7 +734,7 @@ elif [ ${run_docn} = yes ]; then -e "s/_end_hour_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.true.}/g" \ -e "/_mesh_atm_/d" \ - -e "s/_mesh_ocn_/INPUT\/$(basename $mesh_ocn)/g" \ + -e "s/_mesh_ocn_/INPUT\/DOCN_ESMF_mesh.nc/g" \ -e "s/_system_type_/ufs/g" \ -e "s/_atm_model_/fv3/g" \ -e "s/_ocn_model_/docn/g" \ From b1fab5e70d621ca0b9cc22fbdcc0d89efa9006ed Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 28 Jul 2021 10:37:35 -0500 Subject: [PATCH 51/73] Point to pregenerated mesh name in datm mode --- scripts/exhafs_forecast.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 1e45b6d7b..895d616a4 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -665,13 +665,13 @@ if [ ${run_datm} = yes ]; then nowdate=`${NDATE} +24 $nowdate` done - sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm_in + sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm_in sed -i "s/_yearFirst_/$yr/g" datm.streams sed -i "s/_yearLast_/$endyr/g" datm.streams - sed -i "s/_mesh_atm_/INPUT\/$(basename $mesh_atm)/g" datm.streams + sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm.streams ${NLN} ${mesh_atm} INPUT/DATM_ESMF_mesh.nc From 90152a1388507373d94c24ed6eb474642990fc41 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 2 Aug 2021 15:10:54 -0500 Subject: [PATCH 52/73] Correct DOCN/DATM mask usage logic and add forecast.print_esmf option --- jobs/{JHAFS_DATM_MESH => JHAFS_DATM_PREP} | 2 +- jobs/{JHAFS_DOCN_MESH => JHAFS_DOCN_PREP} | 2 +- parm/forecast/globnest/model_configure.tmp | 2 +- .../globnest_hwrf/model_configure.tmp | 2 +- parm/forecast/regional/model_configure.tmp | 2 +- .../regional_hwrf/model_configure.tmp | 2 +- parm/hafs.conf | 12 +++- parm/hafs_docn_oisst.conf | 2 +- parm/hafs_holdvars.conf | 5 +- parm/hafs_holdvars.txt | 5 +- rocoto/hafs_workflow.xml.in | 24 ++++---- rocoto/sites/hera.ent | 3 +- rocoto/sites/kjet.ent | 3 +- rocoto/sites/orion.ent | 3 +- rocoto/sites/wcoss_cray.ent | 3 +- rocoto/sites/wcoss_dell_p3.ent | 3 +- rocoto/sites/xjet.ent | 3 +- rocoto/sites/xjet_hafsv0p2a.ent | 3 +- ...xhafs_datm_mesh.sh => exhafs_datm_prep.sh} | 0 ...xhafs_docn_mesh.sh => exhafs_docn_prep.sh} | 27 ++++----- scripts/exhafs_forecast.sh | 14 ++--- ...hrsst_prep_mesh.sh => hafs_ghrsst_prep.sh} | 59 ++++++++----------- ..._oisst_prep_mesh.sh => hafs_oisst_prep.sh} | 42 ++++++++----- ..._rtofs_prep_mesh.sh => hafs_rtofs_prep.sh} | 0 24 files changed, 120 insertions(+), 103 deletions(-) rename jobs/{JHAFS_DATM_MESH => JHAFS_DATM_PREP} (98%) rename jobs/{JHAFS_DOCN_MESH => JHAFS_DOCN_PREP} (98%) rename scripts/{exhafs_datm_mesh.sh => exhafs_datm_prep.sh} (100%) rename scripts/{exhafs_docn_mesh.sh => exhafs_docn_prep.sh} (74%) rename ush/{hafs_ghrsst_prep_mesh.sh => hafs_ghrsst_prep.sh} (62%) rename ush/{hafs_oisst_prep_mesh.sh => hafs_oisst_prep.sh} (75%) rename ush/{hafs_rtofs_prep_mesh.sh => hafs_rtofs_prep.sh} (100%) diff --git a/jobs/JHAFS_DATM_MESH b/jobs/JHAFS_DATM_PREP similarity index 98% rename from jobs/JHAFS_DATM_MESH rename to jobs/JHAFS_DATM_PREP index 7ad8ba237..37c206a98 100755 --- a/jobs/JHAFS_DATM_MESH +++ b/jobs/JHAFS_DATM_PREP @@ -71,7 +71,7 @@ cd $DATA ##################################### # Execute the script. -${HOMEhafs}/scripts/exhafs_datm_mesh.sh +${HOMEhafs}/scripts/exhafs_datm_prep.sh export err=$? ##################################### diff --git a/jobs/JHAFS_DOCN_MESH b/jobs/JHAFS_DOCN_PREP similarity index 98% rename from jobs/JHAFS_DOCN_MESH rename to jobs/JHAFS_DOCN_PREP index 4ae5052f2..db07ebe1d 100755 --- a/jobs/JHAFS_DOCN_MESH +++ b/jobs/JHAFS_DOCN_PREP @@ -71,7 +71,7 @@ cd $DATA ##################################### # Execute the script. -${HOMEhafs}/scripts/exhafs_docn_mesh.sh +${HOMEhafs}/scripts/exhafs_docn_prep.sh export err=$? ##################################### diff --git a/parm/forecast/globnest/model_configure.tmp b/parm/forecast/globnest/model_configure.tmp index 2d98bbc73..65f35fe82 100644 --- a/parm/forecast/globnest/model_configure.tmp +++ b/parm/forecast/globnest/model_configure.tmp @@ -1,4 +1,4 @@ -print_esmf: .false. +print_esmf: _print_esmf_ total_member: 1 PE_MEMBER01: NTASKS start_year: YR diff --git a/parm/forecast/globnest_hwrf/model_configure.tmp b/parm/forecast/globnest_hwrf/model_configure.tmp index 2d98bbc73..65f35fe82 100644 --- a/parm/forecast/globnest_hwrf/model_configure.tmp +++ b/parm/forecast/globnest_hwrf/model_configure.tmp @@ -1,4 +1,4 @@ -print_esmf: .false. +print_esmf: _print_esmf_ total_member: 1 PE_MEMBER01: NTASKS start_year: YR diff --git a/parm/forecast/regional/model_configure.tmp b/parm/forecast/regional/model_configure.tmp index 2d98bbc73..65f35fe82 100644 --- a/parm/forecast/regional/model_configure.tmp +++ b/parm/forecast/regional/model_configure.tmp @@ -1,4 +1,4 @@ -print_esmf: .false. +print_esmf: _print_esmf_ total_member: 1 PE_MEMBER01: NTASKS start_year: YR diff --git a/parm/forecast/regional_hwrf/model_configure.tmp b/parm/forecast/regional_hwrf/model_configure.tmp index 2d98bbc73..65f35fe82 100644 --- a/parm/forecast/regional_hwrf/model_configure.tmp +++ b/parm/forecast/regional_hwrf/model_configure.tmp @@ -1,4 +1,4 @@ -print_esmf: .false. +print_esmf: _print_esmf_ total_member: 1 PE_MEMBER01: NTASKS start_year: YR diff --git a/parm/hafs.conf b/parm/hafs.conf index 2b70a9bfd..8f6f2f28e 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -69,6 +69,11 @@ utilexec={HOMEhafs}/exec ;; utility exe location (placeholder DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM +# Processed input files and meshes during workflow execution: +docn_input_path={intercom}/cdeps +datm_input_path={intercom}/cdeps + + ## Executable program locations # Currently not used in the workflow script system [exe] @@ -235,6 +240,8 @@ npx=2561 npy=2161 npz=64 +print_esmf=.false. ;; .true. to generate ESMF log files or .false. not to + # The write_grid_component related options quilting=.true. write_groups=2 @@ -282,11 +289,10 @@ ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31 merge_import=.true. # Data model defaults -merged_docn_input={WORKhafs}/intercom/DOCN_input_merged.nc mesh_ocn_in=missing ;; premade mesh to use if make_mesh_ocn=no -mesh_ocn_gen={WORKhafs}/intercom/DOCN_ESMF_mesh.nc ;; do not change +mesh_ocn_gen={WORKhafs}/intercom/cdeps/DOCN_ESMF_mesh.nc ;; do not change mesh_atm_in=missing ;; premade mesh to use if make_mesh_atm=no -mesh_atm_gen={WORKhafs}/intercom/DATM_ESMF_mesh.nc ;; do not change +mesh_atm_gen={WORKhafs}/intercom/cdeps/DATM_ESMF_mesh.nc ;; do not change docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction diff --git a/parm/hafs_docn_oisst.conf b/parm/hafs_docn_oisst.conf index 8bb6a0189..5f318c68b 100644 --- a/parm/hafs_docn_oisst.conf +++ b/parm/hafs_docn_oisst.conf @@ -5,4 +5,4 @@ docn_source=OISST docn_mesh_nx_global=1440 docn_mesh_ny_global=720 mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no -mesh_ocn_in={FIXmeshes}/docn_ghrsst_mesh.nc \ No newline at end of file +mesh_ocn_in={FIXmeshes}/docn_oisst_mesh.nc \ No newline at end of file diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index b9e3668b6..a22970b49 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -113,6 +113,8 @@ write_tasks_per_group={forecast/write_tasks_per_group} write_dopost={forecast/write_dopost} output_history={forecast/output_history} +print_esmf={forecast/print_esmf} + glob_k_split={forecast/glob_k_split} glob_n_split={forecast/glob_n_split} glob_layoutx={forecast/glob_layoutx} @@ -193,4 +195,5 @@ merge_import={forecast/merge_import} mesh_atm={forecast/mesh_atm} mesh_ocn={forecast/mesh_ocn} -merged_docn_input={forecast/merged_docn_input} +docn_input_path={dir/docn_input_path} +datm_input_path={dir/datm_input_path} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 2caa524a4..9af743477 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -58,7 +58,8 @@ export docn_mesh_ny_global={docn_mesh_ny_global} export docn_download_jobs={docn_download_jobs} export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} -export merged_docn_input={merged_docn_input} +export docn_input_path={docn_input_path} +export datm_input_path={datm_input_path} export DOCN_SOURCE={DOCN_SOURCE} export DATM_SOURCE={DATM_SOURCE} @@ -120,6 +121,8 @@ export write_tasks_per_group={write_tasks_per_group} export write_dopost={write_dopost} export output_history={output_history} +export print_esmf={print_esmf} + export glob_k_split={glob_k_split} export glob_n_split={glob_n_split} export glob_layoutx={glob_layoutx} diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 4581347d7..28e1414be 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -371,15 +371,15 @@ @** if RUN_DOCN==YES - - &JOBhafs;/JHAFS_DOCN_MESH - hafs_docn_mesh_&SID;_@Y@m@d@H - &WORKhafs;/hafs_docn_mesh.log + + &JOBhafs;/JHAFS_DOCN_PREP + hafs_docn_prep_&SID;_@Y@m@d@H + &WORKhafs;/hafs_docn_prep.log &ACCOUNT; &RESERVATION; &QUEUE_PE; &PE_EXTRA; - &MAKE_ESMF_MESH_RESOURCES; + &DOCN_PREP_RESOURCES; &ENV_VARS; @@ -598,15 +598,15 @@ @** if RUN_DATM==YES - - &JOBhafs;/JHAFS_DATM_MESH - hafs_datm_mesh_&SID;_@Y@m@d@H - &WORKhafs;/hafs_datm_mesh.log + + &JOBhafs;/JHAFS_DATM_PREP + hafs_datm_prep_&SID;_@Y@m@d@H + &WORKhafs;/hafs_datm_prep.log &ACCOUNT; &RESERVATION; &QUEUE_PE; &PE_EXTRA; - &MAKE_ESMF_MESH_RESOURCES; + &DATM_PREP_RESOURCES; &ENV_VARS; @@ -666,7 +666,7 @@ &MAKE_MESH_ATM;YES - + @** endif @** if RUN_DOCN==YES @@ -674,7 +674,7 @@ &DOWNLOAD_DOCN;YES - + @** endif &RUN_DATM;YES diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 681540cd8..28f5f5f20 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -16,7 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 080a6dea2..588fb14c7 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -16,7 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index ca47a1dc7..cf5509f92 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -16,7 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index 9d4639670..e9940a899 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -16,7 +16,8 @@ 1G"> 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=12:tpp=1TOTAL_TASKS360NCTSK12OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 89522cacd..a2fc98dc6 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -17,7 +17,8 @@ 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 1:ppn=6:tpp=1TOTAL_TASKS6NCTSK6OMP_THREADS4-R affinity[core\(4\):distribute=balance]00:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index eaa0f81f7..a9ff16938 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -16,7 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 690bf906a..82abbb836 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -16,7 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=24:tpp=1TOTAL_TASKS720NCTSK24OMP_THREADS101:30:00"> diff --git a/scripts/exhafs_datm_mesh.sh b/scripts/exhafs_datm_prep.sh similarity index 100% rename from scripts/exhafs_datm_mesh.sh rename to scripts/exhafs_datm_prep.sh diff --git a/scripts/exhafs_docn_mesh.sh b/scripts/exhafs_docn_prep.sh similarity index 74% rename from scripts/exhafs_docn_mesh.sh rename to scripts/exhafs_docn_prep.sh index 8407c9a06..a76571afc 100755 --- a/scripts/exhafs_docn_mesh.sh +++ b/scripts/exhafs_docn_prep.sh @@ -22,6 +22,7 @@ mesh_ocn="$mesh_ocn" mesh_dir=$( dirname "$mesh_ocn" ) docn_source=${DOCN_SOURCE:-OISST} +[ -d "$docn_input_path" ] || mkdir -p "$docn_input_path" [ -d "$mesh_dir" ] || mkdir "$mesh_dir" test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" @@ -30,48 +31,44 @@ if [[ "$make_mesh_ocn" == yes ]] ; then fi if [[ "$docn_source" == OISST ]] ; then - "$USHhafs/hafs_oisst_prep_mesh.sh" "$merged" + "$USHhafs/hafs_oisst_prep.sh" "$docn_input_path" elif [[ "${docn_source}" == RTOFS ]] ; then - "$USHhafs/hafs_rtofs_prep_mesh.sh" "$merged" + "$USHhafs/hafs_rtofs_prep.sh" "$docn_input_path" elif [[ "${docn_source}" == GHRSST ]] ; then - "$USHhafs/hafs_ghrsst_prep_mesh.sh" "$merged" + "$USHhafs/hafs_ghrsst_prep.sh" "$docn_input_path" else echo "ERROR: Unknown data ocean source $docn_source. Giving up." 2>&1 echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DOCN_SOURCE VALUE <- " exit 1 fi -test -s "$merged" -test -r "$merged" -$USHhafs/produtil_deliver.py -m "$merged" "$merged_docn_input" -test -s "$merged_docn_input" -test -r "$merged_docn_input" - if [[ "$make_mesh_ocn" != yes ]] ; then set +x - echo "Delivered merged file to $merged" + echo "Delivered processed ocean files to $docn_input_path" echo "Will use a premade mesh." - echo "Please enjoy your merged files and have a nice day." + echo "Please enjoy your files and have a nice day." set -x exit 0 fi set +x -echo "Subprocess successfully merged ocean input files." +echo "Delivered processed ocean files to $docn_input_path" echo "Will now generate mesh in \"$ofile\"" echo "Will deliver to \"$mesh_ocn\"" set -x +file0=$docn_input_path/DOCN_input_00000.nc + # Generate the mesh from the merged file. if [[ "$docn_source" == OISST ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --maskvar sst --maskcal --double --overwrite elif [[ "${docn_source}" == RTOFS ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --overwrite --latvar Latitude --lonvar Longitude \ --maskvar sst --maskcal —double elif [[ "${docn_source}" == GHRSST ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$merged" --ofile "$ofile" \ + $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --maskvar analysed_sst --maskcal --overwrite --double fi test -s "$ofile" diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 895d616a4..27ffc0f76 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -693,7 +693,7 @@ if [ ${run_datm} = yes ]; then elif [ ${run_docn} = yes ]; then MAKE_MESH_OCN=$( echo "${make_mesh_ocn:-no}" | tr a-z A-Z ) - ${NLN} "$merged_docn_input" INPUT/ + ${NLN} "$docn_input_path"/DOCN_input*nc INPUT/ #${NCP} ${PARMhafs}/cdeps/docn_in . #${NCP} ${PARMhafs}/cdeps/docn.streams . @@ -706,22 +706,19 @@ elif [ ${run_docn} = yes ]; then -e "s/_ny_global_/$docn_mesh_ny_global/g" \ < docn_in_template > docn_in - if [ "$docn_source" == GHRSST ] ; then - [ -d docn ] || mkdir docn - ${NLN} "$merged_docn_input" docn/ghrsst_v1.nc - fi - # Generate docn_streams from template specific to the model: ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn.streams sed -i "s/_yearFirst_/$yr/g" docn.streams sed -i "s/_yearLast_/$endyr/g" docn.streams sed -i "s/_mesh_ocn_/INPUT\/DOCN_ESMF_mesh.nc/g" docn.streams - for file in INPUT/oisst*.nc INPUT/sst*.nc INPUT/DOCN_input_merged.nc ; do + for file in INPUT/oisst*.nc INPUT/sst*.nc INPUT/DOCN_input*.nc ; do if [[ -s "$file" ]] ; then - sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$(basename $file)/" docn.streams + sed -i "/^stream_data_files01:/ s/$/\ \"INPUT\/$(basename $file)\"/" docn.streams fi done +# sed -i 's/\(stream_data_files01: *\)\(.*\)$/\1 \2"/g' docn.streams + ${NLN} "${mesh_ocn}" INPUT/DOCN_ESMF_mesh.nc ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ @@ -764,6 +761,7 @@ sed -e "s/NTASKS/${TOTAL_TASKS}/g" -e "s/YR/$yr/g" \ -e "s/_DLON_/$output_grid_dlon/g" \ -e "s/_DLAT_/$output_grid_dlat/g" \ -e "s/_cpl_/${cplflx:-.false.}/g" \ + -e "s/_print_esmf_/${print_esmf:-.false.}/g" \ model_configure.tmp > model_configure # Copy fix files needed by inline_post diff --git a/ush/hafs_ghrsst_prep_mesh.sh b/ush/hafs_ghrsst_prep.sh similarity index 62% rename from ush/hafs_ghrsst_prep_mesh.sh rename to ush/hafs_ghrsst_prep.sh index 8459346dd..0b237189a 100755 --- a/ush/hafs_ghrsst_prep_mesh.sh +++ b/ush/hafs_ghrsst_prep.sh @@ -2,7 +2,9 @@ set -xe -subset="${1:-ghrsst_subset.nc}" +set -u +output_path="$1" +set +u if ( ! which cdo ) ; then echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 @@ -34,21 +36,33 @@ set -x # Generate the filenames. missing='' -mergefiles='' -itry=0 -itrytoohard=99 # infinite loop guard -while (( now <= end && itry <= itrytoohard )) ; do +usefiles='' +itime=0 +infinity=9999 # infinite loop guard +while (( now <= end && itime < infinity )) ; do infile="$DOCNdir/JPL-L4_GHRSST-SSTfnd-MUR-GLOB-${now:0:8}.nc" if [[ ! -s "$infile" || ! -r "$infile" ]] ; then echo "GHRSST input file is missing: $infile" 2>&1 missing="$missing $infile" else - mergefiles="$mergefiles $infile" + usefiles="$mergefiles $infile" + + rm -f subset.nc + # Subset data over HAFS region (lat, 250 to 355 and lon, 0 to 50) + cdo -sellonlatbox,-118,-5,-15.0,60.0 "$infile" subset.nc + + # Convert temperature units: + aos_old=`ncdump -c subset.nc | grep add_offset | grep analysed_sst | awk '{print $3}'` + aos_new=$(echo "scale=3; $aos_old-273.15" | bc) + ncatted -O -a add_offset,analysed_sst,o,f,"$aos_new" subset.nc + + outfile=$( printf "%s/DOCN_input_%05d.nc" "$output_path" $itime ) + $USHhafs/produtil_deliver.py -m subset.nc "$outfile" fi now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) - itry=$(( itry+1 )) + itime=$(( itime+1 )) done -if (( itry > itrytoohard )) ; then +if (( itime >= infinity )) ; then echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 exit 1 fi @@ -63,34 +77,9 @@ if [[ "${missing:-}Q" != Q ]] ; then exit 1 fi -set +x -echo "GHRSST input files are:" -for f in $mergefiles ; do - echo " - $f" -done -echo "Will merge ghrsst files into $merged" - echo "Will subset data over HAFS region into $subset" -set -x - -# Merge all ghrsst files into one, as expected by hafs_esmf_mesh.py -cdo mergetime $mergefiles ghrsst_v1.nc - -# Subset data over HAFS region (lat, 250 to 355 and lon, 0 to 50) -cdo -sellonlatbox,-118,-5,-15.0,60.0 ghrsst_v1.nc ghrsst_v2.nc - -# Convert temperature units: -aos_old=`ncdump -c ghrsst_v2.nc | grep add_offset | grep analysed_sst | awk '{print $3}'` -aos_new=$(echo "scale=3; $aos_old-273.15" | bc) -ncatted -O -a add_offset,analysed_sst,o,f,"$aos_new" ghrsst_v2.nc - -# Deliver file: -rm -f "$subset" -cp -fp ghrsst_v2.nc "$subset" -test -s "$subset" -test -r "$subset" - #ncks -O -d lon,6999,17499 -d lat,8999,13999 "$merged" ghrsst_v1.nc # Rejoice. set +x -echo "Successfully merged GHRSST files and subsetted area." +echo "Successfully subsetted and corrected units of GHRSST files." +echo "Please enjoy your files and have a nice day." diff --git a/ush/hafs_oisst_prep_mesh.sh b/ush/hafs_oisst_prep.sh similarity index 75% rename from ush/hafs_oisst_prep_mesh.sh rename to ush/hafs_oisst_prep.sh index 575e9e459..bca9f51ce 100755 --- a/ush/hafs_oisst_prep_mesh.sh +++ b/ush/hafs_oisst_prep.sh @@ -2,15 +2,21 @@ set -xe -nozlev="${1:-oisst-avhrr-v02r01.merged_nozlev.nc}" +set -u +output_path="$1" +set +u if ( ! which cdo ) ; then + set +x echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 + set -x exit 1 fi if ( ! which ncwa ) ; then + set +x echo "The \"ncwa\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 + set -x exit 1 fi @@ -19,15 +25,16 @@ WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} USHhafs=${USHhafs:-${HOMEhafs}/ush} CDATE=${CDATE:-${YMDH}} -merged=oisst-avhrr-v02r01.merged.nc -mesh_ocn="$mesh_ocn" -mesh_dir=$( dirname "$mesh_ocn" ) +set -u # Start & end times are at day precision, not hour m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) now=$m1date end=$p1date +merged=oisst-avhrr-v02r01.merged.nc +nozlev=oisst-avhrr-v02r01.nozlev.nc +outfile=$( printf "%s/DOCN_input_%05d.nc" "$output_path" 0 ) set +x echo "Generating ESMF mesh from OISST files." @@ -35,30 +42,32 @@ echo "Running in dir \"$PWD\"" echo "OISST Date range is $now to $end" set -x +rm -f DOCN_input* merged.nc + # Generate the filenames. -mergefiles='' +usefiles='' missing='' -itry=0 -itrytoohard=99 # infinite loop guard -while (( now <= end && itry <= itrytoohard )) ; do +itime=0 +infinity=9999 # infinite loop guard +while (( now <= end && itime < infinity )) ; do infile="$DOCNdir/oisst-avhrr-v02r01.${now:0:8}.nc" if [[ ! -s "$infile" || ! -r "$infile" ]] ; then echo "OISST input file is missing: $infile" 2>&1 missing="$missing $infile" else - mergefiles="$mergefiles $infile" + usefiles="$usefiles $infile" fi now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) - itry=$(( itry+1 )) + itime=$(( itime+1 )) done -if (( itry > itrytoohard )) ; then +if (( itime >= infinity )) ; then echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 exit 1 fi if [[ "${missing:-}Q" != Q ]] ; then set +x - echo "You are missing some GHRSST input files!" + echo "You are missing some OISST input files!" for infile in $missing ; do echo " missing: $infile" done @@ -68,7 +77,7 @@ fi set +x echo "OISST input files are:" -for f in $mergefiles ; do +for f in $usefiles ; do echo " - $f" done echo "Will merge oisst files into $merged" @@ -76,7 +85,7 @@ echo "Will remove z levels into $nozlev" set -x # Merge all oisst files into one, as expected by hafs_esmf_mesh.py -cdo mergetime $mergefiles "$merged" +cdo mergetime $usefiles "$merged" test -s "$merged" test -r "$merged" @@ -85,6 +94,11 @@ ncwa -O -a zlev "$merged" "$nozlev" test -s "$nozlev" test -r "$nozlev" +# Deliver to intercom: +$USHhafs/produtil_deliver.py -m "$nozlev" "$outfile" + # Rejoice. set +x echo "Successfully merged OISST files and removed z dimension." +echo "Merged file is at: $outfile" +echo "Please enjoy your files and have a nice day." diff --git a/ush/hafs_rtofs_prep_mesh.sh b/ush/hafs_rtofs_prep.sh similarity index 100% rename from ush/hafs_rtofs_prep_mesh.sh rename to ush/hafs_rtofs_prep.sh From 4855bfd61d6b917e1d23286e9512c5252f9f7bb0 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 3 Aug 2021 20:04:29 +0000 Subject: [PATCH 53/73] Remove all unneeded vars from ghrsst before spatial subsetting --- ush/hafs_ghrsst_prep.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/hafs_ghrsst_prep.sh b/ush/hafs_ghrsst_prep.sh index 0b237189a..50d436f26 100755 --- a/ush/hafs_ghrsst_prep.sh +++ b/ush/hafs_ghrsst_prep.sh @@ -47,9 +47,13 @@ while (( now <= end && itime < infinity )) ; do else usefiles="$mergefiles $infile" - rm -f subset.nc + # Discard all vars except what we need; convert to NetCDF3: + rm -f vars.nc + ncks -v time,lat,lon,analysed_sst -6 "$infile" vars.nc + # Subset data over HAFS region (lat, 250 to 355 and lon, 0 to 50) - cdo -sellonlatbox,-118,-5,-15.0,60.0 "$infile" subset.nc + rm -f subset.nc + cdo -sellonlatbox,-118,-5,-15.0,60.0 vars.nc subset.nc # Convert temperature units: aos_old=`ncdump -c subset.nc | grep add_offset | grep analysed_sst | awk '{print $3}'` From 64012477bedf9285c9f9c6ef74df4ab4c94d1b93 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 3 Aug 2021 20:27:46 +0000 Subject: [PATCH 54/73] Point to alternative python on Jet --- modulefiles/modulefile.hafs.jet | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modulefiles/modulefile.hafs.jet b/modulefiles/modulefile.hafs.jet index 94d50d20b..a4207d840 100644 --- a/modulefiles/modulefile.hafs.jet +++ b/modulefiles/modulefile.hafs.jet @@ -66,4 +66,10 @@ setenv WGRIB2 /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/intel-18.0.5.274/impi-201 module load nco/4.9.1 module load rocoto/1.3.3 -module load intelpython/3.6.5 +#module load intelpython/3.6.5 +# Alternative python with prerequisites for DATM and DOCN: +module use /lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/modulefiles +module load python/3.7.10-hafs + +# For CDEPS data models: +module load cdo From 5f79bbcb675a090a700c945907aa263da7e7a64c Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 3 Aug 2021 20:28:25 +0000 Subject: [PATCH 55/73] Reduce runtime for data model prep jobs now that GHRSST prep is much faster --- rocoto/sites/hera.ent | 4 ++-- rocoto/sites/kjet.ent | 7 ++++--- rocoto/sites/orion.ent | 4 ++-- rocoto/sites/wcoss_cray.ent | 4 ++-- rocoto/sites/wcoss_dell_p3.ent | 4 ++-- rocoto/sites/xjet.ent | 5 +++-- rocoto/sites/xjet_hafsv0p2a.ent | 4 ++-- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 28f5f5f20..6b2c72300 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 588fb14c7..0c2b55459 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> @@ -29,7 +29,7 @@ 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 07:59:00"> + 00:59:00"> OMP_THREADS2"> @@ -82,6 +82,7 @@ 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 116:ppn=12:tpp=2+1:ppn=8:tpp=2TOTAL_TASKS1400NCTSK12&FORECAST_EXTRA;"> 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index cf5509f92..6484f2aaf 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index e9940a899..68ddc1323 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -16,8 +16,8 @@ 1G"> 1G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=12:tpp=1TOTAL_TASKS360NCTSK12OMP_THREADS100:30:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index a2fc98dc6..9b3d4365b 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -17,8 +17,8 @@ 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> 1:ppn=6:tpp=1TOTAL_TASKS6NCTSK6OMP_THREADS4-R affinity[core\(4\):distribute=balance]00:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index a9ff16938..af122bcf2 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS101:30:00"> @@ -83,6 +83,7 @@ 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 116:ppn=12:tpp=2+1:ppn=8:tpp=2TOTAL_TASKS1400NCTSK12&FORECAST_EXTRA;"> 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent index 82abbb836..7be9ecb3d 100644 --- a/rocoto/sites/xjet_hafsv0p2a.ent +++ b/rocoto/sites/xjet_hafsv0p2a.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS101:15:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> 3:ppn=4:tpp=6TOTAL_TASKS12NCTSK4OMP_THREADS600:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 30:ppn=24:tpp=1TOTAL_TASKS720NCTSK24OMP_THREADS101:30:00"> From f97c0a8e15cf8e120fdee6fbdf77e068fc3c1cc9 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 3 Aug 2021 20:49:43 +0000 Subject: [PATCH 56/73] Update paths for jet in cronjob_hafs_docn.sh --- rocoto/cronjob_hafs_docn.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh index c3517ca2c..46e9a99de 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_docn.sh @@ -13,14 +13,14 @@ date #PYTHON3=/opt/intel/intelpython3/bin/python3 # NOAA RDHPCS Jet -#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS -#dev="-s sites/xjet.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 +# HOMEhafs=/lfs4/BMC/wrfruc/${USER}/HAFS/HAFS +# dev="-s sites/xjet.ent -f" +# PYTHON3=/lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python # MSU Orion - HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS - dev="-s sites/orion.ent -f" - PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python +HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS +dev="-s sites/orion.ent -f" +PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python # NOAA RDHPCS Hera #HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS @@ -35,11 +35,11 @@ EXPT=$(basename ${HOMEhafs}) # Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh # Run data ocean with OISST -# ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ -# config.SUBEXPT=${EXPT}_oisst \ -# forecast.output_history=.true. \ -# ../parm/hafs_regional_static.conf \ -# ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf +${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + config.SUBEXPT=${EXPT}_oisst \ + forecast.output_history=.true. \ + ../parm/hafs_regional_static.conf \ + ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf # Run data ocean with GHRSST ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ From 218ed26fd1ef7d1f8d16edcf57b927d6d6bf07c1 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 3 Aug 2021 17:53:35 -0500 Subject: [PATCH 57/73] Put DATM input in intercom/cdeps just like DOCN --- .../cdeps/{datm.streams => datm_era5.streams} | 0 rocoto/hafs_workflow.xml.in | 6 +- scripts/exhafs_datm_prep.sh | 50 +++++++---- scripts/exhafs_forecast.sh | 42 ++++----- ush/hafs_era5_prep.sh | 88 +++++++++++++++++++ 5 files changed, 140 insertions(+), 46 deletions(-) rename parm/cdeps/{datm.streams => datm_era5.streams} (100%) create mode 100755 ush/hafs_era5_prep.sh diff --git a/parm/cdeps/datm.streams b/parm/cdeps/datm_era5.streams similarity index 100% rename from parm/cdeps/datm.streams rename to parm/cdeps/datm_era5.streams diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 28e1414be..cc692b0c1 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -612,7 +612,6 @@ - &MAKE_MESH_ATM;YES &DOWNLOAD_DATM;YES @@ -664,10 +663,7 @@ &DOWNLOAD_DATM;YES - - &MAKE_MESH_ATM;YES - - + @** endif @** if RUN_DOCN==YES diff --git a/scripts/exhafs_datm_prep.sh b/scripts/exhafs_datm_prep.sh index e186fb188..2e156bc4f 100755 --- a/scripts/exhafs_datm_prep.sh +++ b/scripts/exhafs_datm_prep.sh @@ -1,8 +1,7 @@ #!/bin/bash -if [[ "$make_mesh_atm" != yes || "$run_datm" != yes ]] ; then - echo "This job should only be run if \$run_datm and \$make_mesh_atm are both yes." - echo " \$make_mesh_atm=\"$make_mesh_atm\"" +if [[ "$run_datm" != yes ]] ; then + echo "This job should only be run if \$run_datm is yes." echo " \$run_datm=\"$run_datm\"" echo "Billions of electrons have whizzed by, the universe's entropy has increased, and yet nothing has been accomplished." echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " @@ -17,33 +16,52 @@ USHhafs=${USHhafs:-${HOMEhafs}/ush} CDATE=${CDATE:-${YMDH}} APRUNS=${APRUNS:-"aprun -b -j1 -n1 -N1 -d1 -cc depth"} +ifile=$datm_input_path/DATM_input_00000.nc ofile=ofile.nc mesh_atm="$mesh_atm" mesh_dir=$( dirname "$mesh_atm" ) +datm_source=${DATM_SOURCE:-ERA5} -# Pick an input file. This will get the newest one, if the names are -# identical except for YYYYMMDD date string: -test -d "$DATMdir" -#ifile=$( ls -1 "$DATMdir"/*nc | sort -ur | head -1 ) -ifile="$DATMdir/ERA5_${CDATE:0:8}.nc" +[ -d "$docn_input_path" ] || mkdir -p "$docn_input_path" +[ -d "$mesh_dir" ] || mkdir "$mesh_dir" +test -e "$ofile" -o -L "$ofile" && rm -f "$ofile" -test -s "$ifile" -test -r "$ifile" +if [[ "$make_mesh_ocn" == yes ]] ; then + rm -f "$mesh_ocn" +fi + +if [[ "$datm_source" == ERA5 ]] ; then + "$USHhafs/hafs_era5_prep.sh" "$datm_input_path" +else + echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 + echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DATM_SOURCE VALUE <- " + exit 1 +fi -[ -d "$mesh_dir" ] || mkdir "$mesh_dir" -rm -f "$mesh_atm" -[ -e "$ofile" -o -L "$ofile" ]] && rm -f "$ofile" +if [[ "$make_mesh_atm" != yes ]] ; then + set +x + echo "Processed atmosphere files are in $datm_input_path" + echo "Will use a premade mesh." + echo "Please enjoy your files and have a nice day." + set -x + exit 0 +fi set +x echo "Generating ESMF mesh from $datm_source files." echo "Running in dir \"$PWD\"" -echo "ERA5 grid generation input file is \"$ifile\"" +set -x + +test -s "$ifile" +test -r "$ifile" + +set +x +echo "Grid generation $datm_source input file is \"$ifile\"" echo "Temporary output mesh is $ofile" echo "Will deliver to \"$mesh_atm\"" set -x -# Generate the mesh from the merged file: -datm_source=${DATM_SOURCE:-ERA5} +# Generate the mesh. if [[ "$datm_source" == ERA5 ]] ; then $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ --overwrite --latvar latitude --lonvar longitude --double diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 27ffc0f76..342375406 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -644,36 +644,30 @@ if [ ${run_datm} = no ]; then cat temp diag_table.tmp > diag_table fi -if [ ${run_datm} = yes ]; then +#--------------------------------------------------- +# Copy CDEPS input, parm, and fix files if required. +#--------------------------------------------------- -#---------------------------------------------- -# Copy CDEPS parm files if required. -#---------------------------------------------- +if [ ${run_datm} = yes ]; then + datm_source=${DATM_SOURCE:-ERA5} ${NCP} ${PARMforecast}/model_configure.tmp . - ${NCP} ${PARMhafs}/cdeps/datm_in . - ${NCP} ${PARMhafs}/cdeps/datm.streams . - - m1date=`${NDATE} -24 $CDATE` - p1date=`${NDATE} +$(( NHRS+6 )) $CDATE` - nowdate=$m1date - while (( nowdate <= p1date )) ; do - era5_name=ERA5_${nowdate:0:8}.nc - if [[ -s $DATMdir/$era5_name ]] ; then - ${NLN} $DATMdir/$era5_name INPUT/$era5_name - sed -i "/^stream_data_files01:/ s/$/\ INPUT\/$era5_name/" datm.streams + ${NLN} ${mesh_atm} INPUT/DATM_ESMF_mesh.nc + ${NLN} "$datm_input_path"/DATM_input*nc INPUT/ + + # Generate docn.streams from template specific to the model: + ${NCP} ${PARMhafs}/cdeps/datm_$( echo "$datm_source" | tr A-Z a-z ).streams datm.streams + for file in INPUT/DATM_input*nc ; do + if [[ -s "$file" ]] ; then + sed -i "/^stream_data_files01:/ s/$/\ \"INPUT\/$(basename $file)\"/" datm.streams fi - nowdate=`${NDATE} +24 $nowdate` done - - sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm_in - sed -i "s/_yearFirst_/$yr/g" datm.streams - sed -i "s/_yearLast_/$endyr/g" datm.streams - sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm.streams - ${NLN} ${mesh_atm} INPUT/DATM_ESMF_mesh.nc + # Generate datm_in and nems.configure from model-independent templates: + ${NCP} ${PARMhafs}/cdeps/datm_in . + sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm_in ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ @@ -706,7 +700,7 @@ elif [ ${run_docn} = yes ]; then -e "s/_ny_global_/$docn_mesh_ny_global/g" \ < docn_in_template > docn_in - # Generate docn_streams from template specific to the model: + # Generate docn.streams from template specific to the model: ${NCP} ${PARMhafs}/cdeps/docn_$( echo "$docn_source" | tr A-Z a-z ).streams docn.streams sed -i "s/_yearFirst_/$yr/g" docn.streams sed -i "s/_yearLast_/$endyr/g" docn.streams @@ -717,8 +711,6 @@ elif [ ${run_docn} = yes ]; then fi done -# sed -i 's/\(stream_data_files01: *\)\(.*\)$/\1 \2"/g' docn.streams - ${NLN} "${mesh_ocn}" INPUT/DOCN_ESMF_mesh.nc ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ diff --git a/ush/hafs_era5_prep.sh b/ush/hafs_era5_prep.sh new file mode 100755 index 000000000..d072ac3fa --- /dev/null +++ b/ush/hafs_era5_prep.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -xe + +set -u +output_path="$1" +set +u + +if ( ! which cdo ) ; then + set +x + echo "The \"cdo\" command isn't in your path! Go find it and rerun this job." 1>&2 + set -x + exit 1 +fi + +if ( ! which ncwa ) ; then + set +x + echo "The \"ncwa\" command from the NetCDF Data Operators (nco) is not in your path! Go find the nco and rerun this job." 1>&2 + set -x + exit 1 +fi + +HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} +WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} +USHhafs=${USHhafs:-${HOMEhafs}/ush} +CDATE=${CDATE:-${YMDH}} + +set -u + +# Start & end times are at day precision, not hour +m1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d ) +p1date=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d ) +now=$m1date +end=$p1date + +set +x +echo "Linking ERA5 files." +echo "Running in dir \"$PWD\"" +echo "Will link ERA5 files into $output_path" +echo "ERA5 Date range is $now to $end" +set -x + +rm -f DATM_input* merged.nc + +# Generate the filenames. +usefiles='' +missing='' +itime=0 +infinity=9999 # infinite loop guard +while (( now <= end && itime < infinity )) ; do + infile="$DATMdir/ERA5_${now:0:8}.nc" + if [[ ! -s "$infile" || ! -r "$infile" ]] ; then + echo "ERA5 input file is missing: $infile" 2>&1 + missing="$missing $infile" + else + usefiles="$usefiles $infile" + outfile=$( printf "%s/DATM_input_%05d.nc" "$output_path" $itime ) + ln -sf "$infile" "$outfile" + fi + now=$( date -d "${now:0:4}-${now:4:2}-${now:6:2}t00:00:00+00 +24 hours" +%Y%m%d ) + itime=$(( itime+1 )) +done +if (( itime >= infinity )) ; then + echo "Infinite loop detected! The \"date\" command did not behave as expected. Aborting!" 1>&2 + exit 1 +fi + +if [[ "${missing:-}Q" != Q ]] ; then + set +x + echo "You are missing some ERA5 input files!" + for infile in $missing ; do + echo " missing: $infile" + done + echo " -> SCRIPT IS ABORTING BECAUSE INPUT FILES ARE MISSING <- " + exit 1 +fi + +set +x +echo "ERA5 input files are:" +for f in $usefiles ; do + echo " - $f" +done +set -x + +# Rejoice. +set +x +echo "Successfully linked ERA5 files at $output_path" +echo "Please enjoy your files and have a nice day." From 15e1e1e13789efee912a046b67cc0b021df64a24 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 4 Aug 2021 21:49:59 +0000 Subject: [PATCH 58/73] Put Hera DOCN resources in rocoto/sites/hera.ent --- rocoto/sites/hera.ent | 1 + 1 file changed, 1 insertion(+) diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index 6b2c72300..e69fce37f 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -95,6 +95,7 @@ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 71:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> From ea72780c4d70d892a10575332ff55faa64e28f78 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 4 Aug 2021 21:50:30 +0000 Subject: [PATCH 59/73] Update modulefiles for hera --- modulefiles/modulefile.hafs.hera | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modulefiles/modulefile.hafs.hera b/modulefiles/modulefile.hafs.hera index 23dfb177e..400c0ffc6 100644 --- a/modulefiles/modulefile.hafs.hera +++ b/modulefiles/modulefile.hafs.hera @@ -70,4 +70,9 @@ module load nco/4.9.1 module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles module load rocoto/1.3.3 -module load intelpython/3.6.8 +# CDEPS needs extra python modules, of specific versions: +module use /scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/modulefiles +module load python/3.7.10-hafs +#module load intelpython/3.6.8 + +module load cdo/1.9.10 \ No newline at end of file From e64fee2f9708922fdd1fa77b86036fb1fa7d876a Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Tue, 10 Aug 2021 21:48:53 +0000 Subject: [PATCH 60/73] Correct the docn/datm prep resources for kjet and hera --- rocoto/sites/hera.ent | 4 ++-- rocoto/sites/kjet.ent | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index e69fce37f..87b346991 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 0c2b55459..17c879c58 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -16,8 +16,8 @@ 1G"> 5G"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> - 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=12TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> 6:ppn=2:tpp=6TOTAL_TASKS12NCTSK2OMP_THREADS600:30:0024G"> 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS100:30:00"> From a520b813d6f6a1000b7656e19359164cf7eb071b Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Tue, 10 Aug 2021 21:52:05 +0000 Subject: [PATCH 61/73] Correct python paths in data model cron scripts --- rocoto/cronjob_hafs_datm.sh | 12 ++++++------ rocoto/cronjob_hafs_docn.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rocoto/cronjob_hafs_datm.sh b/rocoto/cronjob_hafs_datm.sh index 6c28d6ac6..99062b006 100755 --- a/rocoto/cronjob_hafs_datm.sh +++ b/rocoto/cronjob_hafs_datm.sh @@ -13,9 +13,9 @@ date #PYTHON3=/opt/intel/intelpython3/bin/python3 # NOAA RDHPCS Jet -#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS -#dev="-s sites/xjet.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 +# HOMEhafs=/lfs4/BMC/wrfruc/${USER}/HAFS/HAFS +# dev="-s sites/xjet.ent -f" +# PYTHON3=/lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python # MSU Orion HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS @@ -23,9 +23,9 @@ date PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python # NOAA RDHPCS Hera -#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -#dev="-s sites/hera.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 +# HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +# dev="-s sites/hera.ent -f" +# PYTHON3=/scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python cd ${HOMEhafs}/rocoto diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh index 46e9a99de..3c26d0f3d 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_docn.sh @@ -23,9 +23,9 @@ dev="-s sites/orion.ent -f" PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python # NOAA RDHPCS Hera -#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -#dev="-s sites/hera.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 +# HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +# dev="-s sites/hera.ent -f" +# PYTHON3=/scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python cd ${HOMEhafs}/rocoto From a63039f0e3f8ff6b74f1373be9f1453919dc7eae Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 16 Aug 2021 10:28:46 -0500 Subject: [PATCH 62/73] Remove the redundant mesh_ocn_in entry in the docn configuration files. --- parm/hafs_docn_ghrsst.conf | 3 +-- parm/hafs_docn_oisst.conf | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/parm/hafs_docn_ghrsst.conf b/parm/hafs_docn_ghrsst.conf index 642b27109..7a879b561 100644 --- a/parm/hafs_docn_ghrsst.conf +++ b/parm/hafs_docn_ghrsst.conf @@ -4,6 +4,5 @@ docn_source=GHRSST [forecast] docn_mesh_nx_global=11301 docn_mesh_ny_global=7501 -mesh_ocn_in={DOCNdir}/mesh/GHRSST_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no ocean_tasks=80 -mesh_ocn_in={FIXmeshes}/docn_ghrsst_mesh.nc \ No newline at end of file +mesh_ocn_in={FIXmeshes}/docn_ghrsst_mesh.nc ;; premade mesh to use if make_mesh_ocn=no diff --git a/parm/hafs_docn_oisst.conf b/parm/hafs_docn_oisst.conf index 5f318c68b..3505ca4dc 100644 --- a/parm/hafs_docn_oisst.conf +++ b/parm/hafs_docn_oisst.conf @@ -4,5 +4,4 @@ docn_source=OISST [forecast] docn_mesh_nx_global=1440 docn_mesh_ny_global=720 -mesh_ocn_in={DOCNdir}/mesh/TX025_201215_ESMFmesh_direct.nc ;; premade mesh to use if make_mesh_ocn=no -mesh_ocn_in={FIXmeshes}/docn_oisst_mesh.nc \ No newline at end of file +mesh_ocn_in={FIXmeshes}/docn_oisst_mesh.nc ;; premade mesh to use if make_mesh_ocn=no From 463810a5f6421b10c6b02098f5a054c3abd63fbc Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 16 Aug 2021 12:36:21 -0500 Subject: [PATCH 63/73] Point to the support/HAFS branch of ufs-weather-model --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index bbb88c41a..38119e2b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-weather-model.git - branch = feature/hafs_couplehycom_cdeps + branch = support/HAFS [submodule "EMC_post"] path = sorc/hafs_post.fd url = https://github.com/hafs-community/EMC_post.git From 7ebd9a8b896df1b6b08892a4cd527b78a566deb0 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 23 Aug 2021 15:06:38 -0500 Subject: [PATCH 64/73] Changes requested in review: 1. do not run the atmosphere initialization when running a data atmosphere model 2. Move all cdeps ush stuff into ush/cdeps_utils 3. Remove the docn/datm download jobs, and associated j-jobs and ex-scripts. Users will run the ush-level download scripts directly. 4. Remove the docn/datm prep workflow jobs and J-jobs. Call their ex-scripts from the ocn/atm prep jobs instead. 5. Remove Sam's python, and ensure the workflow runs without it when mesh generation is disabled. --- jobs/JHAFS_ATM_PREP | 9 +- jobs/JHAFS_DATM_DOWNLOAD | 78 ----------- jobs/JHAFS_DATM_PREP | 78 ----------- jobs/JHAFS_DOCN_DOWNLOAD | 78 ----------- jobs/JHAFS_DOCN_PREP | 78 ----------- jobs/JHAFS_OCN_PREP | 9 +- modulefiles/modulefile.hafs.hera | 6 +- modulefiles/modulefile.hafs.jet | 5 +- modulefiles/modulefile.hafs.orion | 5 +- parm/hafs.conf | 6 - parm/hafs_basic.conf | 2 - parm/hafs_datm.conf | 1 - parm/hafs_docn.conf | 1 - parm/hafs_holdvars.txt | 2 - rocoto/cronjob_hafs_datm.sh | 18 +-- rocoto/cronjob_hafs_docn.sh | 22 +-- rocoto/hafs_workflow.xml.in | 130 ++---------------- scripts/exhafs_datm_download.sh | 54 -------- scripts/exhafs_datm_prep.sh | 4 +- scripts/exhafs_docn_download.sh | 50 ------- scripts/exhafs_docn_prep.sh | 12 +- ush/{ => cdeps_utils}/hafs_era5_download.py | 0 ush/{ => cdeps_utils}/hafs_era5_prep.sh | 0 ush/{ => cdeps_utils}/hafs_esmf_mesh.py | 0 ush/{ => cdeps_utils}/hafs_ghrsst_download.py | 0 ush/{ => cdeps_utils}/hafs_ghrsst_prep.sh | 0 ush/{ => cdeps_utils}/hafs_oisst_download.py | 0 ush/{ => cdeps_utils}/hafs_oisst_prep.sh | 0 ush/{ => cdeps_utils}/hafs_rtofs_download.py | 0 ush/{ => cdeps_utils}/hafs_rtofs_prep.sh | 0 ush/cdeps_utils/produtil | 1 + ush/hafs/launcher.py | 18 --- 32 files changed, 55 insertions(+), 612 deletions(-) delete mode 100755 jobs/JHAFS_DATM_DOWNLOAD delete mode 100755 jobs/JHAFS_DATM_PREP delete mode 100755 jobs/JHAFS_DOCN_DOWNLOAD delete mode 100755 jobs/JHAFS_DOCN_PREP delete mode 100755 scripts/exhafs_datm_download.sh delete mode 100755 scripts/exhafs_docn_download.sh rename ush/{ => cdeps_utils}/hafs_era5_download.py (100%) rename ush/{ => cdeps_utils}/hafs_era5_prep.sh (100%) rename ush/{ => cdeps_utils}/hafs_esmf_mesh.py (100%) rename ush/{ => cdeps_utils}/hafs_ghrsst_download.py (100%) rename ush/{ => cdeps_utils}/hafs_ghrsst_prep.sh (100%) rename ush/{ => cdeps_utils}/hafs_oisst_download.py (100%) rename ush/{ => cdeps_utils}/hafs_oisst_prep.sh (100%) rename ush/{ => cdeps_utils}/hafs_rtofs_download.py (100%) rename ush/{ => cdeps_utils}/hafs_rtofs_prep.sh (100%) create mode 120000 ush/cdeps_utils/produtil diff --git a/jobs/JHAFS_ATM_PREP b/jobs/JHAFS_ATM_PREP index 19b074078..1aa31ac98 100755 --- a/jobs/JHAFS_ATM_PREP +++ b/jobs/JHAFS_ATM_PREP @@ -62,8 +62,13 @@ mkdir -p $OUTDIR $DATA cd $DATA # Execute ex-script -${HOMEhafs}/scripts/exhafs_atm_prep.sh -export err=$? +if [[ "${run_datm:-no}" == yes ]] ; then + ${HOMEhafs}/scripts/exhafs_datm_prep.sh + export err=$? +else + ${HOMEhafs}/scripts/exhafs_atm_prep.sh + export err=$? +fi exit $err export KEEPDATA=${KEEPDATA:-YES} diff --git a/jobs/JHAFS_DATM_DOWNLOAD b/jobs/JHAFS_DATM_DOWNLOAD deleted file mode 100755 index 1d1a251be..000000000 --- a/jobs/JHAFS_DATM_DOWNLOAD +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -set -xe - -date -source ${USHhafs}/hafs_pre_job.sh.inc -HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} -source ${HOLDVARS} - -export EXPT=${EXPT:-HAFS} -export SUBEXPT=${SUBEXPT:-${EXPT}} - -export CDATE=${CDATE:-${YMDH}} -export cyc=${cyc:-00} -export STORM=${STORM:-FAKE} -export STORMID=${STORMID:-00L} - -#################################### -# Specify Execution Areas -#################################### -export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -export USHhafs=${USHhafs:-${HOMEhafs}/ush} - -#################################### -# Load Modules if Needed -#################################### -export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet -#export RUN_ENVIR=${RUN_ENVIR:-prod} -export RUN_ENVIR=${envir:-prod} -if [ "$RUN_ENVIR" = prod ]; then - #module use ${HOMEhafs}/modulefiles - #module load modulefile.hafs.run.${machine} - if [ "$machine" = wcoss_dell_p3 ]; then - module unload NetCDF - module unload HDF5-serial - fi - #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} - module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles - module load build.${machine}.intel - module list -fi - -##################################### -# Set up job node/core/run environment variables -##################################### -source ${USHhafs}/hafs_runcmd.sh.inc - -#################################### -# Run setpdy and initialize PDY variables -#################################### -#setpdy.sh -#. ./PDY - -PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) -export PDY=${PDY:-${PDYtmp}} - -##################################### -# Working directories -##################################### -export DATA=${WORKhafs}/datm_mesh - -export SCRUBDATA=${SCRUBDATA:-YES} - -if [ "${SCRUBDATA}" = YES ]; then - rm -rf $DATA -fi - -mkdir -p $DATA -cd $DATA - -##################################### -# Execute the script. -${HOMEhafs}/scripts/exhafs_datm_download.sh -export err=$? -##################################### - -exit $err diff --git a/jobs/JHAFS_DATM_PREP b/jobs/JHAFS_DATM_PREP deleted file mode 100755 index 37c206a98..000000000 --- a/jobs/JHAFS_DATM_PREP +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -set -xe - -date -source ${USHhafs}/hafs_pre_job.sh.inc -HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} -source ${HOLDVARS} - -export EXPT=${EXPT:-HAFS} -export SUBEXPT=${SUBEXPT:-${EXPT}} - -export CDATE=${CDATE:-${YMDH}} -export cyc=${cyc:-00} -export STORM=${STORM:-FAKE} -export STORMID=${STORMID:-00L} - -#################################### -# Specify Execution Areas -#################################### -export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -export USHhafs=${USHhafs:-${HOMEhafs}/ush} - -#################################### -# Load Modules if Needed -#################################### -export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet -#export RUN_ENVIR=${RUN_ENVIR:-prod} -export RUN_ENVIR=${envir:-prod} -if [ "$RUN_ENVIR" = prod ]; then - #module use ${HOMEhafs}/modulefiles - #module load modulefile.hafs.run.${machine} - if [ "$machine" = wcoss_dell_p3 ]; then - module unload NetCDF - module unload HDF5-serial - fi - #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} - module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles - module load build.${machine}.intel - module list -fi - -##################################### -# Set up job node/core/run environment variables -##################################### -source ${USHhafs}/hafs_runcmd.sh.inc - -#################################### -# Run setpdy and initialize PDY variables -#################################### -#setpdy.sh -#. ./PDY - -PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) -export PDY=${PDY:-${PDYtmp}} - -##################################### -# Working directories -##################################### -export DATA=${WORKhafs}/datm_mesh - -export SCRUBDATA=${SCRUBDATA:-YES} - -if [ "${SCRUBDATA}" = YES ]; then - rm -rf $DATA -fi - -mkdir -p $DATA -cd $DATA - -##################################### -# Execute the script. -${HOMEhafs}/scripts/exhafs_datm_prep.sh -export err=$? -##################################### - -exit $err diff --git a/jobs/JHAFS_DOCN_DOWNLOAD b/jobs/JHAFS_DOCN_DOWNLOAD deleted file mode 100755 index ce6e37563..000000000 --- a/jobs/JHAFS_DOCN_DOWNLOAD +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -set -xe - -date -source ${USHhafs}/hafs_pre_job.sh.inc -HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} -source ${HOLDVARS} - -export EXPT=${EXPT:-HAFS} -export SUBEXPT=${SUBEXPT:-${EXPT}} - -export CDATE=${CDATE:-${YMDH}} -export cyc=${cyc:-00} -export STORM=${STORM:-FAKE} -export STORMID=${STORMID:-00L} - -#################################### -# Specify Execution Areas -#################################### -export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -export USHhafs=${USHhafs:-${HOMEhafs}/ush} - -#################################### -# Load Modules if Needed -#################################### -export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet -#export RUN_ENVIR=${RUN_ENVIR:-prod} -export RUN_ENVIR=${envir:-prod} -if [ "$RUN_ENVIR" = prod ]; then - #module use ${HOMEhafs}/modulefiles - #module load modulefile.hafs.run.${machine} - if [ "$machine" = wcoss_dell_p3 ]; then - module unload NetCDF - module unload HDF5-serial - fi - #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} - module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles - module load build.${machine}.intel - module list -fi - -##################################### -# Set up job node/core/run environment variables -##################################### -source ${USHhafs}/hafs_runcmd.sh.inc - -#################################### -# Run setpdy and initialize PDY variables -#################################### -#setpdy.sh -#. ./PDY - -PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) -export PDY=${PDY:-${PDYtmp}} - -##################################### -# Working directories -##################################### -export DATA=${WORKhafs}/docn_mesh - -export SCRUBDATA=${SCRUBDATA:-YES} - -if [ "${SCRUBDATA}" = YES ]; then - rm -rf $DATA -fi - -mkdir -p $DATA -cd $DATA - -##################################### -# Execute the script. -${HOMEhafs}/scripts/exhafs_docn_download.sh -export err=$? -##################################### - -exit $err diff --git a/jobs/JHAFS_DOCN_PREP b/jobs/JHAFS_DOCN_PREP deleted file mode 100755 index db07ebe1d..000000000 --- a/jobs/JHAFS_DOCN_PREP +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -set -xe - -date -source ${USHhafs}/hafs_pre_job.sh.inc -HOLDVARS=${HOLDVARS:-/can/not/find/storm1.holdvars.txt} -source ${HOLDVARS} - -export EXPT=${EXPT:-HAFS} -export SUBEXPT=${SUBEXPT:-${EXPT}} - -export CDATE=${CDATE:-${YMDH}} -export cyc=${cyc:-00} -export STORM=${STORM:-FAKE} -export STORMID=${STORMID:-00L} - -#################################### -# Specify Execution Areas -#################################### -export HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -export WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -export USHhafs=${USHhafs:-${HOMEhafs}/ush} - -#################################### -# Load Modules if Needed -#################################### -export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet -#export RUN_ENVIR=${RUN_ENVIR:-prod} -export RUN_ENVIR=${envir:-prod} -if [ "$RUN_ENVIR" = prod ]; then - #module use ${HOMEhafs}/modulefiles - #module load modulefile.hafs.run.${machine} - if [ "$machine" = wcoss_dell_p3 ]; then - module unload NetCDF - module unload HDF5-serial - fi - #source ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles/build.${machine} - module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles - module load build.${machine}.intel - module list -fi - -##################################### -# Set up job node/core/run environment variables -##################################### -source ${USHhafs}/hafs_runcmd.sh.inc - -#################################### -# Run setpdy and initialize PDY variables -#################################### -#setpdy.sh -#. ./PDY - -PDYtmp=$(echo ${YMDH} | cut -c 1-8 ) -export PDY=${PDY:-${PDYtmp}} - -##################################### -# Working directories -##################################### -export DATA=${WORKhafs}/docn_mesh - -export SCRUBDATA=${SCRUBDATA:-YES} - -if [ "${SCRUBDATA}" = YES ]; then - rm -rf $DATA -fi - -mkdir -p $DATA -cd $DATA - -##################################### -# Execute the script. -${HOMEhafs}/scripts/exhafs_docn_prep.sh -export err=$? -##################################### - -exit $err diff --git a/jobs/JHAFS_OCN_PREP b/jobs/JHAFS_OCN_PREP index 9e1060929..6f6eb5af6 100755 --- a/jobs/JHAFS_OCN_PREP +++ b/jobs/JHAFS_OCN_PREP @@ -51,8 +51,13 @@ mkdir -p $DATA cd $DATA # Execute ex-script -${HOMEhafs}/scripts/exhafs_ocn_prep.py -export err=$? +if [[ "${run_docn:-no}" == yes ]] ; then + ${HOMEhafs}/scripts/exhafs_docn_prep.sh + export err=$? +else + ${HOMEhafs}/scripts/exhafs_ocn_prep.py + export err=$? +fi exit $err export KEEPDATA=${KEEPDATA:-YES} diff --git a/modulefiles/modulefile.hafs.hera b/modulefiles/modulefile.hafs.hera index abd750f33..a2d03f70b 100644 --- a/modulefiles/modulefile.hafs.hera +++ b/modulefiles/modulefile.hafs.hera @@ -68,9 +68,7 @@ module load nco/4.9.1 module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles module load rocoto/1.3.3 -# CDEPS needs extra python modules, of specific versions: -module use /scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/modulefiles -module load python/3.7.10-hafs -#module load intelpython/3.6.8 +module load intelpython/3.6.8 +# For CDEPS data models: module load cdo/1.9.10 diff --git a/modulefiles/modulefile.hafs.jet b/modulefiles/modulefile.hafs.jet index af431bb70..f3f524046 100644 --- a/modulefiles/modulefile.hafs.jet +++ b/modulefiles/modulefile.hafs.jet @@ -64,10 +64,7 @@ setenv WGRIB2 /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/intel-18.0.5.274/impi-201 module load nco/4.9.1 module load rocoto/1.3.3 -#module load intelpython/3.6.5 -# Alternative python with prerequisites for DATM and DOCN: -module use /lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/modulefiles -module load python/3.7.10-hafs +module load intelpython/3.6.5 # For CDEPS data models: module load cdo diff --git a/modulefiles/modulefile.hafs.orion b/modulefiles/modulefile.hafs.orion index 799594001..49b0c2e01 100644 --- a/modulefiles/modulefile.hafs.orion +++ b/modulefiles/modulefile.hafs.orion @@ -64,10 +64,7 @@ module load nco/4.9.3 module load rocoto/1.3.3 -#module load intelpython3/2020 -module use /work/noaa/zrtrr/strahan/modulefiles -module load python/3.7.10-hafs +module load intelpython3/2020 # For CDEPS data models: module load cdo -module load nco diff --git a/parm/hafs.conf b/parm/hafs.conf index b000e62ed..bd06376ba 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -393,10 +393,6 @@ COMgfs={dir/COMgfs} ;; input GFS com directory COMrtofs={dir/COMrtofs} ;; input RTOFS com directory gtype={grid/gtype} ;; grid type: uniform, stretch, nest, or regional (currently only nest and regional have been tested and supported) GFSVER={config/GFSVER} ;; Version of GFS input data, e.g., PROD2019, PROD2021 -DOWNLOAD_DATM_LIST={download_datm_list} ;; List of task ids for era5 downloader -DOWNLOAD_DATM_THRESHOLD={download_datm_threshold} ;; 1.0/(length of download_datm_list) -DOWNLOAD_DOCN_LIST={download_docn_list} ;; List of task ids for era5 downloader -DOWNLOAD_DOCN_THRESHOLD={download_docn_threshold} ;; 1.0/(length of download_docn_list) # Specify the forecast job resources. Only a few combinations are provided. If # needed, you may add other options in the site entity files under rocoto/sites. @@ -424,6 +420,4 @@ RUN_EMCGRAPHICS={run_emcgraphics} ;; Do we run EMC graphics? SCRUB_COM={scrub_com} ;; Should Rocoto scrub the COM directory? SCRUB_WORK={scrub_work} ;; Should Rocoto scrub the WORK directory? MAKE_MESH_ATM={make_mesh_atm} ;; Should the DATM mesh be generated by the workflow? -DOWNLOAD_DATM={download_datm} ;; Should the workflow download DATM files? MAKE_MESH_OCN={make_mesh_ocn} ;; Should the DOCN mesh be generated by the workflow? -DOWNLOAD_DOCN={download_docn} ;; Should the workflow download DOCN files? diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index 020a4a83a..b245aa370 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -54,8 +54,6 @@ run_hrdgraphics=no ;; Run HRD graphics run_emcgraphics=no ;; Run EMC graphics make_mesh_atm=no ;; Generate DATM mesh in workflow (only for run_datm=yes) make_mesh_ocn=no ;; Generate DOCN mesh in workflow (only for run_docn=yes) -datm_download_jobs=0 ;; Number of download_datm jobs (0=disable) -docn_download_jobs=0 ;; Number of download_docn jobs (0=disable) # warm_start_opt: 0, coldstart from chgres; 1, warmstart from init; 2, # warmstart from prior cycle's restart files; 3, warmstart from vortex diff --git a/parm/hafs_datm.conf b/parm/hafs_datm.conf index 87acefeee..b5b20f7c8 100644 --- a/parm/hafs_datm.conf +++ b/parm/hafs_datm.conf @@ -4,7 +4,6 @@ run_docn=no ;; Data ocean using CDEPS run_ocean=yes ;; Whether to run the ocean model. run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_atm=no ;; yes=generate mesh_atm_gen; no=copy from FIXmeshes -datm_download_jobs=3 ;; >0=number of jobs to download missing DATM files; 0=none, fail later if files are missing scrub_com=no ;; the archive job is not set up to handle files generated from datm or docn scrub_work=no diff --git a/parm/hafs_docn.conf b/parm/hafs_docn.conf index 10a8920a8..63f68c7f0 100644 --- a/parm/hafs_docn.conf +++ b/parm/hafs_docn.conf @@ -4,7 +4,6 @@ run_docn=yes ;; Data ocean using CDEPS run_ocean=no ;; Whether to run the ocean model. Must be no if run_docn=yes. run_dwav=no ;; Data waves using CDEPS. Not implemented. make_mesh_ocn=no ;; yes=generate mesh_ocn_gen; no=copy from FIXmeshes -docn_download_jobs=2 ;; >0=number of jobs to download missing DOCN files; 0=none, fail later if files are missing # A second file sets this option: docn_source=OISST ;; OISST, RTOFS, or GHRSST diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 9af743477..1efdd0fd7 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -49,13 +49,11 @@ export run_datm={run_datm} export run_docn={run_docn} export run_dwav={run_dwav} -export datm_download_jobs={datm_download_jobs} export make_mesh_atm={make_mesh_atm} export mesh_atm={mesh_atm} export docn_mesh_nx_global={docn_mesh_nx_global} export docn_mesh_ny_global={docn_mesh_ny_global} -export docn_download_jobs={docn_download_jobs} export make_mesh_ocn={make_mesh_ocn} export mesh_ocn={mesh_ocn} export docn_input_path={docn_input_path} diff --git a/rocoto/cronjob_hafs_datm.sh b/rocoto/cronjob_hafs_datm.sh index 99062b006..3ce40e2a0 100755 --- a/rocoto/cronjob_hafs_datm.sh +++ b/rocoto/cronjob_hafs_datm.sh @@ -13,19 +13,19 @@ date #PYTHON3=/opt/intel/intelpython3/bin/python3 # NOAA RDHPCS Jet -# HOMEhafs=/lfs4/BMC/wrfruc/${USER}/HAFS/HAFS -# dev="-s sites/xjet.ent -f" -# PYTHON3=/lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion - HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS dev="-s sites/orion.ent -f" - PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 -# NOAA RDHPCS Hera -# HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -# dev="-s sites/hera.ent -f" -# PYTHON3=/scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python +#NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 cd ${HOMEhafs}/rocoto diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_docn.sh index 3c26d0f3d..e2ace4a9f 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_docn.sh @@ -13,19 +13,19 @@ date #PYTHON3=/opt/intel/intelpython3/bin/python3 # NOAA RDHPCS Jet -# HOMEhafs=/lfs4/BMC/wrfruc/${USER}/HAFS/HAFS -# dev="-s sites/xjet.ent -f" -# PYTHON3=/lfs4/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion -HOMEhafs=/work/noaa/zrtrr/${USER}/HAFS -dev="-s sites/orion.ent -f" -PYTHON3=/work/noaa/zrtrr/strahan/python-3.7.10/bin/python - -# NOAA RDHPCS Hera -# HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -# dev="-s sites/hera.ent -f" -# PYTHON3=/scratch2/BMC/wrfruc/Samuel.Trahan/HAFS/python-3.7.10/bin/python + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +#NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 cd ${HOMEhafs}/rocoto diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index cc692b0c1..4174d479b 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -81,14 +81,6 @@ - - - - - - - - @@ -210,70 +202,6 @@ -@** if RUN_DATM==YES - - &DOWNLOAD_DATM_LIST; - - &JOBhafs;/JHAFS_DATM_DOWNLOAD - hafs_input_&SID;_@Y@m@d@H_#idx# - &WORKhafs;/hafs_download_datm_#idx#.log - &ACCOUNT; - &RESERVATION; - &QUEUE_SERVICE; - &SERVICE_EXTRA; - 1 - &CORES_EXTRA; - TOTAL_TASKS1 - 06:00:00 - &MEMORY; - &ENV_VARS; - - - - - &DOWNLOAD_DATM;YES - &RUN_DATM;YES - - - - - - - -@** endif - -@** if RUN_DOCN==YES - - &DOWNLOAD_DOCN_LIST; - - &JOBhafs;/JHAFS_DOCN_DOWNLOAD - hafs_input_&SID;_@Y@m@d@H_#idx# - &WORKhafs;/hafs_download_docn_#idx#.log - &ACCOUNT; - &RESERVATION; - &QUEUE_SERVICE; - &SERVICE_EXTRA; - 1 - &CORES_EXTRA; - TOTAL_TASKS1 - 06:00:00 - &MORE_MEMORY; - &ENV_VARS; - - - - - &DOWNLOAD_DOCN;YES - &RUN_DOCN;YES - - - - - - - -@** endif - &JOBhafs;/JHAFS_ATM_PREP hafs_atm_prep_&SID;_@Y@m@d@H @@ -371,10 +299,10 @@ @** if RUN_DOCN==YES - - &JOBhafs;/JHAFS_DOCN_PREP - hafs_docn_prep_&SID;_@Y@m@d@H - &WORKhafs;/hafs_docn_prep.log + + &JOBhafs;/JHAFS_OCN_PREP + hafs_ocn_prep_&SID;_@Y@m@d@H + &WORKhafs;/hafs_ocn_prep.log &ACCOUNT; &RESERVATION; &QUEUE_PE; @@ -383,18 +311,10 @@ &ENV_VARS; - - - - &DOWNLOAD_DOCN;YES - - - + -@** endif - -@** if RUN_OCEAN==YES +@** elseif RUN_OCEAN==YES &JOBhafs;/JHAFS_OCN_PREP hafs_ocn_prep_&SID;_@Y@m@d@H @@ -596,32 +516,6 @@ @** endif @** endif -@** if RUN_DATM==YES - - - &JOBhafs;/JHAFS_DATM_PREP - hafs_datm_prep_&SID;_@Y@m@d@H - &WORKhafs;/hafs_datm_prep.log - &ACCOUNT; - &RESERVATION; - &QUEUE_PE; - &PE_EXTRA; - &DATM_PREP_RESOURCES; - &ENV_VARS; - - - - - - &DOWNLOAD_DATM;YES - - - - - - -@** endif - @** if RUN_ATM_INIT==YES &JOBhafs;/JHAFS_ATM_INIT @@ -659,18 +553,10 @@ @** if RUN_DATM==YES - - &DOWNLOAD_DATM;YES - - - + @** endif @** if RUN_DOCN==YES - - &DOWNLOAD_DOCN;YES - - - + @** endif &RUN_DATM;YES diff --git a/scripts/exhafs_datm_download.sh b/scripts/exhafs_datm_download.sh deleted file mode 100755 index 3b7bc051a..000000000 --- a/scripts/exhafs_datm_download.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -if [[ "$run_datm" != yes || ! "$datm_download_jobs" -gt 0 ]] ; then - echo "This job should only be run if \$run_datm=yes and \$datm_download_jobs>0" - echo " \$datm_download_jobs=\"$datm_download_jobs\"" - echo " \$run_datm=\"$run_datm\"" - echo "Billions of electrons have whizzed by, the universe's entropy has increased, and yet nothing has been accomplished." - echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " - exit 0 -fi - -if ( ! which cdo ) ; then - echo "The \"cdo\" program is not in your \$PATH. This script uses cdo to" 1>&2 - echo "reverse latitudes. Go get cdo." 1>&2 - echo " -> SCRIPT IS EXITING BECAUSE cdo IS NOT IN \$PATH <- " 1>&2 - exit 1 -fi - -set -uxe - -HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -USHhafs=${USHhafs:-${HOMEhafs}/ush} -CDATE=${CDATE:-${YMDH}} -NHRS=${NHRS:?NHRS is not set} -DATMdir=${DATMdir:?DATMdir is not set} - -export TZ=UTC # Orion workaround - -M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) -P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) - -[ -d "$DATMdir" ] || mkdir -p "$DATMdir" -test -d "$DATMdir" -cd "$DATMdir" - -echo "Downloading ERA5 files and reversing latitudes." -echo "Start date: ${M1DATE:0:8}" -echo "End date: ${P1DATE:0:8}" -echo "Destination dir: \"$DATMdir\"" - -datm_source=${DATM_SOURCE:-ERA5} - -if [[ "$datm_source" == ERA5 ]] ; then - "$USHhafs/hafs_era5_download.py" ${M1DATE:0:8}-${P1DATE:0:8} -else - echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 - echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DATM_SOURCE VALUE <- " - exit 1 -fi - - -echo "Successfully downloaded all DATM $datm_source files." -echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_datm_prep.sh b/scripts/exhafs_datm_prep.sh index 2e156bc4f..20b6b1e38 100755 --- a/scripts/exhafs_datm_prep.sh +++ b/scripts/exhafs_datm_prep.sh @@ -31,7 +31,7 @@ if [[ "$make_mesh_ocn" == yes ]] ; then fi if [[ "$datm_source" == ERA5 ]] ; then - "$USHhafs/hafs_era5_prep.sh" "$datm_input_path" + $APRUNS "$USHhafs/cdeps_utils/hafs_era5_prep.sh" "$datm_input_path" else echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DATM_SOURCE VALUE <- " @@ -63,7 +63,7 @@ set -x # Generate the mesh. if [[ "$datm_source" == ERA5 ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ + $APRUNS $USHhafs/cdeps_utils/hafs_esmf_mesh.py --ifile "$ifile" --ofile "$ofile" \ --overwrite --latvar latitude --lonvar longitude --double else echo "ERROR: Unknown data atmosphere source $datm_source. Giving up." 2>&1 diff --git a/scripts/exhafs_docn_download.sh b/scripts/exhafs_docn_download.sh deleted file mode 100755 index 7bef09715..000000000 --- a/scripts/exhafs_docn_download.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -if [[ "$run_docn" != yes || ! "$docn_download_jobs" -gt 0 ]] ; then - echo "This job should only be run if \$run_docn=yes and \$docn_download_jobs>0" - echo " \$docn_download_jobs=\"$docn_download_jobs\"" - echo " \$run_docn=\"$run_docn\"" - echo "Beware! You may anger Poseidon by misusing this script. Avoid coastlines." - echo " -> SCRIPT IS EXITING BECAUSE THIS JOB SHOULD NOT BE RUN <- " - exit 0 -fi - -set -uxe - -HOMEhafs=${HOMEhafs:-/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS} -WORKhafs=${WORKhafs:-/gpfs/hps3/ptmp/${USER}/${SUBEXPT}/${CDATE}/${STORMID}} -USHhafs=${USHhafs:-${HOMEhafs}/ush} -CDATE=${CDATE:-${YMDH}} -NHRS=${NHRS:?NHRS is not set} -DOCNdir=${DOCNdir:?DOCNdir is not set} - -export TZ=UTC # Orion workaround - -M1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 -24 hours" +%Y%m%d%H ) -P1DATE=$( date -d "${CDATE:0:4}-${CDATE:4:2}-${CDATE:6:2}t${CDATE:8:2}:00:00+00 +$(( NHRS+24 )) hours" +%Y%m%d%H ) - -[ -d "$DOCNdir" ] || mkdir -p "$DOCNdir" -test -d "$DOCNdir" -cd "$DOCNdir" - -echo "Downloading OISST files." -echo "Start date: ${M1DATE:0:8}" -echo "End date: ${P1DATE:0:8}" -echo "Destination dir: \"$DOCNdir\"" - -docn_source=${DOCN_SOURCE:-OISST} - -if [[ "$docn_source" == OISST ]] ; then - "$USHhafs/hafs_oisst_download.py" ${M1DATE:0:8}-${P1DATE:0:8} -elif [[ "${docn_source}" == RTOFS ]] ; then - "$USHhafs/hafs_rtofs_download.py" ${CDATE:0:8} -elif [[ "${docn_source}" == GHRSST ]] ; then - "$USHhafs/hafs_ghrsst_download.py" ${M1DATE:0:8}-${P1DATE:0:8} -else - echo "ERROR: Unknown data ocean source $docn_source. Giving up." 2>&1 - echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DOCN_SOURCE VALUE <- " - exit 1 -fi - -echo "Successfully downloaded all DOCN $docn_source files." -echo "Enjoy your files and have a nice day." diff --git a/scripts/exhafs_docn_prep.sh b/scripts/exhafs_docn_prep.sh index a76571afc..7a532c359 100755 --- a/scripts/exhafs_docn_prep.sh +++ b/scripts/exhafs_docn_prep.sh @@ -31,11 +31,11 @@ if [[ "$make_mesh_ocn" == yes ]] ; then fi if [[ "$docn_source" == OISST ]] ; then - "$USHhafs/hafs_oisst_prep.sh" "$docn_input_path" + $APRUNS "$USHhafs/cdeps_utils/hafs_oisst_prep.sh" "$docn_input_path" elif [[ "${docn_source}" == RTOFS ]] ; then - "$USHhafs/hafs_rtofs_prep.sh" "$docn_input_path" + $APRUNS "$USHhafs/cdeps_utils/hafs_rtofs_prep.sh" "$docn_input_path" elif [[ "${docn_source}" == GHRSST ]] ; then - "$USHhafs/hafs_ghrsst_prep.sh" "$docn_input_path" + $APRUNS "$USHhafs/cdeps_utils/hafs_ghrsst_prep.sh" "$docn_input_path" else echo "ERROR: Unknown data ocean source $docn_source. Giving up." 2>&1 echo " -> SCRIPT IS FAILING BECAUSE OF INVALID \$DOCN_SOURCE VALUE <- " @@ -61,14 +61,14 @@ file0=$docn_input_path/DOCN_input_00000.nc # Generate the mesh from the merged file. if [[ "$docn_source" == OISST ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ + $APRUNS $USHhafs/cdeps_utils/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --maskvar sst --maskcal --double --overwrite elif [[ "${docn_source}" == RTOFS ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ + $APRUNS $USHhafs/cdeps_utils/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --overwrite --latvar Latitude --lonvar Longitude \ --maskvar sst --maskcal —double elif [[ "${docn_source}" == GHRSST ]] ; then - $APRUNS $USHhafs/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ + $APRUNS $USHhafs/cdeps_utils/hafs_esmf_mesh.py --ifile "$file0" --ofile "$ofile" \ --maskvar analysed_sst --maskcal --overwrite --double fi test -s "$ofile" diff --git a/ush/hafs_era5_download.py b/ush/cdeps_utils/hafs_era5_download.py similarity index 100% rename from ush/hafs_era5_download.py rename to ush/cdeps_utils/hafs_era5_download.py diff --git a/ush/hafs_era5_prep.sh b/ush/cdeps_utils/hafs_era5_prep.sh similarity index 100% rename from ush/hafs_era5_prep.sh rename to ush/cdeps_utils/hafs_era5_prep.sh diff --git a/ush/hafs_esmf_mesh.py b/ush/cdeps_utils/hafs_esmf_mesh.py similarity index 100% rename from ush/hafs_esmf_mesh.py rename to ush/cdeps_utils/hafs_esmf_mesh.py diff --git a/ush/hafs_ghrsst_download.py b/ush/cdeps_utils/hafs_ghrsst_download.py similarity index 100% rename from ush/hafs_ghrsst_download.py rename to ush/cdeps_utils/hafs_ghrsst_download.py diff --git a/ush/hafs_ghrsst_prep.sh b/ush/cdeps_utils/hafs_ghrsst_prep.sh similarity index 100% rename from ush/hafs_ghrsst_prep.sh rename to ush/cdeps_utils/hafs_ghrsst_prep.sh diff --git a/ush/hafs_oisst_download.py b/ush/cdeps_utils/hafs_oisst_download.py similarity index 100% rename from ush/hafs_oisst_download.py rename to ush/cdeps_utils/hafs_oisst_download.py diff --git a/ush/hafs_oisst_prep.sh b/ush/cdeps_utils/hafs_oisst_prep.sh similarity index 100% rename from ush/hafs_oisst_prep.sh rename to ush/cdeps_utils/hafs_oisst_prep.sh diff --git a/ush/hafs_rtofs_download.py b/ush/cdeps_utils/hafs_rtofs_download.py similarity index 100% rename from ush/hafs_rtofs_download.py rename to ush/cdeps_utils/hafs_rtofs_download.py diff --git a/ush/hafs_rtofs_prep.sh b/ush/cdeps_utils/hafs_rtofs_prep.sh similarity index 100% rename from ush/hafs_rtofs_prep.sh rename to ush/cdeps_utils/hafs_rtofs_prep.sh diff --git a/ush/cdeps_utils/produtil b/ush/cdeps_utils/produtil new file mode 120000 index 000000000..f6a8dbf85 --- /dev/null +++ b/ush/cdeps_utils/produtil @@ -0,0 +1 @@ +../produtil/ \ No newline at end of file diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index e4ef19a31..acd762515 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -853,34 +853,16 @@ def set_data_model_variables(self): self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_gen')) else: self.set('forecast','mesh_atm',self.getraw('forecast','mesh_atm_in')) - datm_download_jobs=max(0,self.getint('config','datm_download_jobs',0)) - download_datm_list=" ".join(["%02d"%x for x in range(max(1,datm_download_jobs))]) - download_datm_threshold="%7.5f"%(1.0/(max(1,datm_download_jobs)),) - self.set('config','download_datm',datm_download_jobs>0) - self.set('config','download_datm_list',download_datm_list) - self.set('config','download_datm_threshold',download_datm_threshold) else: self.set('forecast','mesh_atm','dummy.nc') - self.set('config','download_datm',False) - self.set('config','download_datm_list','0') - self.set('config','download_datm_threshold','1.0') if run_docn: make_mesh_ocn=self.getbool('config','make_mesh_ocn',True) if make_mesh_ocn: self.set('forecast','mesh_ocn',self.getraw('forecast','mesh_ocn_gen')) else: self.set('forecast','mesh_ocn',self.getraw('forecast','mesh_ocn_in')) - docn_download_jobs=max(0,self.getint('config','docn_download_jobs',0)) - download_docn_list=" ".join(["%02d"%x for x in range(max(1,docn_download_jobs))]) - download_docn_threshold="%7.5f"%(1.0/(max(1,docn_download_jobs)),) - self.set('config','download_docn',docn_download_jobs>0) - self.set('config','download_docn_list',download_docn_list) - self.set('config','download_docn_threshold',download_docn_threshold) else: self.set('forecast','mesh_ocn','dummy.nc') - self.set('config','download_docn',False) - self.set('config','download_docn_list','0') - self.set('config','download_docn_threshold','1.0') def set_storm(self,syndat,oldsyndat): """!Sets the storm that is to be run. From 75eb3212c616e586880535a63fced257ce428f01 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Mon, 23 Aug 2021 15:07:15 -0500 Subject: [PATCH 65/73] Use batch for forecast job on Orion, to avoid overuse of "urgent" --- rocoto/sites/orion.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 56d98f7b6..b6bd4b2b2 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -2,7 +2,7 @@ - + --partition=orion"> --partition=orion"> From 83a08a2c1965ee769c4d43b889613398c24ce842 Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 24 Aug 2021 17:21:15 +0000 Subject: [PATCH 66/73] Change mapalgo01 from bilinear to redist since the same grid is used for the input data and data model (suggested by uturuncoglu). --- parm/cdeps/datm_era5.streams | 2 +- parm/cdeps/docn_ghrsst.streams | 2 +- parm/cdeps/docn_oisst.streams | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/cdeps/datm_era5.streams b/parm/cdeps/datm_era5.streams index b8e3a91e3..c1332f62f 100644 --- a/parm/cdeps/datm_era5.streams +++ b/parm/cdeps/datm_era5.streams @@ -1,6 +1,6 @@ stream_info: ERA5_HOURLY01 taxmode01: limit -mapalgo01: bilinear +mapalgo01: redist tInterpAlgo01: linear readMode01: single dtlimit01: 1.5 diff --git a/parm/cdeps/docn_ghrsst.streams b/parm/cdeps/docn_ghrsst.streams index afff5191c..dbbf450c4 100644 --- a/parm/cdeps/docn_ghrsst.streams +++ b/parm/cdeps/docn_ghrsst.streams @@ -1,6 +1,6 @@ stream_info: PRESCRIBED01 taxmode01: limit -mapalgo01: bilinear +mapalgo01: redist tInterpAlgo01: linear readMode01: single dtlimit01: 1.5 diff --git a/parm/cdeps/docn_oisst.streams b/parm/cdeps/docn_oisst.streams index 68501c732..f72a86858 100644 --- a/parm/cdeps/docn_oisst.streams +++ b/parm/cdeps/docn_oisst.streams @@ -1,6 +1,6 @@ stream_info: PRESCRIBED01 taxmode01: limit -mapalgo01: bilinear +mapalgo01: redist tInterpAlgo01: linear readMode01: single dtlimit01: 1.5 From 44701f699558e8783a57796cb3016ef4b1836bd3 Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 24 Aug 2021 17:51:06 +0000 Subject: [PATCH 67/73] Revert changes to build_forecast.sh. We agreed to only specify -DAPP=HAFS in this file. The user can change it to -DAPP=HAFS-ALL if they want to build with CDEPS support, as described in the documentation. --- sorc/build_forecast.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index ebe72a14a..9bf7f918f 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -10,10 +10,6 @@ if [ $target = cheyenne ]; then target=cheyenne.intel ; fi cd hafs_forecast.fd/tests -# For normal HAFS: -#./compile.sh "$target" "-DAPP=HAFS -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO - -# For normal HAFS + HAFS with data models: -./compile.sh "$target" "-DAPP=HAFS-ALL -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO +./compile.sh "$target" "-DAPP=HAFS -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO exit From 984957888cd52c66393f4dace1ef1d0bb602c219 Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 30 Aug 2021 22:57:48 +0000 Subject: [PATCH 68/73] Small adjustments to job resources on Hera. --- rocoto/sites/hera.ent | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index e1c94af61..1f0067d1d 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -95,10 +95,13 @@ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> - 71:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 71:ppn=20:tpp=2TOTAL_TASKS1420NCTSK20&FORECAST_EXTRA;"> 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=40:tpp=1TOTAL_TASKS240NCTSK40OMP_THREADS103:00:00"> + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> From f19d1f2b49c56fc53b1a2bf655b8401276072a03 Mon Sep 17 00:00:00 2001 From: evankalina Date: Mon, 30 Aug 2021 23:06:24 +0000 Subject: [PATCH 69/73] Small adjustment to resources on xjet. --- rocoto/sites/xjet.ent | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 887cb12c2..695c68ba8 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -100,6 +100,9 @@ 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + + 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00"> + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> From 2539762c442246e606a2ef0b27a316face75040e Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 31 Aug 2021 15:44:30 -0500 Subject: [PATCH 70/73] Changes requested from code review * Delete unnecessary whitespace changes in hafs_workflow.xml.in * Revert unintended change to forecast walltime on kjet * Modify build_forecast.sh to build app=HAFS-ALL, except on wcoss_cray * Combine the DATM and DOCN cronjob scripts into a general CDEPS cronjob, and delete commented out run commands. --- ...job_hafs_docn.sh => cronjob_hafs_cdeps.sh} | 17 +++-- rocoto/cronjob_hafs_datm.sh | 63 ------------------- rocoto/hafs_workflow.xml.in | 10 +-- rocoto/sites/kjet.ent | 2 +- sorc/build_forecast.sh | 7 ++- 5 files changed, 19 insertions(+), 80 deletions(-) rename rocoto/{cronjob_hafs_docn.sh => cronjob_hafs_cdeps.sh} (81%) delete mode 100755 rocoto/cronjob_hafs_datm.sh diff --git a/rocoto/cronjob_hafs_docn.sh b/rocoto/cronjob_hafs_cdeps.sh similarity index 81% rename from rocoto/cronjob_hafs_docn.sh rename to rocoto/cronjob_hafs_cdeps.sh index e2ace4a9f..41a07144c 100755 --- a/rocoto/cronjob_hafs_docn.sh +++ b/rocoto/cronjob_hafs_cdeps.sh @@ -34,6 +34,13 @@ EXPT=$(basename ${HOMEhafs}) #=============================================================================== # Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh +# Run data atmosphere with ERA5 +${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + config.SUBEXPT=${EXPT}_era5 \ + forecast.output_history=.true. \ + ../parm/hafs_regional_static.conf ../parm/hafs_hycom.conf \ + ../parm/hafs_datm.conf ../parm/hafs_datm_era5.conf + # Run data ocean with OISST ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ config.SUBEXPT=${EXPT}_oisst \ @@ -48,16 +55,6 @@ ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ ../parm/hafs_regional_static.conf \ ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf -# Run all cycles of a storm -#${PYTHON3} ./run_hafs.py ${dev} 2020 13L HISTORY config.EXPT=${EXPT} # Laura - -# Run specified cycles of a storm -#${PYTHON3} ./run_hafs.py ${dev} 2020082506-2020082512 13L HISTORY \ -# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} # Laura - -# Run one cycle of a storm -# ${PYTHON3} ./run_hafs.py -t ${dev} 2020082512 13L HISTORY config.EXPT=${EXPT} - #=============================================================================== date diff --git a/rocoto/cronjob_hafs_datm.sh b/rocoto/cronjob_hafs_datm.sh deleted file mode 100755 index 3ce40e2a0..000000000 --- a/rocoto/cronjob_hafs_datm.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -set -x -date - -# NOAA WCOSS Dell Phase3 -#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS -#dev="-s sites/wcoss_dell_p3.ent -f" -#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 - -# NOAA WCOSS Cray -#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS -#dev="-s sites/wcoss_cray.ent -f" -#PYTHON3=/opt/intel/intelpython3/bin/python3 - -# NOAA RDHPCS Jet -#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS -#dev="-s sites/xjet.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 - -# MSU Orion - HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS - dev="-s sites/orion.ent -f" - PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 - -#NOAA RDHPCS Hera -#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -#dev="-s sites/hera.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 - -cd ${HOMEhafs}/rocoto - -EXPT=$(basename ${HOMEhafs}) - -#=============================================================================== -# Here are some simple examples, more examples can be seen in cronjob_hafs_rt.sh - - # ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - # config.SUBEXPT=${EXPT}_ghrsst \ - # forecast.output_history=.true. \ - # ../parm/hafs_regional_static.conf \ - # ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf - - ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - config.SUBEXPT=${EXPT}_era5 \ - forecast.output_history=.true. \ - ../parm/hafs_regional_static.conf ../parm/hafs_hycom.conf \ - ../parm/hafs_datm.conf ../parm/hafs_datm_era5.conf - -# Run all cycles of a storm -#${PYTHON3} ./run_hafs.py ${dev} 2020 13L HISTORY config.EXPT=${EXPT} # Laura - -# Run specified cycles of a storm -#${PYTHON3} ./run_hafs.py ${dev} 2020082506-2020082512 13L HISTORY \ -# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} # Laura - -# Run one cycle of a storm -# ${PYTHON3} ./run_hafs.py -t ${dev} 2020082512 13L HISTORY config.EXPT=${EXPT} - -#=============================================================================== - -date - -echo 'cronjob done' diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 4174d479b..b1500fefb 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -233,7 +233,7 @@ - + &RUN_DATM;NO @@ -284,14 +284,14 @@ @Y@m@d@H@M@S -@** if GFSVER==PROD2019 +@** if GFSVER==PROD2019 &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf126.nemsio &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.pgrb2.0p25.f126 -@** endif -@** if GFSVER==PROD2021 +@** endif +@** if GFSVER==PROD2021 &COMgfs;/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f126 &COMgfs;/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.atmf0126.nc -@** endif +@** endif diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 3fa441b38..803ddd55a 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -29,7 +29,7 @@ 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 00:59:00"> + 07:59:00"> OMP_THREADS2"> diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 9bf7f918f..70b51939a 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -8,8 +8,13 @@ if [ $target = orion ]; then target=orion.intel ; fi if [ $target = jet ]; then target=jet.intel ; fi if [ $target = cheyenne ]; then target=cheyenne.intel ; fi +if [ $target = wcoss_cray ]; then + app=HAFS +else + app=HAFS-ALL +fi cd hafs_forecast.fd/tests -./compile.sh "$target" "-DAPP=HAFS -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO +./compile.sh "$target" "-DAPP=$app -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO exit From d76f4506061ede3c05dd0899b86cdee744834e81 Mon Sep 17 00:00:00 2001 From: evankalina Date: Tue, 31 Aug 2021 15:49:34 -0500 Subject: [PATCH 71/73] Correct whitespace in CDEPS cron script. --- rocoto/cronjob_hafs_cdeps.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rocoto/cronjob_hafs_cdeps.sh b/rocoto/cronjob_hafs_cdeps.sh index 41a07144c..980d372df 100755 --- a/rocoto/cronjob_hafs_cdeps.sh +++ b/rocoto/cronjob_hafs_cdeps.sh @@ -49,11 +49,11 @@ ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ ../parm/hafs_docn.conf ../parm/hafs_docn_oisst.conf # Run data ocean with GHRSST - ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ - config.SUBEXPT=${EXPT}_ghrsst \ - forecast.output_history=.true. \ - ../parm/hafs_regional_static.conf \ - ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf +${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY config.EXPT=${EXPT} \ + config.SUBEXPT=${EXPT}_ghrsst \ + forecast.output_history=.true. \ + ../parm/hafs_regional_static.conf \ + ../parm/hafs_docn.conf ../parm/hafs_docn_ghrsst.conf #=============================================================================== From 9daecef4e2a5ef26690ef28ba269a5775ef072e1 Mon Sep 17 00:00:00 2001 From: evankalina Date: Wed, 1 Sep 2021 10:34:43 -0500 Subject: [PATCH 72/73] Fix logic error in hafs_workflow.xml.in * ocn_prep should be a dependency for the forecast job regardless of whether we are running the global nesting or regional HAFS. From Lin Zhu (IMSG at NOAA/EMC). --- rocoto/hafs_workflow.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index b1500fefb..8c5062532 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -569,12 +569,12 @@ @** endif @** if RUN_GSI_VR==YES +@** endif @** endif @** if RUN_OCEAN==YES -@** endif @** endif From a98516a045a86f523de207147f15c94c11ec6bad Mon Sep 17 00:00:00 2001 From: "john.steffen" Date: Thu, 9 Sep 2021 14:07:14 -0500 Subject: [PATCH 73/73] Adds 2 ENTITY FORECAST_RESOURCES lines for running HAFS-CDEPS on wcoss_dell_p3 and adds CDO module load to modulefile --- modulefiles/modulefile.hafs.wcoss_dell_p3 | 3 +++ rocoto/sites/wcoss_dell_p3.ent | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/modulefiles/modulefile.hafs.wcoss_dell_p3 b/modulefiles/modulefile.hafs.wcoss_dell_p3 index 183be309b..3ceb46949 100644 --- a/modulefiles/modulefile.hafs.wcoss_dell_p3 +++ b/modulefiles/modulefile.hafs.wcoss_dell_p3 @@ -73,3 +73,6 @@ setenv CMAKE_Platform wcoss_dell_p3 module use /usrx/local/dev/emc_rocoto/modulefiles module load ruby/2.5.1 module load rocoto/1.3.0rc2 + +# For CDEPS data models: +module load cdo diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 1542d68de..95586e5c4 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -92,6 +92,11 @@ 79:ppn=20:tpp=1TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> 103:ppn=20:tpp=1TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00"> + 114:ppn=12:tpp=1+1:ppn=8:tpp=1TOTAL_TASKS1376NCTSK12&FORECAST_EXTRA;"> + + 52:ppn=12:tpp=1TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> 140:ppn=12:tpp=1TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">