diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..b2c92b483fc --- /dev/null +++ b/.gitignore @@ -0,0 +1,161 @@ +__pycache__ +exec/ +fix/0readme +fix/fix_am +fix/fix_chem +fix/fix_fv3 +fix/fix_fv3_gmted2010 +fix/fix_gldas +fix/fix_gsi +fix/fix_gsi_MISSING +fix/fix_orog +fix/fix_sfc_climo +fix/fix_verif +fix/fix_wave_gfs +fix/gdas/ +fix/wafs +jobs/JGDAS_ENKF_FCST +jobs/JGDAS_ENKF_POST +jobs/JGDAS_ENKF_RECENTER +jobs/JGDAS_VERFOZN +jobs/JGDAS_VERFRAD +jobs/JGDAS_VMINMON +jobs/JGFS_VMINMON +jobs/JGFS_WAFS +jobs/JGFS_WAFS_BLENDING +jobs/JGFS_WAFS_GCIP +jobs/JGFS_WAFS_GRIB2 +jobs/JGLOBAL_ANALYSIS +jobs/JGLOBAL_ENKF_INNOVATE_OBS +jobs/JGLOBAL_ENKF_SELECT_OBS +jobs/JGLOBAL_ENKF_UPDATE +jobs/JGLOBAL_NCEPPOST +jobs/JGLOBAL_POST_MANAGER +parm/config/config.base +parm/mon/ +parm/post +parm/wafs +scripts/exemcsfc_global_sfc_prep.sh.ecf +scripts/exgdas_nceppost.sh.ecf +scripts/exgdas_vrfminmon.sh.ecf +scripts/exgdas_vrfyozn.sh.ecf +scripts/exgdas_vrfyrad.sh.ecf +scripts/exgfs_nceppost.sh.ecf +scripts/exgfs_vrfminmon.sh.ecf +scripts/exgfs_wafs_blending.sh.ecf +scripts/exgfs_wafs_gcip.sh.ecf +scripts/exgfs_wafs_grib.sh.ecf +scripts/exgfs_wafs_grib2.sh.ecf +scripts/exglobal_analysis_fv3gfs.sh.ecf +scripts/exglobal_enkf_fcst_fv3gfs.sh.ecf +scripts/exglobal_enkf_innovate_obs_fv3gfs.sh.ecf +scripts/exglobal_enkf_post_fv3gfs.sh.ecf +scripts/exglobal_enkf_recenter_fv3gfs.sh.ecf +scripts/exglobal_enkf_update_fv3gfs.sh.ecf +scripts/exglobal_innovate_obs_fv3gfs.sh.ecf +scripts/exglobal_pmgr.sh.ecf +sorc/calc_increment_ens.fd +sorc/checkout-fv3gfs.log +sorc/checkout-gfs_post.log +sorc/checkout-gfs_wafs.log +sorc/checkout-gsi.log +sorc/checkout-ufs_utils.fd.log +sorc/checkout-verif-global.log +sorc/chgres_cube.fd +sorc/emcsfc_ice_blend.fd +sorc/emcsfc_snow2mdl.fd +sorc/filter_topo.fd +sorc/fregrid.fd +sorc/fv3gfs.fd/ +sorc/getsfcensmeanp.fd +sorc/getsigensmeanp_smooth.fd +sorc/getsigensstatp.fd +sorc/gfs_bufr.fd/bfrhdr.o +sorc/gfs_bufr.fd/bfrize.o +sorc/gfs_bufr.fd/buff.o +sorc/gfs_bufr.fd/calpreciptype.o +sorc/gfs_bufr.fd/funcphys.mod +sorc/gfs_bufr.fd/funcphys.o +sorc/gfs_bufr.fd/gslp.o +sorc/gfs_bufr.fd/lcl.o +sorc/gfs_bufr.fd/machine.mod +sorc/gfs_bufr.fd/machine.o +sorc/gfs_bufr.fd/meteorg.o +sorc/gfs_bufr.fd/modstuff1.o +sorc/gfs_bufr.fd/mstadb.o +sorc/gfs_bufr.fd/newsig1.o +sorc/gfs_bufr.fd/physcons.mod +sorc/gfs_bufr.fd/physcons.o +sorc/gfs_bufr.fd/rsearch.o +sorc/gfs_bufr.fd/svp.o +sorc/gfs_bufr.fd/tdew.o +sorc/gfs_bufr.fd/terp3.o +sorc/gfs_bufr.fd/vintg.o +sorc/gfs_ncep_post.fd +sorc/gfs_post.fd/ +sorc/gfs_wafs.fd/ +sorc/global_chgres.fd +sorc/global_cycle.fd +sorc/global_enkf.fd +sorc/global_gsi.fd +sorc/gsi.fd/ +sorc/logs/ +sorc/make_hgrid.fd +sorc/make_solo_mosaic.fd +sorc/mkgfsnemsioctl.fd +sorc/nemsio_get.fd +sorc/nemsio_read.fd +sorc/nst_tf_chg.fd +sorc/orog.fd +sorc/oznmon_horiz.fd +sorc/oznmon_time.fd +sorc/radmon_angle.fd +sorc/radmon_bcoef.fd +sorc/radmon_bcor.fd +sorc/radmon_time.fd +sorc/recentersigp.fd +sorc/shave.fd +sorc/tocsbufr.fd/tocsbufr.o +sorc/ufs_utils.fd/ +sorc/verif-global.fd/ +sorc/wafs_awc_wafavn.fd +sorc/wafs_blending.fd +sorc/wafs_cnvgrib2.fd +sorc/wafs_gcip.fd +sorc/wafs_makewafs.fd +sorc/wafs_setmissing.fd +ush/emcsfc_ice_blend.sh +ush/emcsfc_snow.sh +ush/fix_precip.sh +ush/fv3gfs_chgres.sh +ush/fv3gfs_downstream_nems.sh +ush/fv3gfs_driver_grid.sh +ush/fv3gfs_dwn_nems.sh +ush/fv3gfs_filter_topo.sh +ush/fv3gfs_make_grid.sh +ush/fv3gfs_make_orog.sh +ush/gfs_nceppost.sh +ush/gfs_transfer.sh +ush/global_chgres.sh +ush/global_chgres_driver.sh +ush/global_cycle.sh +ush/global_cycle_driver.sh +ush/link_crtm_fix.sh +ush/minmon_xtrct_costs.pl +ush/minmon_xtrct_gnorms.pl +ush/minmon_xtrct_reduct.pl +ush/mkwfsgbl.sh +ush/ozn_xtrct.sh +ush/radmon_ck_stdout.sh +ush/radmon_err_rpt.sh +ush/radmon_verf_angle.sh +ush/radmon_verf_bcoef.sh +ush/radmon_verf_bcor.sh +ush/radmon_verf_time.sh +ush/trim_rh.sh +ush/wafs_blending.sh +ush/wafs_intdsk.sh +util/sub_slurm +util/sub_wcoss_c +util/sub_wcoss_d +workflow/user.yaml diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..949019d8c9a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "CROW"] + path = workflow/CROW + url = https://github.com/NOAA-EMC/CROW.git diff --git a/workflow/CROW b/workflow/CROW new file mode 160000 index 00000000000..55a5e3010f7 --- /dev/null +++ b/workflow/CROW @@ -0,0 +1 @@ +Subproject commit 55a5e3010f754461489bec97f2279bac7278dda4 diff --git a/workflow/_expdir_main_auto.yaml b/workflow/_expdir_main_auto.yaml new file mode 100644 index 00000000000..e90b34a0e62 --- /dev/null +++ b/workflow/_expdir_main_auto.yaml @@ -0,0 +1,19 @@ +# This file is copied into the experiment directory as _main.yaml. The +# Rocoto and ecFlow workflow generation scripts use it to decide what +# YAML files to read from the experiment directory + +#NOTE: If you update this file, you must also update worktools.py +# variables: YAML_DIRS_TO_COPY, YAML_FILES_TO_COPY, and +# function make_yaml_files + +include: + - names.yaml + - platform.yaml + - resources_sum.yaml + - 'static*yaml' + - schema/ + - defaults/ + - case.yaml + - runtime/ + - workflow.yaml + - config/ diff --git a/workflow/_expdir_main_manual.yaml b/workflow/_expdir_main_manual.yaml new file mode 100644 index 00000000000..0d0a83a8ebb --- /dev/null +++ b/workflow/_expdir_main_manual.yaml @@ -0,0 +1,21 @@ +# This file is copied into the experiment directory as _main.yaml. The +# Rocoto and ecFlow workflow generation scripts use it to decide what +# YAML files to read from the experiment directory + +#NOTE: If you update this file, you must also update worktools.py +# variables: YAML_DIRS_TO_COPY, YAML_FILES_TO_COPY, and +# function make_yaml_files + +include: + - names.yaml + - platform.yaml + - case.yaml + - schema.yaml + - varnames.yaml + - runtime/ + - top.yaml + - resources.yaml + - workflow.yaml + - settings.yaml + - places.yaml + - base.yaml diff --git a/workflow/_main.yaml b/workflow/_main.yaml new file mode 100644 index 00000000000..1f66325c4d0 --- /dev/null +++ b/workflow/_main.yaml @@ -0,0 +1,9 @@ +# NOTE: This file is only used during setup_case.sh to tell that +# script what YAML files to read when initializing the experiment +# directory. This file is NOT copied to the experiment directory; +# there, _expdir_main.yaml is used in its place. +include: + - user.yaml + - schema/ + - defaults/ + - runtime/ diff --git a/workflow/cases/coupled_free_forecast.yaml b/workflow/cases/coupled_free_forecast.yaml new file mode 100644 index 00000000000..2740c01ff18 --- /dev/null +++ b/workflow/cases/coupled_free_forecast.yaml @@ -0,0 +1,48 @@ +case: + fv3_settings: + nst_anl: False + + places: + workflow_file: layout/free_forecast_gfs.yaml + MOM6IC: /scratch1/NCEPDEV/nems/Bin.Li/S2S/FROM_HPSS + + settings: + SDATE: 2016-04-01t00:00:00 + EDATE: 2016-04-01t00:00:00 + + cplflx: .true. + print_esmf: .true. + nems_temp: 'med_atm_ocn_ice' + nems_temp_cold: 'medcold_atm_ocn_ice' +# cplflx: .false. + + nsst: + NST_MODEL: 0 + + gfs_output_settings: + OCN_INTERVAL: 24 + FHOUT_GFS: 6 + FHMIN_GFS: 0 + FHMAX_GFS: 72 + FHMAX_HF_GFS: 0 + FHOUT_HF_GFS: 1 + + fv3_gfs_settings: + CASE: C384 + LEVS: 65 + DELTIM: 450 + layout_x: 6 + layout_y: 8 + fv3_threads: 2 + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 24 + WRTIOBUF: "32M" + OCNPETS: 120 + ICEPETS: 48 + hord_mt_nh_nonmono: 5 + hord_xx_nh_nonmono: 5 + cdmbgwd: "1.0,1.2" + + post: + downset: 2 + GOESF: no diff --git a/workflow/cases/dev/Run-From-HPSS-ICs.yaml b/workflow/cases/dev/Run-From-HPSS-ICs.yaml new file mode 100644 index 00000000000..9627d778b68 --- /dev/null +++ b/workflow/cases/dev/Run-From-HPSS-ICs.yaml @@ -0,0 +1,69 @@ +## This case file is intended to be modified by the user (you) to run +## any arbitrary case that pulls data off of the NOAA HPSS tape +## archiving system. Obviously, you need access to HPSS to use this +## script. If you cannot access HPSS, then you will have to use a staged data case. + +## Follow the instructions below to modify the configuration. + +case: + fv3_settings: + # Set the resolution here, in the "CASE" option. Fully tested + # resolutions are C192, C384, and C768. + CASE: C768 + + # LEVS: Number of vertical levels. Do not change. In this + # release, only 65 vertical levels have been tested. + LEVS: 65 + + # NOTE: You can configure physics, dynamics, and other model + # settings here. However, that is not supported. See + # schema/fv3.yaml for options. + + gfs_output_settings: + # Set the GFS output times here. The output times are: + # + # 0 to FHMAX_HF_GFS stepping by FHOUT_HF_GFS + # FHMAX_HF_GFS to FHMAX_GFS stepping by FHOUT_GFS + # + # The high- and low-frequency output times contain the same + # variables at the same resolution. To disable the high-frequency + # output time period and only output at one frequency, remove the + # FHMAX_HF_GFS and FHOUT_HF_GFS variables. + FHMAX_GFS: 168 + FHOUT_GFS: 6 + FHMAX_HF_GFS: 48 + FHOUT_HF_GFS: 3 + + settings: + # SDATE: The initial time in an ISO 8601 date format with both day and time: + SDATE: 2017-08-22t00:00:00 + + #################################################################### + ############## DO NOT CHANGE ANYTHING AFTER THIS LINE ############## + #################################################################### + + # EDATE: Initial time of the last cycle. Here we set it to the + # initial time of the first cycle so that we only run one cycle. + # You can run more than one cycle by setting this to a time after + # SDATE. The difference between EDATE and SDATE must be a multiple + # of six hours. The public release has only been tested with EDATE=SDATE + EDATE: !calc SDATE + + # run_gsi: Do not run data assimilation (its scripts and code are missing) + run_gsi: No + + # chgres_and_convert_ics: Convert GSM GFS analysis files to FV3. This is + # turned off in the staged data cases because they are using FV3 input + # files instead. + chgres_and_convert_ics: yes + + # gfs_cyc: this setting is used in a full data assimilation cycled + # workflow to only run the GFS every N cycles. Keep this at "4" to + # ensure it is run every cycle. + gfs_cyc: 4 + + places: + # workflow_file: which workflow we're running. This specifies the + # jobs to run and their dependencies. The only workflow included + # in this release is the public release workflow. + workflow_file: layout/public_release_v1.yaml diff --git a/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml new file mode 100644 index 00000000000..0541c3cf45a --- /dev/null +++ b/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml @@ -0,0 +1,140 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 10 + NMEM_EFCSGRP: 10 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + FHOUT_HF_GFS: 1 + + places: + workflow_file: layout/cycled_gfs.yaml + ROTDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/emc.fv3/com/gfs/exp/{doc.names.experiment}" + EXPDIR: !expand "{HOMEgfs}/ecf/exp/{doc.names.experiment}" + DATAROOT: /gpfs/dell2/ptmp/emc.fv3/tmpnwprd + SHORT_TERM_TEMP: /gpfs/dell2/ptmp/emc.fv3/tmpnwprd + LONG_TERM_TEMP: /gpfs/dell2/ptmp/emc.fv3/tmpnwprd + NOSCRUB: !expand "/gpfs/dell2/emc/modeling/noscrub/emc.fv3/com/emcstat/{doc.names.experiment}/arch" + HOMEDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/emc.fv3/com/emcstat/{doc.names.experiment}/archsave" + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 72 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + run_prep: !JobRequest + - mpi_ranks: 7 + max_ppn: 7 + OMP_NUM_THREADS: 4 + walltime: !timedelta '01:15:00' + + run_anal: !JobRequest + - mpi_ranks: 490 + max_ppn: 7 + OMP_NUM_THREADS: 4 + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '02:00:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '06:00:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + run_eobs: !JobRequest + - mpi_ranks: 140 + max_ppn: 14 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 2 + + run_eomg: !JobRequest + - mpi_ranks: 140 + max_ppn: 14 + walltime: !timedelta '01:00:00' + OMP_NUM_THREADS: 2 + + run_eupd: !JobRequest + - mpi_ranks: 360 + max_ppn: 4 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 4 + + run_ecen: !JobRequest + - mpi_ranks: 80 + max_ppn: 4 + walltime: !timedelta '01:00:00' + OMP_NUM_THREADS: 4 + + run_epos: !JobRequest + - mpi_ranks: 80 + max_ppn: 4 + walltime: !timedelta '02:00:00' + OMP_NUM_THREADS: 4 + + settings: + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-08-13t18:00:00 + EDATE: 2018-08-30t18:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 + run_gsi: Yes diff --git a/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml new file mode 100644 index 00000000000..d25582577d4 --- /dev/null +++ b/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml @@ -0,0 +1,149 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + layout_y: 6 + layout_x: 4 + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 24 + QUILTING: yes + + fv3_gfs_settings: + layout_x: 12 + layout_y: 16 + WRTTASK_PER_GROUP: 60 + WRITE_GROUP: 4 + hord_mt_nh_nonmono: 5 + hord_xx_nh_nonmono: 5 + + fv3_gdas_settings: + layout_x: 6 + layout_y: 8 + WRTTASK_PER_GROUP: 48 + WRITE_GROUP: 1 + hord_mt_nh_nonmono: 6 + hord_xx_nh_nonmono: 6 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 8 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + FHOUT_HF_GFS: 1 + FHOUT_GDAS: 1 + + places: + # DMPDIR: /gpfs/dell2/emc/modeling/noscrub/emc.fv3/com/globaldump + workflow_file: layout/cycled_gfs.yaml + COMROOT: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/" + ROTDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/gfs/{doc.names.experiment}" + EXPDIR: !expand "{HOMEgfs}/parm/config" + YAMLDIR: !expand "{HOMEgfs}/ecf/ecfutils/expdir/{doc.names.experiment}" + NOSCRUB: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/emcstat/{doc.names.experiment}/arch" + HOMEDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/emcstat/{doc.names.experiment}/archsave" + DATAROOT: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + SHORT_TERM_TEMP: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + LONG_TERM_TEMP: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 72 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + postsnd_walltime: !timedelta "02:00:00" + gdasfcst_walltime: !timedelta "01:00:00" + + run_anal: !JobRequest + - mpi_ranks: 480 + max_ppn: 2 + OMP_NUM_THREADS: max + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:45:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + settings: + + # Enable NCO-style suite definitions: + four_cycle_mode: yes + nco_mode: yes + #run_vrfy_jobs: no + #archive_to_hpss: no + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + BASE_SVN: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-08-09t00:00:00 + EDATE: 2018-08-10t18:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 + run_gsi: Yes + + schedvar: + cpu_project: PROJ + diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml new file mode 100644 index 00000000000..a3f89871f38 --- /dev/null +++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml @@ -0,0 +1,138 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 10 + NMEM_EFCSGRP: 10 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 48 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + run_prep: !JobRequest + - mpi_ranks: 7 + max_ppn: 7 + OMP_NUM_THREADS: 4 + walltime: !timedelta '01:15:00' + + run_anal: !JobRequest + - mpi_ranks: 490 + max_ppn: 7 + OMP_NUM_THREADS: 4 + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '02:00:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '06:00:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + run_eobs: !JobRequest + - mpi_ranks: 280 + max_ppn: 14 + walltime: !timedelta '00:45:00' + OMP_NUM_THREADS: 2 + + run_eomg: !JobRequest + - mpi_ranks: 280 + max_ppn: 14 + walltime: !timedelta '03:00:00' + OMP_NUM_THREADS: 2 + + run_eupd: !JobRequest + - mpi_ranks: 350 + max_ppn: 7 + walltime: !timedelta '03:00:00' + OMP_NUM_THREADS: 4 + + run_ecen: !JobRequest + - mpi_ranks: 140 + max_ppn: 7 + walltime: !timedelta '02:00:00' + OMP_NUM_THREADS: 4 + + run_epos: !JobRequest + - mpi_ranks: 81 + max_ppn: 3 + walltime: !timedelta '03:00:00' + OMP_NUM_THREADS: 8 + + settings: + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + BASE_SVN: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2016-04-18t00:00:00 + EDATE: 2016-04-18t12:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml new file mode 100644 index 00000000000..3ba368393e3 --- /dev/null +++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml @@ -0,0 +1,176 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + fv3_enkf_settings: + CASE: C384 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 5 + NMEM_EFCSGRP: 8 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_BUFRSND: No + DO_GEMPAK: No + DO_AWIPS: No + VRFYFITS: NO + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: NO + VRFYGENESIS: NO + RUNMOS: NO + RMOLDSTD: 144 + RMOLDEND: 72 + + archiving: + ATARDIR: /5year/NCEPDEV/emc-global/emc.glopara/JET/Q2FY19/fv3q2fy19retro5 + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + resources: + run_postsnd: !JobRequest + - batch_memory: "254M" + mpi_ranks: 12 + max_ppn: 1 + walltime: !timedelta '04:30' + exe: placeholder + + run_ecen: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 80 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: max + memory_per_rank: 8500 + + efcs_walltime: !timedelta '01:45:00' + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/prep/j*_prep*' + partition: !calc doc.platform.partitions.bigmem + max_tries: 2 + + - Search: '**/analysis/j*_analysis*|gfs/jgfs_analysis*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 360 + walltime: !timedelta '03:45:00' + OMP_NUM_THREADS: max + memory_per_rank: 10000 + + # - Search: 'gfs/jgfs_analysis*' + # partition: !calc doc.platform.partitions.tjet_or_ujet + # max_tries: 2 + + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: '**/forecast/jgdas_forecast_high' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: '**/post/jgfs_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '8*16' + walltime: !timedelta '00:15:00' + + - Search: '**/post/jgdas_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '12*12' + walltime: !timedelta '00:20:00' + + - Search: '**/enkf/post/jgdas_enkf_post_f*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 3 + + - Search: '**/enkf/innovate/j*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: '**/enkf/forecast/j*_enkf*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_inflate_recenter*' + partition: !calc doc.platform.partitions.tjet_or_ujet + #accounting: ! BAD: this blocks the job if any other "urgent" jobs are active + # queue: urgent + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_select*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_update*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - Search: 'gfs/post_processing/bufr_sounding/j*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + + - Search: '**/*_emc_vrfy' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - otherwise: [] + + settings: + rocoto_task_throttle: 100 + rocoto_cycle_throttle: 20 + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /mnt/lfs3/projects/hfv3gfs/glopara/git + BASE_SVN: /mnt/lfs3/projects/hfv3gfs/glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2015-11-25t00:00:00 + EDATE: 2016-05-31t18:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml new file mode 100644 index 00000000000..77839721086 --- /dev/null +++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml @@ -0,0 +1,190 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + fv3_enkf_settings: + CASE: C384 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 5 + NMEM_EFCSGRP: 8 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHOUT_GDAS: 1 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_BUFRSND: No + DO_GEMPAK: No + DO_AWIPS: No + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + RMOLDEND: 72 + + archiving: + ATARDIR: /5year/NCEPDEV/emc-global/emc.glopara/JET/Q2FY19/fv3q2fy19retro5_dell_restart + scrub_in_archive_enkf: No + scrub_in_archive: No + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + resources: + run_postsnd: !JobRequest + - batch_memory: "254M" + mpi_ranks: 12 + max_ppn: 1 + walltime: !timedelta '04:30' + exe: placeholder + + run_ecen: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 80 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: max + memory_per_rank: 8500 + + efcs_walltime: !timedelta '01:45:00' + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/prep/j*_prep*' + partition: !calc doc.platform.partitions.bigmem + max_tries: 2 + + - Search: '**/analysis/j*_analysis*' + partition: !calc doc.platform.partitions.vjet + max_tries: 2 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 360 + walltime: !timedelta '03:45:00' + OMP_NUM_THREADS: max + memory_per_rank: 10000 + +# - Search: '**/analysis/jgdas_analysis_high' +# partition: !calc doc.platform.partitions.sjet_or_vjet +# max_tries: 2 +# resources: !JobRequest +# - batch_memory: "3072M" +# exe: placeholder +# mpi_ranks: 360 +# walltime: !timedelta '03:45:00' +# OMP_NUM_THREADS: max +# memory_per_rank: 10000 + + - Search: 'gfs/jgfs_analysis*' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - Search: '**/forecast/jgdas_forecast_high' + partition: !calc doc.platform.partitions.vjet + max_tries: 2 + + - Search: '**/post/jgfs_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.ujet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '13*12' + walltime: !timedelta '00:15:00' + + - Search: '**/post/jgdas_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.ujet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '13*12' + walltime: !timedelta '00:25:00' + + - Search: '**/enkf/post/jgdas_enkf_post_f*' + partition: !calc doc.platform.partitions.tjet + max_tries: 3 + + - Search: '**/enkf/innovate/j*' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - Search: '**/enkf/forecast/j*_enkf*' + partition: !calc doc.platform.partitions.vjet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_inflate_recenter*' + partition: !calc doc.platform.partitions.sjet + #accounting: ! BAD: this blocks the job if any other "urgent" jobs are active + # queue: urgent + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_select*' + partition: !calc doc.platform.partitions.tjet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_update*' + partition: !calc doc.platform.partitions.tjet + max_tries: 2 + + - Search: 'gfs/post_processing/bufr_sounding/j*' + partition: !calc doc.platform.partitions.ujet + max_tries: 2 + + - Search: '**/*_emc_vrfy' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - otherwise: [] + + settings: + rocoto_task_throttle: 100 + rocoto_cycle_throttle: 3 + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /mnt/lfs3/projects/hfv3gfs/glopara/git + BASE_SVN: /mnt/lfs3/projects/hfv3gfs/glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2016-02-14t00:00:00 + EDATE: 2016-02-14t12:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml new file mode 100644 index 00000000000..32964f0caf3 --- /dev/null +++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml @@ -0,0 +1,190 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + fv3_enkf_settings: + CASE: C384 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 5 + NMEM_EFCSGRP: 8 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHOUT_GDAS: 1 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_BUFRSND: No + DO_GEMPAK: No + DO_AWIPS: No + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + RMOLDEND: 72 + + archiving: + ATARDIR: /5year/NCEPDEV/emc-global/emc.glopara/JET/Q2FY19/fv3q2fy19retro5_dell_restart + scrub_in_archive_enkf: No + scrub_in_archive: No + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + resources: + run_postsnd: !JobRequest + - batch_memory: "254M" + mpi_ranks: 12 + max_ppn: 1 + walltime: !timedelta '04:30' + exe: placeholder + + run_ecen: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 80 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: max + memory_per_rank: 8500 + + efcs_walltime: !timedelta '01:45:00' + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/prep/j*_prep*' + partition: !calc doc.platform.partitions.bigmem + max_tries: 2 + + - Search: '**/analysis/j*_analysis*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 360 + walltime: !timedelta '03:45:00' + OMP_NUM_THREADS: max + memory_per_rank: 10000 + +# - Search: '**/analysis/jgdas_analysis_high' +# partition: !calc doc.platform.partitions.sjet_or_vjet +# max_tries: 2 +# resources: !JobRequest +# - batch_memory: "3072M" +# exe: placeholder +# mpi_ranks: 360 +# walltime: !timedelta '03:45:00' +# OMP_NUM_THREADS: max +# memory_per_rank: 10000 + + - Search: 'gfs/jgfs_analysis*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/forecast/jgdas_forecast_high' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/post/jgfs_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '13*12' + walltime: !timedelta '00:15:00' + + - Search: '**/post/jgdas_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '13*12' + walltime: !timedelta '00:15:00' + + - Search: '**/enkf/post/jgdas_enkf_post_f*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 3 + + - Search: '**/enkf/innovate/j*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/enkf/forecast/j*_enkf*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_inflate_recenter*' + partition: !calc doc.platform.partitions.tjet_or_wherever + #accounting: ! BAD: this blocks the job if any other "urgent" jobs are active + # queue: urgent + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_select*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_update*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: 'gfs/post_processing/bufr_sounding/j*' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - Search: '**/*_emc_vrfy' + partition: !calc doc.platform.partitions.tjet_or_wherever + max_tries: 2 + + - otherwise: [] + + settings: + rocoto_task_throttle: 100 + rocoto_cycle_throttle: 3 + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /mnt/lfs3/projects/hfv3gfs/glopara/git + BASE_SVN: /mnt/lfs3/projects/hfv3gfs/glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2016-02-14t00:00:00 + EDATE: 2016-02-14t12:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml new file mode 100644 index 00000000000..59a6780d4ef --- /dev/null +++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml @@ -0,0 +1,99 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHOUT_GDAS: 1 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/free_forecast_gfs.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_BUFRSND: No + DO_GEMPAK: No + DO_AWIPS: No + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + RMOLDEND: 72 + + archiving: + ATARDIR: /5year/NCEPDEV/emc-global/emc.glopara/JET/Q2FY19/fv3q2fy19retro5_dell_restart + scrub_in_archive: No + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + resources: + run_postsnd: !JobRequest + - batch_memory: "254M" + mpi_ranks: 12 + max_ppn: 1 + walltime: !timedelta '04:30' + exe: placeholder + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - Search: '**/post/jgfs_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.ujet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '13*12' + walltime: !timedelta '00:15:00' + + - Search: 'gfs/post_processing/bufr_sounding/j*' + partition: !calc doc.platform.partitions.ujet + max_tries: 2 + + - Search: '**/*_emc_vrfy' + partition: !calc doc.platform.partitions.sjet + max_tries: 2 + + - otherwise: [] + + settings: + rocoto_task_throttle: 100 + rocoto_cycle_throttle: 3 + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /mnt/lfs3/projects/hfv3gfs/glopara/git + BASE_SVN: /mnt/lfs3/projects/hfv3gfs/glopara/git + + IC_CDUMP: gfs # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2016-02-14t00:00:00 + EDATE: 2016-02-14t12:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: No diff --git a/workflow/cases/dev/gaea-test.yaml b/workflow/cases/dev/gaea-test.yaml new file mode 100644 index 00000000000..f59cb6275f0 --- /dev/null +++ b/workflow/cases/dev/gaea-test.yaml @@ -0,0 +1,25 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 240 + FHOUT_GFS: 3 + + places: + workflow_file: layout/public_release_v1.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + settings: + chgres_and_convert_ics: No # disable jgfs_emc_fv3ic job + + # Set the date range to one cycle: + SDATE: 2015-11-26t00:00:00 + EDATE: 2015-11-26t00:00:00 + realtime: false + gfs_cyc: 4 # run gfs four cycles a day + run_gsi: No # disable data assimilation diff --git a/workflow/cases/dev/lowrsecf@C384.yaml b/workflow/cases/dev/lowrsecf@C384.yaml new file mode 100644 index 00000000000..4ee8f3697f4 --- /dev/null +++ b/workflow/cases/dev/lowrsecf@C384.yaml @@ -0,0 +1,52 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + fv3_enkf_settings: + CASE: C192 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 4 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: YES + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHOUT_ENKF: 1 + FHMAX_HF_GFS: 120 + + places: + workflow_file: layout/cycled_gfs.yaml + ICSDIR: /gpfs/hps3/emc/global/noscrub/Jian.Kuang/ICS +#### HOMEgfs: /gpfs/hps3/emc/global/noscrub/emc.glopara/parallel-ecf/builds/replace-setup-expt-20180529 + HOMEgfs: /gpfs/hps3/emc/global/noscrub/Jian.Kuang/fv3gfs/ + + settings: + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-05-31t00:00:00 + EDATE: 2018-05-31t12:00:00 + chgres_and_convert_ics: No + REALTIME: true + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: YES + + post: + NPOSTGRP: 42 + downset: 1 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYTRAK: YES + VRFYGENESIS: YES + + archiving: + RMOLDEND: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + arch_cyc: 00 diff --git a/workflow/cases/dev/merge2dell_baseline.yaml b/workflow/cases/dev/merge2dell_baseline.yaml new file mode 100644 index 00000000000..39641f897e9 --- /dev/null +++ b/workflow/cases/dev/merge2dell_baseline.yaml @@ -0,0 +1,147 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + layout_y: 6 + layout_x: 4 + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 24 + QUILTING: yes + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 8 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: Yes + DO_EMCSFC: NO + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + FHOUT_HF_GFS: 1 + + fv3_gfs_settings: + layout_x: 12 + layout_y: 16 + WRTTASK_PER_GROUP: 60 + WRITE_GROUP: 4 + hord_mt_nh_nonmono: 5 + hord_xx_nh_nonmono: 5 + + fv3_gdas_settings: + layout_x: 6 + layout_y: 8 + WRTTASK_PER_GROUP: 48 + WRITE_GROUP: 1 + hord_mt_nh_nonmono: 6 + hord_xx_nh_nonmono: 6 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_POST_PROCESSING: NO + DO_BUFRSND: NO + DO_GEMPAK: NO + DO_AWIPS: NO + DO_FAX: NO + DO_WAFS: NO + DO_BULLETINS: NO + VRFYFITS: NO + VSDB_STEP1: NO + VSDB_STEP2: NO + VRFYG2OBS: NO + VRFYPRCP: NO + VRFYRAD: NO + VRFYOZN: NO + VRFYMINMON: NO + VRFYTRAK: NO + VRFYGENESIS: NO + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 72 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + postsnd_walltime: !timedelta "02:00:00" + gdasfcst_walltime: !timedelta "01:00:00" + + run_prep: !JobRequest + - mpi_ranks: 7 + max_ppn: 7 + OMP_NUM_THREADS: 4 + walltime: !timedelta '01:15:00' + + run_anal: !JobRequest + - mpi_ranks: 480 + max_ppn: 2 + OMP_NUM_THREADS: max + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:45:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + settings: + + # Enable NCO-style suite definitions: + four_cycle_mode: no + nco_mode: no + + run_vrfy_jobs: NO + archive_to_hpss: NO + + DUMP_SUFFIX: "p" + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + rocoto_task_throttle: 15 + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-11-26t06:00:00 + EDATE: 2018-11-26t12:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 + run_gsi: Yes + + schedvar: + cpu_project: PROJ + diff --git a/workflow/cases/dev/merge2jet_baseline.yaml b/workflow/cases/dev/merge2jet_baseline.yaml new file mode 100644 index 00000000000..4a4ba71afa5 --- /dev/null +++ b/workflow/cases/dev/merge2jet_baseline.yaml @@ -0,0 +1,185 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + fv3_enkf_settings: + CASE: C192 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 5 + NMEM_EFCSGRP: 8 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 144 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 0 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_POST_PROCESSING: NO + DO_BUFRSND: NO + DO_GEMPAK: NO + DO_AWIPS: NO + DO_FAX: NO + DO_WAFS: NO + DO_BULLETINS: NO + VRFYFITS: NO + VSDB_STEP1: NO + VSDB_STEP2: NO + VRFYG2OBS: NO + VRFYPRCP: NO + VRFYRAD: NO + VRFYOZN: NO + VRFYMINMON: NO + VRFYTRAK: NO + VRFYGENESIS: NO + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: /5year/NCEPDEV/emc-global/emc.glopara/JET/Q2FY19/fv3q2fy19retro5 + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + resources: + run_postsnd: !JobRequest + - batch_memory: "254M" + mpi_ranks: 12 + max_ppn: 1 + walltime: !timedelta '04:30' + exe: placeholder + + run_ecen: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 80 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: max + memory_per_rank: 8500 + + efcs_walltime: !timedelta '01:45:00' + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/prep/j*_prep*' + partition: !calc doc.platform.partitions.bigmem + max_tries: 2 + + - Search: '**/analysis/j*_analysis*|gfs/jgfs_analysis*' + partition: !calc doc.platform.partitions.xjet + max_tries: 2 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 360 + walltime: !timedelta '03:45:00' + OMP_NUM_THREADS: max + memory_per_rank: 10000 + + # - Search: 'gfs/jgfs_analysis*' + # partition: !calc doc.platform.partitions.tjet_or_ujet + # max_tries: 2 + + - Search: '**/forecast/jgdas_forecast_high' + partition: !calc doc.platform.partitions.kjet + max_tries: 2 + resources: !JobRequest + - mpi_ranks: !calc '11*24' + walltime: !timedelta '00:20:00' + + - Search: '**/post/jgfs_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '8*16' + walltime: !timedelta '00:15:00' + + - Search: '**/post/jgdas_post*' + # Make sure you update "resources" to match "partition" + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 3 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !calc '12*12' + walltime: !timedelta '00:20:00' + + - Search: '**/enkf/post/jgdas_enkf_post_f*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 3 + + - Search: '**/enkf/innovate/j*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + + - Search: '**/enkf/forecast/j*_enkf*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_inflate_recenter*' + partition: !calc doc.platform.partitions.xjet + #accounting: ! BAD: this blocks the job if any other "urgent" jobs are active + # queue: urgent + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_select*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + + - Search: '**/enkf/jgdas_enkf_update*' + partition: !calc doc.platform.partitions.xjet + max_tries: 2 + + - Search: 'gfs/post_processing/bufr_sounding/j*' + partition: !calc doc.platform.partitions.sjet_or_vjet + max_tries: 2 + + - Search: '**/*_emc_vrfy' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + + - otherwise: [] + + data_assimilation: + DO_EMCSFC: no + + settings: + rocoto_task_throttle: 100 + rocoto_cycle_throttle: 20 + + four_cycle_mode: no + nco_mode: no + + DUMP_SUFFIX: "" + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /mnt/lfs3/projects/hfv3gfs/glopara/git + BASE_SVN: /mnt/lfs3/projects/hfv3gfs/glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-02t00:00:00 + chgres_and_convert_ics: No + realtime: false + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/nco.yaml b/workflow/cases/dev/nco.yaml new file mode 100644 index 00000000000..9d1d855acca --- /dev/null +++ b/workflow/cases/dev/nco.yaml @@ -0,0 +1,144 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + layout_y: 6 + layout_x: 4 + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 24 + QUILTING: yes + + fv3_gfs_settings: + layout_x: 12 + layout_y: 16 + WRTTASK_PER_GROUP: 60 + WRITE_GROUP: 4 + hord_mt_nh_nonmono: 5 + hord_xx_nh_nonmono: 5 + + fv3_gdas_settings: + layout_x: 6 + layout_y: 8 + WRTTASK_PER_GROUP: 48 + WRITE_GROUP: 1 + hord_mt_nh_nonmono: 6 + hord_xx_nh_nonmono: 6 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 8 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + FHOUT_HF_GFS: 1 + FHOUT_GDAS: 1 + + places: + DMPDIR: /gpfs/dell2/emc/modeling/noscrub/emc.fv3/com/globaldump + workflow_file: layout/nco.yaml + COMROOT: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/" + ROTDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/gfs/{doc.names.experiment}" +# EXPDIR: !expand "{HOMEgfs}/parm/config" +# YAMLDIR: !expand "{HOMEgfs}/ecf/ecfutils/expdir/{doc.names.experiment}" +# NOSCRUB: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/emcstat/{doc.names.experiment}/arch" +# HOMEDIR: !expand "/gpfs/dell2/emc/modeling/noscrub/Jian.Kuang/com/emcstat/{doc.names.experiment}/archsave" + DATAROOT: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + SHORT_TERM_TEMP: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + LONG_TERM_TEMP: /gpfs/dell2/ptmp/Jian.Kuang/tmpnwprd + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 72 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + postsnd_walltime: !timedelta "02:00:00" + gdasfcst_walltime: !timedelta "01:00:00" + + run_anal: !JobRequest + - mpi_ranks: 480 + max_ppn: 2 + OMP_NUM_THREADS: max + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:45:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + settings: + + # Enable NCO-style suite definitions: + four_cycle_mode: yes + nco_mode: no + + run_vrfy_jobs: yes + archive_to_hpss: yes + + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-09-19t18:00:00 + EDATE: 2018-09-22t06:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 + run_gsi: Yes + + schedvar: + cpu_project: PROJ + diff --git a/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml b/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml new file mode 100644 index 00000000000..e3eba3ceeb0 --- /dev/null +++ b/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml @@ -0,0 +1,56 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + fv3_enkf_settings: + CASE: C384 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 4 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: YES + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHOUT_ENKF: 1 + FHMAX_HF_GFS: 120 + + places: + workflow_file: layout/cycled_gfs.yaml + ICSDIR: !FirstTrue + - when: !calc doc.platform.name == "THEIA" + do: "/scratch4/NCEPDEV/global/noscrub/glopara/ICS/FV3GFS" + - when: !calc doc.platform.name == "WCOSS_C" + do: "/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS" + - when: !calc doc.platform.name == "GAEA" + do: "/lustre/f1/pdata/ncep_shared/fv3/IC" + - otherwise: !error "No ICs for platform {doc.platform.name}" + + settings: + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-05-03t00:00:00 + EDATE: 2018-05-03t18:00:00 + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes + chgres_and_convert_ics: No + + post: + NPOSTGRP: 42 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYTRAK: YES + VRFYGENESIS: YES + + archiving: + RMOLDEND: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + arch_cyc: 00 diff --git a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml new file mode 100644 index 00000000000..0a170b5df7d --- /dev/null +++ b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml @@ -0,0 +1,123 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + fcst_max_ppn: 12 + + fv3_enkf_settings: + CASE: C384 + fcst_max_ppn: 12 + layout_y: 6 + layout_x: 4 + + fv3_gfs_settings: + layout_x: 12 + layout_y: 16 + WRTTASK_PER_GROUP: 60 + WRITE_GROUP: 4 + fv3_settings.hord_mt_nh_nonmono: 5 + fv3_settings.hord_xx_nh_nonmono: 5 + + fv3_gdas_settings: + layout_x: 6 + layout_y: 8 + WRTTASK_PER_GROUP: 48 + WRITE_GROUP: 1 + fv3_settings.hord_mt_nh_nonmono: 6 + fv3_settings.hord_xx_nh_nonmono: 6 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 8 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + FHOUT_HF_GFS: 1 + + places: + workflow_file: layout/cycled_gfs.yaml + + post: + NPOSTGRP: 10 + downset: 2 + + downstream: + DO_BUFRSND: YES + DO_GEMPAK: YES + DO_AWIPS: YES + VRFYFITS: YES + VSDB_STEP1: YES + VSDB_STEP2: NO + VRFYG2OBS: YES + VRFYPRCP: YES + VRFYRAD: YES + VRFYOZN: YES + VRFYMINMON: YES + VRFYTRAK: YES + VRFYGENESIS: YES + RUNMOS: NO + RMOLDSTD: 144 + + archiving: + ATARDIR: "/NCEPDEV/$HPSS_PROJECT/5year/$USER/$machine/Q2FY19/$PSLOT" + arch_cyc: 00 + scrub_in_archive_end: 72 + arch_warmicfreq: 4 + arch_fcsticfreq: 1 + + resources: + postsnd_walltime: !timedelta "02:00:00" + gdasfcst_walltime: !timedelta "01:00:00" + + run_anal: !JobRequest + - mpi_ranks: 480 + max_ppn: 2 + walltime: !timedelta '02:30:00' + + run_gdaspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:45:00' + OMP_NUM_THREADS: 1 + + run_gfspost: !JobRequest + - mpi_ranks: 84 + max_ppn: 14 + walltime: !timedelta '00:30:00' + OMP_NUM_THREADS: 1 + + run_gdasvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '03:00:00' + + run_gfsvrfy: !JobRequest + - mpi_ranks: 1 + max_ppn: 1 + OMP_NUM_THREADS: 1 + walltime: !timedelta '10:00:00' + + settings: + # max_job_tries: tell rocoto to allow 2 submissions for all jobs, + # unless overridden by that job's max_tries variable. + max_job_tries: 2 + + # This belongs in "places" and putting it here makes it unused. + # The default values are in platforms/jet.yaml. You should delete + # these lines: + BASE_GIT: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + BASE_SVN: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git + + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-09-05t06:00:00 + EDATE: 2018-09-09t18:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes diff --git a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml new file mode 100644 index 00000000000..b10dceb0efd --- /dev/null +++ b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml @@ -0,0 +1,47 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + fv3_enkf_settings: + CASE: C384 + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 4 + NMEM_EFCSGRP: 4 + NMEM_EARCGRP: 10 + l4densvar: Yes + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 120 + + places: + workflow_file: layout/cycled_gfs.yaml + ICSDIR: /gpfs/hps3/emc/global/noscrub/emc.glopara/ICS + + post: + downset: 2 + NPOSTGRP: 42 + + downstream: + DO_BUFRSND: Yes + DO_GEMPAK: Yes + DO_AWIPS: Yes + + archiving: + scrub_in_archive_end: 72 + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + + settings: + IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS + SDATE: 2018-06-04t00:00:00 + EDATE: 2018-06-04t12:00:00 + chgres_and_convert_ics: No + realtime: true + gfs_cyc: 4 # run gfs at 0z, 12z + run_gsi: Yes + diff --git a/workflow/cases/free_forecast_case.yaml b/workflow/cases/free_forecast_case.yaml new file mode 100644 index 00000000000..8dc1c7b2cd2 --- /dev/null +++ b/workflow/cases/free_forecast_case.yaml @@ -0,0 +1,16 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + places: + workflow_file: layout/free_forecast_gfs.yaml + + settings: + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-01t06:00:00 + + DUMP_SUFFIX: "p" + run_gsi: No + chgres_and_convert_ics: Yes + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/jet_cycled.yaml b/workflow/cases/jet_cycled.yaml new file mode 100644 index 00000000000..c00e9aefd4a --- /dev/null +++ b/workflow/cases/jet_cycled.yaml @@ -0,0 +1,19 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + fv3_enkf_settings: + CASE: C192 + + places: + workflow_file: layout/cycled_gfs.yaml + + settings: + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-01t06:00:00 + + DUMP_SUFFIX: "" + run_gsi: Yes + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/jet_free_forecast.yaml b/workflow/cases/jet_free_forecast.yaml new file mode 100644 index 00000000000..80fb80fc8e9 --- /dev/null +++ b/workflow/cases/jet_free_forecast.yaml @@ -0,0 +1,36 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + places: + workflow_file: layout/free_forecast_gfs.yaml + + gfs_output_settings: + FHMAX_GFS: 24 + FHOUT_GFS: 3 + FHMAX_HF_GFS: 0 + + settings: + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-01t00:00:00 + + DUMP_SUFFIX: "p" + run_gsi: No + chgres_and_convert_ics: Yes + gfs_cyc: 4 # run gfs every cycle + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/forecast/jgfs_forecast_high' + partition: !calc doc.platform.partitions.kjet + max_tries: 2 + resources: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: 192 + walltime: !timedelta '00:40:00' + OMP_NUM_THREADS: 1 + + - otherwise: [] diff --git a/workflow/cases/manual_mode.yaml b/workflow/cases/manual_mode.yaml new file mode 100644 index 00000000000..eb216fefd2a --- /dev/null +++ b/workflow/cases/manual_mode.yaml @@ -0,0 +1,109 @@ +# This file is designed to be used in manual mode, where only +# a subset of variables are set by CROW while the majority +# of them are provided by developers through config files. +# +# You are expected to: 1, edit layout/manual_temp.yaml file +# following your desired workflow layout; 2, make sure everything +# in this file is OK; 3, provide your own config files. + +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + fv3_gfs_settings: + CASE: C384 + LEVS: 65 + QUILTING: True + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 2 + DONST: YES + nst_anl: yes + layout_x: 2 + layout_y: 3 + fv3_threads: 4 + imp_physics: 11 + + fv3_gdas_settings: + CASE: C384 + LEVS: 65 + QUILTING: True + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 2 + layout_x: 2 + layout_y: 3 + fv3_threads: 4 + + fv3_enkf_settings: + CASE: C192 + LEVS: 65 + QUILTING: True + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 2 + FHCYC_GDAS: 1 + FHCYC_GFS: 24 + layout_x: 2 + layout_y: 3 + fv3_threads: 4 + + gfs_output_settings: + FHMIN_GFS: 0 + FHMAX_GFS: 384 + FHOUT_GFS: 3 + FHMIN_GDAS: 0 + FHMAX_GDAS: 9 + FHOUT_GDAS: 3 + FHMAX_HF_GFS: 0 + FHOUT_HF_GFS: 3 + FHMIN_ENKF: 3 + FHMAX_ENKF: 9 + FHOUT_ENKF: 3 + OUTPUT_GRID: gaussian_grid + OUTPUT_FILE_TYPE: nemsio + + data_assimilation: + NMEM_ENKF: 80 + NMEM_EOMGGRP: 5 + NMEM_EFCSGRP: 8 + NMEM_EARCGRP: 10 + l4densvar: Yes + assim_freq: 6 + lwrite4danl: NO + DOHYBVAR: YES + SMOOTH_ENKF: YES + INCREMENTS_TO_ZERO: [ delz_inc, clwmr_inc, icmr_inc ] + + archiving: + RMOLDEND: 72 + ATARDIR: !expand "/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/{doc.names.experiment}" + arch_warmicfreq: 2 + arch_fcsticfreq: 1 + arch_cyc: 00 + archive_to_hpss: YES + scrub_in_archive: YES + + places: + workflow_file: layout/manual_temp.yaml + + settings: + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-01t06:00:00 + + DUMP_SUFFIX: "p" + run_gsi: Yes + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle + + downstream: &downstream + DO_POST_PROCESSING: NO + DO_BUFRSND: NO + DO_GEMPAK: NO + DO_AWIPS: NO + DO_FAX: NO + DO_WAFS: NO + DO_BULLETINS: NO + VRFYFITS: NO + VSDB_STEP1: NO + VSDB_STEP2: NO + VRFYG2OBS: NO + VRFYPRCP: NO diff --git a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml new file mode 100644 index 00000000000..ba69708cdf5 --- /dev/null +++ b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 12 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2016-02-10t00:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml new file mode 100644 index 00000000000..d8071f420ed --- /dev/null +++ b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 384 + FHOUT_GFS: 12 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2016-02-10t00:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml new file mode 100644 index 00000000000..7a9b56e8d8b --- /dev/null +++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml @@ -0,0 +1,22 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 168 + FHOUT_GFS: 6 + FHMAX_HF_GFS: 48 + FHOUT_HF_GFS: 3 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2017-08-17t12:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml new file mode 100644 index 00000000000..6fbebe28f4d --- /dev/null +++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml @@ -0,0 +1,47 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 168 + FHOUT_GFS: 6 + FHMAX_HF_GFS: 48 + FHOUT_HF_GFS: 3 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2017-08-17t12:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.tjet_or_ujet + - Search: '**/post/j*_post*' + partition: !calc doc.platform.partitions.xjet + max_tries: 5 + - otherwise: null + + resources: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + run_gempak: !JobRequest + - batch_memory: "254M" + mpi_ranks: 17 + max_ppn: 4 + walltime: !timedelta "02:00:00" + OMP_NUM_THREADS: 12 + exe: placeholder + - otherwise: {} + + + diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml new file mode 100644 index 00000000000..2787710cefa --- /dev/null +++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml @@ -0,0 +1,22 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 168 + FHOUT_GFS: 6 + FHMAX_HF_GFS: 48 + FHOUT_HF_GFS: 3 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2017-08-17t12:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml new file mode 100644 index 00000000000..3146d4a0cd9 --- /dev/null +++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml @@ -0,0 +1,22 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 168 + FHOUT_GFS: 6 + FHMAX_HF_GFS: 48 + FHOUT_HF_GFS: 3 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2017-08-17t12:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml new file mode 100644 index 00000000000..e0221886dde --- /dev/null +++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 240 + FHOUT_GFS: 6 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2018-02-28t00:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml new file mode 100644 index 00000000000..224b24c2a87 --- /dev/null +++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 240 + FHOUT_GFS: 6 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2018-02-28t00:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml new file mode 100644 index 00000000000..1a7e6da8e33 --- /dev/null +++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C768 + LEVS: 65 + + gfs_output_settings: + FHMAX_GFS: 240 + FHOUT_GFS: 6 + + places: + workflow_file: layout/public_release_v1.yaml + ICSDIR: !calc doc.platform.public_release_ics + + settings: + SDATE: 2018-02-28t00:00:00 + EDATE: !calc SDATE + IC_CDUMP: gfs + run_gsi: No + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/tutorial_cycled.yaml b/workflow/cases/tutorial_cycled.yaml new file mode 100644 index 00000000000..f426f45a832 --- /dev/null +++ b/workflow/cases/tutorial_cycled.yaml @@ -0,0 +1,20 @@ +case: + fv3_settings: + CASE: C384 + LEVS: 65 + + fv3_enkf_settings: + CASE: C192 + + places: + workflow_file: layout/cycled_gfs.yaml + + settings: + SDATE: 2019-08-20t00:00:00 + EDATE: 2019-08-21t06:00:00 + + print_esmf: .true. + KEEPDATA: Yes + run_gsi: Yes + chgres_and_convert_ics: No + gfs_cyc: 4 # run gfs every cycle diff --git a/workflow/cases/tutorial_free_forecast.yaml b/workflow/cases/tutorial_free_forecast.yaml new file mode 100644 index 00000000000..e187acb5061 --- /dev/null +++ b/workflow/cases/tutorial_free_forecast.yaml @@ -0,0 +1,24 @@ +case: + fv3_settings: + CASE: C192 + LEVS: 65 + + places: + workflow_file: layout/free_forecast_gfs.yaml + + settings: + SDATE: 2019-05-01t00:00:00 + EDATE: 2019-05-01t06:00:00 + + DUMP_SUFFIX: "p" + run_gsi: No + chgres_and_convert_ics: Yes + gfs_cyc: 4 # run gfs every cycle + + suite_overrides: !FirstTrue + - when: !calc (doc.platform.name=="JET") + do: + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.tjet_or_ujet + max_tries: 2 + - otherwise: [] diff --git a/workflow/config/aliases.hold b/workflow/config/aliases.hold new file mode 100644 index 00000000000..7b33840eb7c --- /dev/null +++ b/workflow/config/aliases.hold @@ -0,0 +1,206 @@ +# Jian Kuang 2019/09/04: renamed it to *_hold, prepare to remove + +# This file is used to generate the thread and processor information +# for the various config files. Such information must be in the +# config files even if it is not used, and must match any actual +# resources in the suite definition. Hence, if a free forecast is +# requested, the GDAS and ENKF resource requirements must be +# available. If the GFS forecast is in tJet and GDAS forecast is in +# xJet, then they must be appropriate for 12-core and 24-core nodes, +# respectively. + +aliases: + alias_template: &alias_template + resources: !FirstTrue + - when: !calc flag + do: !calc task.resources + - otherwise: !calc alt_resources + partition: !FirstTrue + - when: !calc flag + do: !calc task.partition + - otherwise: !calc alt_partition + nth: !calc partition.nodes.omp_threads_for(resources[0] if len(resources)<2 else resources[1]) + npe: !calc resources.total_ranks() + npe_node: !calc partition.nodes.max_ranks_per_node(resources[0] if len(resources)<2 else resources[1]) + + alias_exclusive: &alias_exclusive + <<: *alias_template + alt_partition: !calc doc.accounting.exclusive_partition + + alias_remap: &alias_remap + <<: *alias_template + alt_partition: !calc doc.accounting.exclusive_partition + resources: !FirstTrue + - when: !calc flag + do: !calc task.resources_remap + - otherwise: !calc alt_resources + + prep: + <<: *alias_exclusive + flag: !calc ( 'prep' in doc.suite.gfs ) + task: !calc doc.suite.gfs.prep.jgfs_prep + alt_resources: !calc doc.exclusive_resources.run_prep + + epos: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'enkf' in doc.suite.gdas ) + task: !calc doc.suite.gdas.enkf.post.jgdas_enkf_post_f006 + alt_resources: !calc doc.exclusive_resources.run_epos + + anal: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'analysis' in doc.suite.gdas ) + task: !calc doc.suite.gdas.analysis.jgdas_analysis_high + alt_resources: !calc doc.exclusive_resources.run_anal + + gfsvrfy: + <<: *alias_exclusive + flag: !calc ( 'gfs' in doc.suite and 'jgfs_emc_vrfy' in doc.suite.gfs ) + task: !calc doc.suite.gfs.jgfs_emc_vrfy + alt_resources: !calc doc.exclusive_resources.run_gfsvrfy + + gdasvrfy: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'jgdas_emc_vrfy' in doc.suite.gdas ) + task: !calc doc.suite.gdas.jgdas_emc_vrfy + alt_resources: !calc doc.exclusive_resources.run_gdasvrfy + + eobs: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite ) + task: !calc doc.suite.gdas.enkf.jgdas_enkf_select_obs + alt_resources: !calc doc.exclusive_resources.run_eobs + + gdasfcst: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite ) + task: !calc doc.suite.gdas.forecast.jgdas_forecast_high + alt_resources: !calc doc.exclusive_resources.run_gfsfcst + + gfsfcst: + <<: *alias_exclusive + flag: !calc ( 'gfs' in doc.suite ) + task: !calc doc.suite.gfs.forecast.jgfs_forecast_high + alt_resources: !calc doc.exclusive_resources.run_efcs + + gdasremap: + <<: *alias_remap + flag: !calc ( 'gdas' in doc.suite ) + task: !calc doc.suite.gdas.forecast.jgdas_forecast_high + alt_resources: !calc doc.default_resources.no_gdasfcst_remap + + gfsremap: + <<: *alias_remap + flag: !calc ( 'gfs' in doc.suite ) + task: !calc doc.suite.gfs.forecast.jgfs_forecast_high + alt_resources: !error doc.exclusive_resources.run_gfsremap + + gdaspost: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite ) + task: !calc doc.suite.gdas.post.jgdas_post_f000 + alt_resources: !calc doc.exclusive_resources.run_gdaspost + + gfspost: + <<: *alias_exclusive + flag: !calc ( 'gfs' in doc.suite ) + task: !calc doc.suite.gfs.post.jgfs_post_anl + alt_resources: !calc doc.exclusive_resources.run_gfspost + + efcs: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'enkf' in doc.suite.gdas ) + task: !calc doc.suite.gdas.enkf.forecast.jgdas_enkf_fcst_grp1 + alt_resources: !calc doc.exclusive_resources.run_efcs + + eupd: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'enkf' in doc.suite.gdas ) + task: !calc doc.suite.gdas.enkf.jgdas_enkf_update + alt_resources: !calc doc.exclusive_resources.run_eupd + + eomg: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'enkf' in doc.suite.gdas ) + task: !calc doc.suite.gdas.enkf.innovate.jgdas_enkf_innovate_obs_grp + alt_resources: !calc doc.exclusive_resources.run_eomg + + ecen: + <<: *alias_exclusive + flag: !calc ( 'gdas' in doc.suite and 'enkf' in doc.suite.gdas ) + task: !calc doc.suite.gdas.enkf.jgdas_enkf_inflate_recenter + alt_resources: !calc doc.exclusive_resources.run_ecen + + dwn: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_dwn + + fv3ic: + <<: *alias_exclusive + flag: !calc ( 'ics' in doc.suite and 'jgfs_emc_fv3ic' in doc.suite.ics ) + task: !calc doc.suite.ics.jgfs_emc_fv3ic + alt_resources: !calc doc.exclusive_resources.run_fv3ic + + getic: + <<: *alias_exclusive + flag: !calc ( 'ics' in doc.suite and 'jgfs_emc_getics' in doc.suite.ics ) + task: !calc doc.suite.ics.jgfs_emc_getics + alt_resources: !calc doc.service_resources.run_getic + + earc: + <<: *alias_exclusive + flag: !calc ( 'archive' in doc.suite and 'jgfs_enkf_archive' in doc.suite.archive ) + task: !calc doc.suite.archive.jgdas_enkf_archive.grp + alt_resources: !calc doc.service_resources.run_earc + + arch: + <<: *alias_exclusive + flag: !calc ( 'archive' in doc.suite ) + task: !calc doc.suite.archive.jgfs_archive + alt_resources: !calc doc.service_resources.run_arch + + ###################################################################### + + # PLACEHOLDERS - replace when jobs exist + + ###################################################################### + + prepbufr: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_prepbufr + + awips_g2: + <<: *alias_exclusive + flag: !calc ('gfs' in doc.suite and 'awips_g2' in doc.suite.gfs) + task: doc.suite.awips_g2.jgfs_awips_g2_f000 + alt_resources: !calc doc.exclusive_resources.run_awips_g2 + + awips_20km_1p0: + <<: *alias_exclusive + flag: !calc ('gfs' in doc.suite and 'awips_20km_1p0' in doc.suite.gfs) + task: doc.suite.awips_20km_1p0.jgfs_awips_f000 + alt_resources: !calc doc.exclusive_resources.run_awips_20km_1p0 + + postsnd: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_postsnd + + postsndcfp: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_postsndcfp + + gfs_gempak: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_gfs_gempak + + gdas_gempak: + <<: *alias_exclusive + flag: false + alt_resources: !calc doc.exclusive_resources.run_gdas_gempak + + diff --git a/workflow/config/anal.yaml b/workflow/config/anal.yaml new file mode 100644 index 00000000000..c8de7b6866e --- /dev/null +++ b/workflow/config/anal.yaml @@ -0,0 +1,113 @@ +# This file is used to generate config.anal, which would be used to +# run the GFS analysis. The scripts and code for the analysis are not +# provided in this release, so this config file is ignored. + +config_anal: + filename: config.anal + content: !expand | + #!/bin/ksh -x + + ########## config.anal ########## + # Analysis specific + + echo "BEGIN: config.anal" + + if [ $DONST = "YES" ]; then + . $EXPDIR/config.nsst + fi + + export npe_anal={doc.partition_common.resources.run_anal[0].mpi_ranks} + export npe_node_anal={doc.partition_common.resources.run_anal[0].max_ppn} + export nth_anal={doc.partition_common.resources.run_anal[0].OMP_NUM_THREADS} + export nth_cycle=24 + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export nth_anal=14 + export nth_cycle=28 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_anal="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_anal="3072M" + fi + + if [[ "$CDUMP" = "gfs" ]] ; then + export USE_RADSTAT="NO" # This can be only used when bias correction is not-zero. + export GENDIAG="NO" + export SETUP='diag_rad=.false.,diag_pcp=.false.,diag_conv=.false.,diag_ozone=.false.,write_diag(3)=.false.,' + export DIAG_TARBALL="NO" + fi + + export ANALYSISSH="$HOMEgsi/scripts/exglobal_analysis_fv3gfs.sh.ecf" + export npe_gsi=$npe_anal + export LEVS_ENKF=$((LEVS-1)) + + export GSNDBF=/dev/null + export AMSREBF=/dev/null + export SSMITBF=/dev/null + export AMSR2BF=/dev/null + + # Set directory and structure for COMIN_OBS + export COMIN_OBS="$DMPDIR/$CDUMP$DUMP_SUFFIX.$PDY/$cyc" + + # export REALTIME="{tools.YES_NO(doc.settings.realtime)}" # Run GFS AWIPS + + # Set CONVINFO and SATINFO for retrospective parallels + if [ $REALTIME = "NO" ]; then + + # Set CONVINFO + if [[ "$CDATE" -ge "2018022818" ]]; then + export CONVINFO=$FIXgsi/fv3_historical/global_convinfo.txt.2018022818 + elif [[ "$CDATE" -ge "2018010512" ]]; then + export CONVINFO=$FIXgsi/fv3_historical/global_convinfo.txt.2018010512 + elif [[ "$CDATE" -ge "2017071912" ]]; then + export CONVINFO=$FIXgsi/fv3_historical/global_convinfo.txt.2017071912 + elif [[ "$CDATE" -ge "2016031512" ]]; then + export CONVINFO=$FIXgsi/fv3_historical/global_convinfo.txt.2016031512 + elif [[ "$CDATE" -ge "2014041400" ]]; then + export CONVINFO=$FIXgsi/fv3_historical/global_convinfo.txt.2014041400 + else + echo "WARNING: No CONVINFO for $CDATE" + fi + + # Set OZINFO + if [[ "$CDATE" -ge "2018110700" ]]; then + export OZINFO=$FIXgsi/fv3_historical/global_ozinfo.txt.2018110700 + elif [[ "$CDATE" -ge "2015110500" ]]; then + export OZINFO=$FIXgsi/fv3_historical/global_ozinfo.txt.2015110500 + else + echo "WARNING: No OZINFO for $CDATE" + fi + + # Set SATINFO + if [[ "$CDATE" -ge "2018053012" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2018053012 + elif [[ "$CDATE" -ge "2018021212" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2018021212 + elif [[ "$CDATE" -ge "2017103118" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2017103118 + elif [[ "$CDATE" -ge "2017031612" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2017031612 + elif [[ "$CDATE" -ge "2017030812" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2017030812 + elif [[ "$CDATE" -ge "2016110812" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2016110812 + elif [[ "$CDATE" -ge "2016090912" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2016090912 + elif [[ "$CDATE" -ge "2016020312" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2016020312 + elif [[ "$CDATE" -ge "2016011912" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2016011912 + elif [[ "$CDATE" -ge "2015111012" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2015111012 + elif [[ "$CDATE" -ge "2015100118" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2015100118 + elif [[ "$CDATE" -ge "2015070218" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2015070218 + elif [[ "$CDATE" -ge "2015011412" ]]; then + export SATINFO=$FIXgsi/fv3_historical/global_satinfo.txt.2015011412 + else + echo "WARNING: No SATINFO for $CDATE" + fi + fi + + echo "END: config.anal" diff --git a/workflow/config/arch.yaml b/workflow/config/arch.yaml new file mode 100644 index 00000000000..c4eb91352e1 --- /dev/null +++ b/workflow/config/arch.yaml @@ -0,0 +1,32 @@ +# This file is used to generate config.arch, which would be used to +# control archiving of model output. The scripts and code for +# archiving are not provided in this release, so this config file is +# ignored. + +config_arch: + filename: config.arch + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.arch ########## + # Archive specific + + echo "BEGIN: config.arch" + + #--online archive of nemsio files for fit2obs verification + export FITSARC="{tools.YES_NO(doc.archiving.copy_fit2obs_files)}" + export FHMAX_FITS=132 + if [ $FHMAX_FITS -gt $FHMAX_GFS ]; then export FHMAX_FITS=$FHMAX_GFS; fi + + export npe_arch=1 + export npe_node_arch=1 + + #--starting and ending hours of previous cycles to be removed from rotating directory + export RMOLDSTD={doc.archiving.scrub_in_archive_start} + export RMOLDEND={doc.archiving.scrub_in_archive_end} + + echo "END: config.arch" diff --git a/workflow/config/awips.yaml b/workflow/config/awips.yaml new file mode 100644 index 00000000000..bc9aae6d130 --- /dev/null +++ b/workflow/config/awips.yaml @@ -0,0 +1,42 @@ +# This file is used to generate config.awips, which has settings for the GFS awips step + +config_awips: + filename: config.awips + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.awips ########## + # GFS awips step specific + + echo "BEGIN: config.awips" + + export npe_awips="{doc.partition_common.resources.run_awips[0].mpi_ranks}" + export npe_node_awips="{doc.partition_common.resources.run_awips[0].max_ppn}" + export nth_awips=2 + if [[ "$machine" == "WCOSS_DELL_P3" ]]; then + export npe_awips=2 + export npe_node_awips=2 + export nth_awips=1 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_awips="2048M" + fi + + export SENDCOM="YES" + + export AWIPS20SH=$HOMEgfs/jobs/JGFS_AWIPS_20KM_1P0DEG + export AWIPSG2SH=$HOMEgfs/jobs/JGFS_AWIPS_G2 + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUTwmo="$COMOUT/wmo" + export PCOM="$COMOUT/wmo" + + # No. of concurrent awips jobs + export NAWIPSGRP={doc.settings.NAWIPSGRP} + + echo "END: config.awips" diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml new file mode 100644 index 00000000000..170a0a84150 --- /dev/null +++ b/workflow/config/base.yaml @@ -0,0 +1,243 @@ +# This file is used to generate config.base, which the shell scripts +# process before any other files. The config.base sets global +# variables and default values that are needed throughout the +# scripting system. + +config_base: + filename: config.base + cpl: !FirstTrue + - when: !calc doc.settings.cplflx==".true." + do: .true. + - when: !calc doc.settings.cplice==".true." + do: .true. + - otherwise: .false. + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.base ########## + # Common to all steps + + echo "BEGIN: config.base" + + # Machine environment + export machine="{doc.platform.name}" + + export cpl="{cpl}" + export cplflx="{doc.settings.cplflx}" + export cplice="{doc.settings.cplice}" + export print_esmf="{doc.settings.print_esmf}" + + export nems_temp="{doc.settings.nems_temp}" + export nems_temp_cold="{doc.settings.nems_temp_cold}" + + + # EMC parallel or NCO production + export RUN_ENVIR="emc" + + # Account, queue, etc. + export ACCOUNT="{doc.accounting.cpu_project}" # This comes from user.yaml + export QUEUE="{doc.accounting.exclusive_partition.exclusive_queue}" + export QUEUE_ARCH="{doc.accounting.service_partition.service_queue}" + + # Project to use in mass store: + HPSS_PROJECT={doc.accounting.hpss_project} + + # Directories relative to installation areas: + export HOMEgfs="{doc.places.HOMEgfs}" + export PARMgfs=$HOMEgfs/parm + export FIXgfs=$HOMEgfs/fix + export USHgfs=$HOMEgfs/ush + export UTILgfs=$HOMEgfs/util + export EXECgfs=$HOMEgfs/exec + export SCRgfs=$HOMEgfs/scripts + + export FIXcice=$HOMEgfs/fix/fix_cice5 + export FIXmom=$HOMEgfs/fix/fix_mom6 + export FIXgrid=$HOMEgfs/fix/fix_fv3grid + export OCNFIXDIR=$HOMEgfs/fix/fix_ocnice + + # GLOBAL static environment parameters + export DMPDIR="{doc.places.DMPDIR}" + export RTMFIX="{doc.places.RTMFIX}" + + export HOMEDIR="{doc.places.HOMEgfs}" + export STMP="{doc.places.DATAROOT}" + export PTMP="{doc.places.COMROOT}" + export NOSCRUB="{doc.places.NOSCRUB}" + + # Base directories for various builds + export BASE_GIT="{doc.platform.BASE_GIT}" + export BASE_SVN="{doc.platform.BASE_SVN}" + + # Utilities needed in the scripts (mostly post) + {doc.platform.config_base_extras} + + # Toggle to turn on/off GFS downstream processing. + export DO_BUFRSND="{tools.YES_NO(doc.downstream.DO_BUFRSND)}" # Run GFS_POSTSND + export DO_GEMPAK="{tools.YES_NO(doc.downstream.DO_GEMPAK)}" # Run GFS GEMPAK + export DO_AWIPS="{tools.YES_NO(doc.downstream.DO_AWIPS)}" # Run GFS AWIPS + + export REALTIME="{tools.YES_NO(doc.settings.realtime)}" + + #################################################### + # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE + # IF YOU HAVE TO MAKE MACHINE SPECIFIC CHANGES BELOW + # FEEL FREE TO MOVE THEM ABOVE THIS LINE TO KEEP IT + # CLEAR + #################################################### + # Build paths relative to $HOMEgfs + export HOMEgsi="$HOMEgfs" + export FIXgsi="{doc.places.get('FIXgsi','$HOMEgfs/fix/fix_gsi')}" + export HOMEfv3gfs="{doc.places.get('HOMEfv3gfs','$HOMEgfs/sorc/fv3gfs.fd')}" + export HOMEpost="{doc.places.get('HOMEpost','$HOMEgfs')}" + export HOMEobsproc_prep="{doc.places.get('HOMEobsproc_prep', '$BASE_GIT/obsproc/obsproc_prep_RB-5.2.0')}" + export HOMEobsproc_network="{doc.places.get('HOMEobsproc_network', '$BASE_GIT/obsproc/obsproc_global_RB-3.2.0')}" + export BASE_VERIF="{doc.places.get('BASE_VERIF', '$BASE_SVN/verif/global/tags/vsdb')}" + + # CONVENIENT utility scripts and other environment parameters + export NCP="/bin/cp -p" + export NMV="/bin/mv" + export NLN="/bin/ln -sf" + export VERBOSE="YES" + export KEEPDATA="{doc.settings.KEEPDATA}" + export CHGRP_CMD="{doc.platform.CHGRP_RSTPROD_COMMAND}" + export NEMSIOGET="$HOMEgfs/exec/nemsio_get" + + # For coupled + export UGCSsrc="/scratch2/NCEPDEV/climate/Bin.Li/S2S/fix/ocean_ice_post" + + + # Machine environment, jobs, and other utility scripts + export BASE_ENV="$HOMEgfs/env" + export BASE_JOB="$HOMEgfs/jobs/rocoto" + + # EXPERIMENT specific environment parameters + export SDATE={tools.to_YMDH(doc.settings.SDATE)} + export EDATE={tools.to_YMDH(doc.settings.EDATE)} + export assim_freq={doc.data_assimilation.assim_freq} + export PSLOT="{doc.names.experiment}" + export EXPDIR="{doc.places.EXPDIR}" + export ROTDIR="{doc.places.ROTDIR}" + export ROTDIR_DUMP="YES" + export DUMP_SUFFIX="{doc.settings.DUMP_SUFFIX}" + export RUNDIR="{doc.places.DATAROOT}" + export ARCDIR="$NOSCRUB/archive/$PSLOT" + export ICSDIR="{doc.places.ICSDIR}" + export ATARDIR="{doc.archiving.ATARDIR}" + + # Commonly defined parameters in JJOBS + export envir=${{envir:-"prod"}} + export NET="gfs" + export RUN=${{RUN:-${{CDUMP:-"gfs"}}}} + export jlogfile="${{ROTDIR}}/logs/jlogfile" + #export ERRSCRIPT=${{ERRSCRIPT:-'eval [[ $err = 0 ]]'}} + #export LOGSCRIPT=${{LOGSCRIPT:-""}} + export ERRSCRIPT=${{ERRSCRIPT:-"err_chk"}} + export LOGSCRIPT=${{LOGSCRIPT:-"startmsg"}} + export REDOUT="1>" + export REDERR="2>" + + export SENDECF="NO" + export SENDCOM="NO" + export SENDDBN="NO" + export SENDSDM="NO" + + # Adjusting COM paths for non-prod experiments + #export PDY=`expr $CDATE | cut -c1-8` + #export GDATE=`$NDATE -06 $CDATE` + #export PDY_m6hrs=`expr $GDATE | cut -c1-8` + #export COMINgfs="$COMROOThps/$NET/$envir/$RUN.$PDY" + #export COMINgfs_m6hrs="$COMROOThps/$NET/$envir/$RUN.$PDY_m6hrs" + #export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + #export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + #export COMOUTwmo="$ROTDIR/$CDUMP.$PDY/$cyc/wmo" + + # Resolution specific parameters + export LEVS={doc.fv3_settings.LEVS} + export CASE="{doc.fv3_gfs_settings.CASE}" + export CASE_ENKF="{doc.fv3_enkf_settings.CASE}" + + # Surface cycle update frequency + if [ "$CDUMP" == "gdas" ] ; then + export FHCYC={doc.fv3_enkf_settings.FHCYC_GDAS} + export FTSFS=10 + elif [ "$CDUMP" == "gfs" ] ; then + export FHCYC={doc.fv3_enkf_settings.FHCYC_GFS} + fi + + # Output frequency of the forecast model (for cycling) + export FHMIN={doc.gfs_output_settings.FHMIN_GDAS} + export FHMAX={doc.gfs_output_settings.FHMAX_GDAS} + export FHOUT={doc.gfs_output_settings.FHOUT_GDAS} + + # GFS cycle info + export gfs_cyc={doc.settings.gfs_cyc} # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. + + # GFS output and frequency + export FHMIN_GFS={doc.gfs_output_settings.FHMIN_GFS} + export FHMAX_GFS={doc.gfs_output_settings.FHMAX_GFS} + export FHOUT_GFS={doc.gfs_output_settings.FHOUT_GFS} + export FHMAX_HF_GFS={doc.gfs_output_settings.FHMAX_HF_GFS} + export FHOUT_HF_GFS={doc.gfs_output_settings.FHOUT_HF_GFS} + export FHRLST="{doc.gfs_output_settings.gfs_forecast_hours}" + + # I/O QUILTING, true--use Write Component; false--use GFDL FMS + # if quilting=true, choose OUTPUT_GRID as cubed_sphere_grid in netcdf or gaussian_grid + # if gaussian_grid, set OUTPUT_FILE for nemsio or netcdf + export QUILTING="{tools.fort(doc.fv3_settings.QUILTING)}" + export OUTPUT_GRID="{doc.gfs_output_settings.OUTPUT_GRID}" + export OUTPUT_FILE="{doc.gfs_output_settings.OUTPUT_FILE_TYPE}" + + # IAU related parameters + export DOIAU={doc.settings.DOIAU} + export IAUFHRS={doc.settings.IAUFHRS} + export IAU_DELTHRS={doc.settings.IAU_DELTHRS} + export DOIAU_ENKF={doc.fv3_enkf_settings.DOIAU_ENKF} + export IAUFHRS_ENKF={doc.fv3_enkf_settings.IAUFHRS_ENKF} + export IAU_DELTHRS_ENKF={doc.fv3_enkf_settings.IAU_DELTHRS_ENKF} + [[ "$SDATE" = "$CDATE" ]] && ( export DOIAU="NO"; export DOIAU_ENKF="NO" ) + + # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL + export imp_physics={doc.fv3_settings.imp_physics} + + # Shared parameters + # Hybrid related + export DOHYBVAR="{tools.YES_NO(doc.data_assimilation.DOHYBVAR)}" + export NMEM_ENKF={doc.data_assimilation.NMEM_ENKF} + export SMOOTH_ENKF="{tools.YES_NO(doc.data_assimilation.SMOOTH_ENKF)}" + export l4densvar="{tools.fort(doc.data_assimilation.l4densvar)}" + export lwrite4danl="{tools.fort(doc.data_assimilation.lwrite4danl)}" + + # EnKF output frequency + if [ $DOHYBVAR = "YES" ]; then + export FHMIN_ENKF={doc.gfs_output_settings.FHMIN_ENKF} + export FHMAX_ENKF={doc.gfs_output_settings.FHMAX_ENKF} + export FHOUT_ENKF={doc.gfs_output_settings.FHOUT_ENKF} + if [ $l4densvar = ".true." ]; then + export FHOUT=1 + fi + fi + + # If YES, NSST is turned on in anal and/or fcst steps + export DONST="{tools.YES_NO(doc.fv3_settings.DONST)}" + if [ $DONST = "YES" ]; then export FNTSFA=" "; fi + + # The switch to apply SST elevation correction or not + export nst_anl={tools.fort(doc.fv3_settings.nst_anl)} + + # Analysis increments to zero in CALCINCEXEC + export INCREMENTS_TO_ZERO="{tools.join(doc.data_assimilation.INCREMENTS_TO_ZERO,',')}" + + # Archiving options + export DELETE_COM_IN_ARCHIVE_JOB={tools.YES_NO(doc.archiving.scrub_in_archive)} + export HPSSARCH={tools.YES_NO(doc.archiving.archive_to_hpss)} # save data to HPSS archive + export ARCH_CYC={doc.archiving.arch_cyc:02d} # Archive data at this cycle for warm_start capability. Cannot be 00 + export ARCH_WARMICFREQ={doc.archiving.arch_warmicfreq} # Archive frequency in days for warm_start capability + export ARCH_FCSTICFREQ={doc.archiving.arch_fcsticfreq} # Archive frequency in days for gdas and gfs forecast-only capability + + echo "END: config.base" + diff --git a/workflow/config/bulls_navy.yaml b/workflow/config/bulls_navy.yaml new file mode 100644 index 00000000000..a113e075e77 --- /dev/null +++ b/workflow/config/bulls_navy.yaml @@ -0,0 +1,27 @@ +# This file is used to generate config.bulls_navy + +config_bulls_navy: + filename: config.bulls_navy + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.bulls_navy ########## + echo "BEGIN: config.bulls_navy" + + export SENDCOM="YES" + export SENDDBN="YES" + + export PDY=`expr $CDATE | cut -c1-8` + export MDATE=`$NDATE -24 $CDATE` + export PDYm1=`expr $MDATE | cut -c1-8` + export COMIN="$COMROOThps/$NET/$envir/$RUN.$PDY" + export COMINm1="$COMROOThps/$NET/$envir/$RUN.$PDYm1" + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc + export COMOUTwmo=$ROTDIR/$RUN.$PDY/$cyc/wmo + + echo "END: config.bulls_navy" + diff --git a/workflow/config/earc.yaml b/workflow/config/earc.yaml new file mode 100644 index 00000000000..5fb96d713e5 --- /dev/null +++ b/workflow/config/earc.yaml @@ -0,0 +1,24 @@ +# This file is used to generate config.earc, which controls archiving +# of ESMF member data. The scripts that do that archiving are not in +# this public release, so this file is ignored. +config_earc: + filename: config.earc + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.earc ########## + # Ensemble archive specific + + echo "BEGIN: config.earc" + + export NMEM_EARCGRP={doc.data_assimilation.NMEM_EARCGRP} + + export npe_earc=1 + export npe_node_earc=1 + + echo "END: config.earc" + diff --git a/workflow/config/ecen.yaml b/workflow/config/ecen.yaml new file mode 100644 index 00000000000..b410ed22a9c --- /dev/null +++ b/workflow/config/ecen.yaml @@ -0,0 +1,37 @@ +# This file is used to generate config.ecen, which would be used to +# control ensemble recentering. The scripts and code for that are not +# part of this public release, so this file is ignored. + +config_ecen: + filename: config.ecen + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.ecen ########## + # Ensemble recentering specific + + echo "BEGIN: config.ecen" + + export npe_ecen="{doc.partition_common.resources.run_ecen[0].mpi_ranks}" + export npe_node_ecen="{doc.partition_common.resources.run_ecen[0].max_ppn}" + export nth_ecen="{doc.partition_common.resources.run_ecen[0].OMP_NUM_THREADS}" + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export nth_ecen=7 + fi + export nth_cycle=$nth_ecen + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_ecen="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_ecen="3072M" + fi + + export ENKFRECENSH="$HOMEgsi/scripts/exglobal_enkf_recenter_fv3gfs.sh.ecf" + + export CHGRESEXEC="$HOMEgfs/exec/chgres_recenter.exe" + + echo "END: config.ecen" + diff --git a/workflow/config/efcs.yaml b/workflow/config/efcs.yaml new file mode 100644 index 00000000000..f6e8ad00a81 --- /dev/null +++ b/workflow/config/efcs.yaml @@ -0,0 +1,70 @@ +# This file is used to generate config.efcs, which would be used to +# control the ENKF forecast jobs. The scripts and code for the ENKF +# are not provided in this release, so this config file is ignored. + +config_efcs: + filename: config.efcs + res_interval: !FirstTrue + - when: !calc doc.fv3_enkf_settings.DOIAU_ENKF=="YES" + do: 3 + - otherwise: !calc doc.fv3_enkf_settings.restart_interval + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.efcs ########## + # Ensemble forecast specific, dependency: config.fcst + + echo "BEGIN: config.efcs" + + # Source model specific information that is resolution dependent + . $EXPDIR/config.fv3 enkf + + export npe_efcs="{doc.partition_common.resources.run_efcs[0].mpi_ranks}" + export npe_node_efcs="{doc.partition_common.resources.run_efcs[0].max_ppn}" + export nth_efcs=$nth_fv3 + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_efcs="254M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_efcs="254" + fi + + export ENKFFCSTSH="$HOMEgsi/scripts/exglobal_enkf_fcst_fv3gfs.sh.ecf" + export NMEM_EFCSGRP={doc.data_assimilation.NMEM_EFCSGRP} + export RERUN_EFCSGRP="{tools.YES_NO(doc.data_assimilation.RERUN_EFCSGRP)}" + + # Stochastic physics parameters (only for ensemble forecasts) + export DO_SKEB="{tools.YES_NO(doc.fv3_enkf_settings.DO_SKEB)}" + export SKEB={doc.fv3_enkf_settings.SKEB} + export SKEB_TAU={doc.fv3_enkf_settings.SKEB_TAU} + export SKEB_LSCALE={doc.fv3_enkf_settings.SKEB_LSCALE} + export SKEBNORM={doc.fv3_enkf_settings.SKEBNORM} + export SKEB_NPASS={doc.fv3_enkf_settings.SKEB_NPASS} + export SKEB_VDOF={doc.fv3_enkf_settings.SKEB_VDOF} + export DO_SHUM="{tools.YES_NO(doc.fv3_enkf_settings.DO_SHUM)}" + export SHUM={doc.fv3_enkf_settings.SHUM} + export SHUM_TAU={doc.fv3_enkf_settings.SHUM_TAU} + export SHUM_LSCALE={doc.fv3_enkf_settings.SHUM_LSCALE} + export DO_SPPT="{tools.YES_NO(doc.fv3_enkf_settings.DO_SPPT)}" + export SPPT={doc.fv3_enkf_settings.SPPT} + export SPPT_TAU={doc.fv3_enkf_settings.SPPT_TAU} + export SPPT_LSCALE={doc.fv3_enkf_settings.SPPT_LSCALE} + export SPPT_LOGIT="{tools.fort(doc.fv3_enkf_settings.SPPT_LOGIT)}" + export SPPT_SFCLIMIT="{tools.fort(doc.fv3_enkf_settings.SPPT_SFCLIMIT)}" + + if [ $QUILTING = ".true." -a $OUTPUT_GRID = "gaussian_grid" ]; then + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_da" + else + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_da_orig" + fi + + # FV3 model namelist parameters to over-ride + export restart_interval={res_interval} + export k_split={doc.fv3_enkf_settings.k_split} # model is unstable with k_split=2, n_split=6 and stochastic physics + export n_split={doc.fv3_enkf_settings.n_split} # make the model stable with k_split=1, n_split=12 + + echo "END: config.efcs" + diff --git a/workflow/config/emcsfc.yaml b/workflow/config/emcsfc.yaml new file mode 100644 index 00000000000..907e3b0f194 --- /dev/null +++ b/workflow/config/emcsfc.yaml @@ -0,0 +1,25 @@ +# This file is used to generate config.emcsfc + +config_emcsfc: + filename: config.emcsfc + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.emcsfc ########## + echo "BEGIN: config.emcsfc" + + export SENDCOM="YES" + + export PDY=`expr $CDATE | cut -c1-8` + export GDATE=`$NDATE -06 $CDATE` + export PDY_m6hrs=`expr $GDATE | cut -c1-8` + export COMINgfs="$DMPDIR/$CDATE/$CDUMP" + export COMINgfs_m6hrs="$DMPDIR/$GDATE/$CDUMP" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + + echo "END: config.emcsfc" + diff --git a/workflow/config/eobs.yaml b/workflow/config/eobs.yaml new file mode 100644 index 00000000000..3bf3b25cc69 --- /dev/null +++ b/workflow/config/eobs.yaml @@ -0,0 +1,47 @@ +# This file is used to generate config.eobs, which would be used to +# control the ensemble observation innovation. The code and scripts +# for that are not part of this public release, so this file is +# ignored. + +config_eobs: + filename: config.eobs + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.eobs config.eomg ########## + # Ensemble innovation specific, dependency config.anal + + echo "BEGIN: config.eobs" + + export npe_eobs="{doc.partition_common.resources.run_eobs[0].mpi_ranks}" + export nth_eobs="{doc.partition_common.resources.run_eobs[0].OMP_NUM_THREADS}" + export npe_node_eobs="{doc.partition_common.resources.run_eobs[0].max_ppn}" + + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export npe_node_eobs=14 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_eobs="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_eobs="3072M" + fi + + export INVOBSSH="$HOMEgsi/scripts/exglobal_innovate_obs_fv3gfs.sh.ecf" + export ENKFINVOBSSH="$HOMEgsi/scripts/exglobal_enkf_innovate_obs_fv3gfs.sh.ecf" + export NMEM_EOMGGRP={doc.data_assimilation.NMEM_EOMGGRP} + export RERUN_EOMGGRP="{tools.YES_NO(doc.data_assimilation.RERUN_EOMGGRP)}" + + # GSI namelist options related to observer for EnKF + export OBSINPUT_INVOBS="dmesh(1)=225.0,dmesh(2)=225.0" + export OBSQC_INVOBS="tcp_width=60.0,tcp_ermin=2.0,tcp_ermax=12.0" + + export USE_RADSTAT="NO" # This can be only used when bias correction is non-zero. + # Diagnostic files must be created for EnKF + export GENDIAG="{tools.YES_NO(doc.data_assimilation.GENDIAG)}" + + echo "END: config.eobs" + diff --git a/workflow/config/epos.yaml b/workflow/config/epos.yaml new file mode 100644 index 00000000000..b89fe0d093b --- /dev/null +++ b/workflow/config/epos.yaml @@ -0,0 +1,40 @@ +# This file is used to generate config.epos, which would be used to +# control the ENKF post-processing. The scripts and code for the ENKF +# are not provided in this release, so this config file is ignored. + +config_epos: + filename: config.epos + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.epos ########## + # Ensemble post processing specific + + echo "BEGIN: config.epos" + + export npe_epos="{doc.partition_common.resources.run_epos[0].mpi_ranks}" + export npe_node_epos="{doc.partition_common.resources.run_epos[0].max_ppn}" + export nth_epos="{doc.partition_common.resources.run_epos[0].OMP_NUM_THREADS}" + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export nth_epos=7 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_epos="254M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_epos="254" + fi + + export ENKFPOSTSH="$HOMEgsi/scripts/exglobal_enkf_post_fv3gfs.sh.ecf" + + # No. of concurrent epos jobs [1 implies sequential] + export NEPOSGRP={doc.data_assimilation.NEPOSGRP} + if [ $l4densvar = ".false." ]; then + export NEPOSGRP=3 + fi + + echo "END: config.epos" + diff --git a/workflow/config/eupd.yaml b/workflow/config/eupd.yaml new file mode 100644 index 00000000000..808ff67fefa --- /dev/null +++ b/workflow/config/eupd.yaml @@ -0,0 +1,37 @@ +# This file is used to generate config.eupd, which would be used to +# control the ENKF member updating scripts. The scripts and code for +# the ENKF are not provided in this release, so this config file is +# ignored. + +config_eupd: + filename: config.eupd + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.eupd ########## + # Ensemble update specific, dependency config.anal + + echo "BEGIN: config.eupd" + + export npe_eupd="{doc.partition_common.resources.run_eupd[0].mpi_ranks}" + export npe_node_eupd="{doc.partition_common.resources.run_eupd[0].max_ppn}" + export nth_eupd="{doc.partition_common.resources.run_eupd[0].OMP_NUM_THREADS}" + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export nth_eupd=7 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_eupd="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_eupd="3072M" + fi + + export ENKFUPDSH="$HOMEgsi/scripts/exglobal_enkf_update_fv3gfs.sh.ecf" + + export NAM_ENKF="{doc.data_assimilation.NAM_ENKF}" + + echo "END: config.eupd" + diff --git a/workflow/config/fax.yaml b/workflow/config/fax.yaml new file mode 100644 index 00000000000..219e11a55a7 --- /dev/null +++ b/workflow/config/fax.yaml @@ -0,0 +1,27 @@ +# This file is used to generate config.fax, which has settings for the GFS fax step + +config_fax: + filename: config.fax + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fax ########## + # GFS fax step specific + + echo "BEGIN: config.fax" + + module load NCL/6.4.0 + module load bufr_dumplist/1.5.0 + module load dumpjb/4.0.0 + + export SENDCOM="YES" + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUTwmo="$COMOUT/wmo" + + echo "END: config.fax" diff --git a/workflow/config/fax_wafs.yaml b/workflow/config/fax_wafs.yaml new file mode 100644 index 00000000000..66b38b6c12d --- /dev/null +++ b/workflow/config/fax_wafs.yaml @@ -0,0 +1,38 @@ +# This file is used to generate config.fax_wafs, which has settings for the GFS wafs step + +config_fax_wafs: + filename: config.fax_wafs + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fax_wafs ########## + # GFS wafs step specific + + echo "BEGIN: config.fax_wafs" + + export SENDCOM="YES" + + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export PARMgfs=$HOMEgfs/parm/wafs + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUTwmo="$COMOUT/wmo" + export PCOM="$COMOUT/wmo" + + export DATAROOT="$RUNDIR/$CDATE/$CDUMP" + export DATA="$DATAROOT/$job" + + ########################################## + ## Source machine runtime environment + ########################################### + . $HOMEgfs/env/{doc.platform.name}.env wafs + status=$? + [[ $status -ne 0 ]] && exit $status + + echo "END: config.fax_wafs" diff --git a/workflow/config/fbwind.yaml b/workflow/config/fbwind.yaml new file mode 100644 index 00000000000..963daccac95 --- /dev/null +++ b/workflow/config/fbwind.yaml @@ -0,0 +1,23 @@ +# This file is used to generate config.fbwind, which has settings for the GFS fbwind step + +config_fbwind: + filename: config.fbwind + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fbwind ########## + # GFS fbwind step specific + + echo "BEGIN: config.fbwind" + + export SENDCOM="YES" + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUTwmo="$COMOUT/wmo" + + echo "END: config.fbwind" diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml new file mode 100644 index 00000000000..fa49bb11994 --- /dev/null +++ b/workflow/config/fcst.yaml @@ -0,0 +1,202 @@ +# This file is used to generate config.fcst, which is one of two files +# used to control the forecast job. The other file is config.fv3. +# This file sets physics, dynamics, and I/O options. + +config_fcst: + filename: config.fcst + FCSTEXEC: !FirstTrue + - when: !calc doc.settings.cplflx==".true." + do: "nems_fv3_mom6_cice5.x" + - otherwise: "global_fv3gfs.x" + more_exports_for_microphys: !FirstTrue + - when: !calc ( doc.fv3_settings.imp_physics==11 ) + do: !expand | + export hord_mt_nh_nonmono={doc.fv3_settings.hord_mt_nh_nonmono} + export hord_xx_nh_nonmono={doc.fv3_settings.hord_xx_nh_nonmono} + export vtdm4_nh_nonmono={doc.fv3_settings.vtdm4_nh_nonmono} + export nord={doc.fv3_settings.nord} + export dddmp={doc.fv3_settings.dddmp} + export d4_bg={doc.fv3_settings.d4_bg} + export lgfdlmprad=".true." + export effr_in=".true." + - otherwise: "" + + level_127: !FirstTrue + - when: !calc doc.fv3_settings.LEVS==128 + do: !expand | + export tau=5.0 + export rf_cutoff=1.0e3 + export d2_bg_k1=0.20 + export d2_bg_k2=0.0 + - otherwise: "" + + resource_a: !FirstTrue + - when: !calc doc.settings.cplflx==".true." + do: !calc doc.partition_common.resources.run_coupled_fcst + - otherwise: !calc doc.partition_common.resources.run_gdasfcst + + resource_b: !FirstTrue + - when: !calc doc.settings.cplflx==".true." + do: !calc doc.partition_common.resources.run_coupled_fcst + - otherwise: !calc doc.partition_common.resources.run_gfsfcst + + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fcst ########## + # Forecast specific + + echo "BEGIN: config.fcst" + + # Source model specific information that is resolution dependent + . $EXPDIR/config.fv3 $CDUMP + + if [ $DONST = "YES" ]; then + . $EXPDIR/config.nsst + fi + + export npe_fcst="{resource_a[0].mpi_ranks}" + export npe_fcst_gfs="{resource_b[0].mpi_ranks}" + export npe_node_fcst="{resource_a[0].max_ppn}" + export nth_fcst="{doc.fv3_gfs_settings.fv3_threads}" + export NTHREADS_FV3="{doc.fv3_gfs_settings.fv3_threads}" + export memory_fcst="1024M" + + ####################################################################### + # PE Mapping + export ATMPETS="{doc.fv3_gfs_settings.ATMPETS}" + export OCNPETS="{doc.fv3_gfs_settings.OCNPETS}" + export ICEPETS="{doc.fv3_gfs_settings.ICEPETS}" + # + ####################################################################### + + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_fcst="1024M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_fcst="1024M" + fi + + export FORECASTSH="$HOMEgfs/scripts/exglobal_fcst_nemsfv3gfs.sh" + export FCSTEXECDIR="$HOMEgfs/exec" + export FCSTEXEC="{FCSTEXEC}" + + # Model configuration + export TYPE="{doc.fv3_settings.nh_type}" + export MONO="{doc.fv3_settings.MONO}" + + # Use stratosphere h2o physics + export h2o_phys="{tools.fort(doc.fv3_settings.h2o_phys)}" + + # Options of stratosphere O3 physics reaction coefficients + export new_o3forc="{tools.YES_NO(doc.fv3_settings_template.new_o3force)}" + + # do_ugwp=T: use unified CGWD and OGWD, and turbulent orographic form drag (TOFD) + # do_ugwp=F: use unified CGWD but old OGWD, TOFD is not uded. + export do_ugwp=".false." + export do_tofd=".true." + export launch_level=$(echo "$LEVS/2.35" |bc) + + # PBL/turbulance schemes + export hybedmf=".false." + export satmedmf=".true." + export isatmedmf=1 + tbf="" + if [ $satmedmf = ".true." ]; then tbf="_satmedmf" ; fi + + # Land surface model. (2--NoahMP, landice=F); (1--Noah, landice=T) + export lsm=1 + if [ $lsm -eq 2 ]; then + export lheatstrg=".false." + export landice=".false." + else + export lheatstrg=".true." + export landice=".true." + fi + + # Radiation options + export IAER=5111 + export iovr_lw=3 + export iovr_sw=3 + export icliq_sw=2 + + # Microphysics configuration + export dnats={doc.fv3_settings.dnats} + export cal_pre="{tools.fort(doc.fv3_settings.cal_pre)}" + export do_sat_adj="{tools.fort(doc.fv3_settings.do_sat_adj)}" + export random_clds="{tools.fort(doc.fv3_settings.random_clds)}" + #export cnvcld="{tools.fort(doc.fv3_settings.cnvcld)}" + export ncld={doc.fv3_settings.ncld} + export FIELD_TABLE="{doc.fv3_settings.FIELD_TABLE}" + export nwat={doc.fv3_settings.nwat} + {more_exports_for_microphys} + {level_127} + + #--------------------------------------------------------------------- + # ideflate: netcdf zlib lossless compression (0-9): 0 no compression + # nbits: netcdf lossy compression level (0-32): 0 lossless + export ideflate=1 + export nbits=14 + export ishuffle=0 + # compression for RESTART files written by FMS + export shuffle=1 + export deflate_level=1 + + # Disable the use of coupler.res; get model start time from model_configure + export USE_COUPLER_RES="NO" + + if [[ "$CDUMP" == "gdas" ]] ; then # GDAS cycle specific parameters + + # Variables used in DA cycling + if [ $QUILTING = ".true." -a $OUTPUT_GRID = "gaussian_grid" ]; then + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_da" + else + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_da_orig" + fi + + # Write restart files at next assimilation time + export restart_interval=6 + export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks} + export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS} + + elif [[ "$CDUMP" == "gfs" ]] ; then # GFS cycle specific parameters + + # Write more variables to output + if [ $QUILTING = ".true." -a $OUTPUT_GRID = "gaussian_grid" ]; then + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table" + else + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_orig" + fi + export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks} + export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS} + + fi + + if [[ $cpl == .true. ]] ; then # coupled model + export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_cpl" + fi + + # Regrid tiles to global Gaussian grid in NEMSIO + export REGRID_NEMSIO_SH="$HOMEgfs/ush/fv3gfs_regrid_nemsio.sh" + if [ $DONST = YES ]; then + export REGRID_NEMSIO_TBL="$HOMEgfs/parm/parm_fv3diag/variable_table_da.txt" + else + export REGRID_NEMSIO_TBL="$HOMEgfs/parm/parm_fv3diag/variable_table_da_nonsst.txt" + fi + + # Remap tiles to global latlon grid in NetCDF + export REMAPSH="$HOMEgfs/ush/fv3gfs_remap.sh" + export master_grid="{doc.post.master_grid}" # 1deg 0p5deg 0p25deg 0p125deg etc + + # Global latlon NetCDF to nemsio utility parameters + export NC2NEMSIOSH="$HOMEgfs/ush/fv3gfs_nc2nemsio.sh" + + # Remember config.efcs will over-ride these values for ensemble forecasts + # if these variables are re-defined there. + # Otherwise, the ensemble forecast will inherit from config.fcst + + echo "END: config.fcst" + diff --git a/workflow/config/fv3.yaml b/workflow/config/fv3.yaml new file mode 100644 index 00000000000..0fafa440ef2 --- /dev/null +++ b/workflow/config/fv3.yaml @@ -0,0 +1,96 @@ +# This file is used to generate config.fv3, which is one of two files +# used to control the forecast job. The other file is config.fcst. +# This file sets task geometry, domain decomposition, and I/O server +# options. + +config_fv3: + filename: config.fv3 + enkf_cdmbgwd_settings: !FirstTrue + - when: !calc ( "cdmbgwd" in doc.fv3_enkf_settings ) + do: !expand "export cdmbgwd={doc.fv3_enkf_settings.cdmbgwd}" + - otherwise: "# not setting cdmbgwd" + gfs_cdmbgwd_settings: !FirstTrue + - when: !calc ( "cdmbgwd" in doc.fv3_gfs_settings ) + do: !expand "export cdmbgwd={doc.fv3_gfs_settings.cdmbgwd}" + - otherwise: "# not setting cdmbgwd" + gdas_cdmbgwd_settings: !FirstTrue + - when: !calc ( "cdmbgwd" in doc.fv3_gdas_settings ) + do: !expand "export cdmbgwd={doc.fv3_gdas_settings.cdmbgwd}" + - otherwise: "# not setting cdmbgwd" + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fv3 ########## + # FV3 model resolution specific parameters + # e.g. time-step, processor layout, physics and dynamics parameters + # This config sets default variables for FV3 for a given resolution + # User can over-ride after sourcing this config file + + if [ $# -ne 1 ]; then + + echo "Must specify a forecast mode to set variables!" + echo "argument can be any one of the following:" + echo "EFCS GFS GDAS (upper- or lower-case is okay)" + err_exit "Missing mandatory argument to config.fv3" + exit 1 + + fi + + forecast_mode=$( echo "$1" | tr a-z A-Z ) + + case $forecast_mode in + "gfs") + echo "BEGIN: config.fv3 for $forecast_mode" + export DELTIM={doc.fv3_gfs_settings.DELTIM} + export layout_x={doc.fv3_gfs_settings.layout_x} + export layout_y={doc.fv3_gfs_settings.layout_y} + export layout_x_gfs={doc.fv3_gfs_settings.layout_x} + export layout_y_gfs={doc.fv3_gfs_settings.layout_y} + export npe_node_fcst={doc.partition_common.resources.run_gdasfcst[0].max_ppn} + export nth_fv3=2 + export npe_fv3={doc.partition_common.resources.run_gdasfcst[0].mpi_ranks} + export cdmbgwd={doc.fv3_gfs_settings.cdmbgwd} + # mountain blocking and gravity wave drag + export WRITE_GROUP={doc.fv3_gfs_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP={doc.fv3_gfs_settings.WRTTASK_PER_GROUP} + export WRITE_GROUP_GFS={doc.fv3_gfs_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP_GFS={doc.fv3_gfs_settings.WRTTASK_PER_GROUP} + export WRTIOBUF={doc.fv3_gfs_settings.WRTIOBUF} + "gdas") + echo "BEGIN: config.fv3 for $forecast_mode" + export DELTIM={doc.fv3_gdas_settings.DELTIM} + export layout_x={doc.fv3_gdas_settings.layout_x} + export layout_y={doc.fv3_gdas_settings.layout_y} + export layout_x_gfs={doc.fv3_gdas_settings.layout_x} + export layout_y_gfs={doc.fv3_gdas_settings.layout_y} + export npe_node_fcst={doc.partition_common.resources.run_gdasfcst[0].max_ppn} + export nth_fv3=2 + export npe_fv3={doc.partition_common.resources.run_gdasfcst[0].mpi_ranks} + export cdmbgwd={doc.fv3_gdas_settings.cdmbgwd} + export WRITE_GROUP={doc.fv3_gdas_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP={doc.fv3_gdas_settings.WRTTASK_PER_GROUP} + export WRITE_GROUP_GFS={doc.fv3_gdas_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP_GFS={doc.fv3_gdas_settings.WRTTASK_PER_GROUP} + export WRTIOBUF={doc.fv3_gdas_settings.WRTIOBUF} + "enkf") + echo "BEGIN: config.fv3 for $forecast_mode" + export DELTIM={doc.fv3_enkf_settings.DELTIM} + export layout_x={doc.fv3_enkf_settings.layout_x} + export layout_y={doc.fv3_enkf_settings.layout_y} + export layout_x_gfs={doc.fv3_enkf_settings.layout_x} + export layout_y_gfs={doc.fv3_enkf_settings.layout_y} + export npe_node_fcst={doc.partition_common.resources.run_efcs[0].max_ppn} + export nth_fv3=2 + export npe_fv3={doc.partition_common.resources.run_efcs[0].mpi_ranks} + export cdmbgwd={doc.fv3_enkf_settings.cdmbgwd} + export WRITE_GROUP={doc.fv3_enkf_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP={doc.fv3_enkf_settings.WRTTASK_PER_GROUP} + export WRITE_GROUP_GFS={doc.fv3_enkf_settings.WRITE_GROUP} + export WRTTASK_PER_GROUP_GFS={doc.fv3_enkf_settings.WRTTASK_PER_GROUP} + export WRTIOBUF={doc.fv3_enkf_settings.WRTIOBUF} + + echo "END: config.fv3 for $forecast_mode" diff --git a/workflow/config/fv3ic.yaml b/workflow/config/fv3ic.yaml new file mode 100644 index 00000000000..f1d967e65c1 --- /dev/null +++ b/workflow/config/fv3ic.yaml @@ -0,0 +1,26 @@ +# This file is used to generate config.fv3ic, which controls the +# chgres. That is the program that converts Global Spectral Model +# analysis files to Finite Volume 3 input conditions. + +config_fv3ic: + filename: config.fv3ic + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.fv3ic ########## + # Convert GFS initial conditions into FV3 initial conditions + + echo "BEGIN: config.fv3ic" + + # Task and thread configuration + export npe_fv3ic={doc.partition_common.resources.run_fv3ic[0].mpi_ranks} + export npe_node_fv3ic={doc.partition_common.resources.run_fv3ic[0].mpi_ranks} + + export ORIGIN_ROOT="{doc.places.MOM6IC}" + + echo "END: config.fv3ic" + diff --git a/workflow/config/gempak.yaml b/workflow/config/gempak.yaml new file mode 100644 index 00000000000..a79197581d4 --- /dev/null +++ b/workflow/config/gempak.yaml @@ -0,0 +1,38 @@ +# This file is used to generate config.gempak, which has settings for the GFS gempak step + +config_gempak: + filename: config.gempak + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.gempak ########## + # GFS gempak step specific + + echo "BEGIN: config.gempak" + + export npe_gempak="{doc.partition_common.resources.run_gempak[0].mpi_ranks}" + export npe_node_gempak="{doc.partition_common.resources.run_gempak[0].max_ppn}" + export nth_gempak="{doc.partition_common.resources.run_gempak[0].OMP_NUM_THREADS}" + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_gempak="254M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_gempak="254" + fi + + export SENDCOM="YES" + + export GEMPAKSH=$HOMEgfs/jobs/JGFS_GEMPAK + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export FIXgempak=$HOMEgfs/gempak/fix + export USHgempak=$HOMEgfs/gempak/ush + + export COMIN=$ROTDIR/$RUN.$PDY/$cyc + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc/nawips + export COMINgempak="$ROTDIR" + + echo "END: config.gempak" diff --git a/workflow/config/gempak_meta.yaml b/workflow/config/gempak_meta.yaml new file mode 100644 index 00000000000..5401a85dc86 --- /dev/null +++ b/workflow/config/gempak_meta.yaml @@ -0,0 +1,28 @@ +# This file is used to generate config.gempak_meta, which has settings for the GFS gempak_meta step + +config_gempak_meta: + filename: config.gempak_meta + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.gempak_meta ########## + # GFS gempak_meta step specific + + echo "BEGIN: config.gempak_meta" + + export SENDCOM="YES" + + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export FIXgempak=$HOMEgfs/gempak/fix + export USHgempak=$HOMEgfs/gempak/ush + + export COMIN=$ROTDIR/$RUN.$PDY/$cyc/nawips + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc/nawips/meta + export COMINgempak="$ROTDIR" + + echo "END: config.gempak_meta" diff --git a/workflow/config/gempak_meta_ncdc.yaml b/workflow/config/gempak_meta_ncdc.yaml new file mode 100644 index 00000000000..aa17455a869 --- /dev/null +++ b/workflow/config/gempak_meta_ncdc.yaml @@ -0,0 +1,29 @@ +# This file is used to generate config.gempak_meta_ncdc, which has settings for the GFS gempak_meta_ncdc step + +config_gempak_meta_ncdc: + filename: config.gempak_meta_ncdc + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.gempak_meta_ncdc ########## + # GFS gempak_meta_ncdc step specific + + echo "BEGIN: config.gempak_meta_ncdc" + + export SENDCOM="YES" + + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export FIXgempak=$HOMEgfs/gempak/fix + export USHgempak=$HOMEgfs/gempak/ush + + export COMIN=$ROTDIR/$RUN.$PDY/$cyc/nawips + export COMINgdas=$ROTDIR/$RUN + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc + export COMOUTncdc=$ROTDIR/$RUN.$PDY/$cyc + + echo "END: config.gempak_meta_ncdc" diff --git a/workflow/config/gempak_ncdc_upapgif.yaml b/workflow/config/gempak_ncdc_upapgif.yaml new file mode 100644 index 00000000000..f2e99fd646a --- /dev/null +++ b/workflow/config/gempak_ncdc_upapgif.yaml @@ -0,0 +1,29 @@ +# This file is used to generate config.gempak_ncdc_upapgif, which has settings for the GFS gempak_ncdc_upapgif step + +config_gempak_ncdc_upapgif: + filename: config.gempak_ncdc_upapgif + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.gempak_ncdc_upapgif ########## + # GFS gempak_ncdc_upapgif step specific + + echo "BEGIN: config.gempak_ncdc_upapgif" + + export SENDCOM="YES" + + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export FIXgempak=$NWROOTp1/gempak/fix + export USHgempak=$HOMEgfs/gempak/ush + + export COMIN=$ROTDIR/$RUN.$PDY/$cyc/nawips + export COMINgfs="$COMROOThps/$NET/$envir/$RUN.$PDY" + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc + export COMOUTwmo="$COMOUT/wmo" + + echo "END: config.gempak_ncdc_upapgif" diff --git a/workflow/config/getic.yaml b/workflow/config/getic.yaml new file mode 100644 index 00000000000..b5f682172b9 --- /dev/null +++ b/workflow/config/getic.yaml @@ -0,0 +1,31 @@ +# This file is used to generate config.getic, which is used to control +# the logic that pulls input data from HPSS. + +config_getic: + filename: config.getic + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.getic ########## + # Fetching GFS initial conditions specific + + echo "BEGIN: config.getic" + + # We should just be supporting the OPSGFS only + export ics_from="{doc.settings.ics_from}" # initial conditions from opsgfs or pargfs + + export npe_getic=1 + export npe_node_getic=1 + + # Provide a parallel experiment name and path to HPSS archive + if [ $ics_from = "pargfs" ]; then + export parexp="{doc.settings.parexp}" + export HPSS_PAR_PATH="{doc.places.HPSS_PAR_PATH}" + fi + + echo "END: config.getic" + diff --git a/workflow/config/nsst.yaml b/workflow/config/nsst.yaml new file mode 100644 index 00000000000..d37114b33d8 --- /dev/null +++ b/workflow/config/nsst.yaml @@ -0,0 +1,50 @@ +# This file is used to generate config.nsst, which would be used to +# control NSST settings. Specifically, it controls the nstf_name +# option in the gfs physics namelist (&gfs_physics_nml) + +config_nsst: + filename: config.nsst + nst_spinup_logic: !FirstTrue + - when: !calc doc.nsst.get("NST_SPINUP","") + do: !expand "export NST_SPINUP=$NST_SPINUP" + - otherwise: | + export NST_SPINUP=0 + #if [[ "$CDATE" = $SDATE ]]; then + # export NST_SPINUP=1 + #fi + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.nsst ########## + # NSST specific + + echo "BEGIN: config.nsst" + + # NSST parameters contained within nstf_name + + # nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled + export NST_MODEL={doc.nsst.NST_MODEL} + + # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, + {nst_spinup_logic} + + # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON + export NST_RESV={doc.nsst.NST_RESV} + + # nstf_name(4,5) : ZSEA1, ZSEA2 the two depths to apply vertical average (bias correction) + export ZSEA1={doc.nsst.ZSEA1} + export ZSEA2={doc.nsst.ZSEA2} + + export NST_GSI={doc.nsst.NST_GSI} # default 0: No NST info at all; + # 1: Input NST info but not used in GSI; + # 2: Input NST info, used in CRTM simulation, no Tr analysis + # 3: Input NST info, used in both CRTM simulation and Tr analysis + export NSTINFO={doc.nsst.NSTINFO} # number of elements added in obs. data array (default = 0) + if [ $NST_GSI -gt 0 ]; then export NSTINFO=4; fi + + echo "END: config.nsst" + diff --git a/workflow/config/ocnpost.yaml b/workflow/config/ocnpost.yaml new file mode 100644 index 00000000000..cd37e8b691c --- /dev/null +++ b/workflow/config/ocnpost.yaml @@ -0,0 +1,24 @@ +# This file is used to generate config.ocnpost, which is used to control +# the deterministic ocean post processing. + +config_ocnpost: + filename: config.ocnpost + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.ocnpost ########## + # Post specific + + echo "BEGIN: config.ocnpost" + + # Convert nemsio files to grib files using post job + #------------------------------------------- + + # No. of concurrent post jobs [0 implies sequential] + export NPOSTGRP={doc.ocnpost.NPOSTGRP} + + echo "END: config.ocnpost" diff --git a/workflow/config/pgrb2_spec_npoess.yaml b/workflow/config/pgrb2_spec_npoess.yaml new file mode 100644 index 00000000000..78f95e3fcd7 --- /dev/null +++ b/workflow/config/pgrb2_spec_npoess.yaml @@ -0,0 +1,28 @@ +# This file is used to generate config.pgrb2_spec_npoess, which has settings for the GFS pgrb2_spec_npoess step + +config_pgrb2_spec_npoess: + filename: config.pgrb2_spec_npoess + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.pgrb2_spec_npoess ########## + # GFS pgrb2_spec_npoess step specific + + echo "BEGIN: config.pgrb2_spec_npoess" + + export SENDCOM="YES" + + export GEMPAKSH=$HOMEgfs/jobs/JGFS_GEMPAK + export FIXgfs=$HOMEgfs/gempak/fix + export USHgfs=$HOMEgfs/gempak/ush + export FIXgempak=$HOMEgfs/gempak/fix + export USHgempak=$HOMEgfs/gempak/ush + + export COMIN=$ROTDIR/$RUN.$PDY/$cyc + export COMOUT=$ROTDIR/$RUN.$PDY/$cyc + + echo "END: config.pgrb2_spec_npoess" diff --git a/workflow/config/post.yaml b/workflow/config/post.yaml new file mode 100644 index 00000000000..6a8d46afc9c --- /dev/null +++ b/workflow/config/post.yaml @@ -0,0 +1,68 @@ +# This file is used to generate config.post, which is used to control +# the deterministic atmospheric post processing. + +config_post: + filename: config.post + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.post ########## + # Post specific + + echo "BEGIN: config.post" + + # Convert nemsio files to grib files using post job + #------------------------------------------- + + # No. of concurrent post jobs [0 implies sequential] + export NPOSTGRP={doc.post.NPOSTGRP} + + export npe_post="{doc.partition_common.resources.run_gdaspost[0].mpi_ranks}" + export nth_post="{doc.partition_common.resources.run_gdaspost[0].OMP_NUM_THREADS}" + export npe_node_post="{doc.partition_common.resources.run_gdaspost[0].max_ppn}" + export npe_node_dwn=24 + if [[ "$machine" == "WCOSS_DELL_P3" ]]; then + export npe_node_post=14 + export npe_node_dwn=28 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_post="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_post="3072M" + fi + + # Post driver job that calls gfs_nceppost.sh and downstream jobs + export POSTJJOBSH="$HOMEpost/jobs/JGLOBAL_NCEPPOST" + export GFSDOWNSH="$HOMEpost/ush/fv3gfs_downstream_nems.sh" + export GFSDWNSH="$HOMEpost/ush/fv3gfs_dwn_nems.sh" + + export POSTGPSH="$HOMEpost/ush/gfs_nceppost.sh" + export POSTGPEXEC="$HOMEpost/exec/gfs_ncep_post" + export GOESF={tools.YES_NO(doc.post.GOESF)} # goes image + export GTGF={tools.YES_NO(doc.post.GTGF)} # gtg icing product + export FLXF={tools.YES_NO(doc.post.FLXF)} # grib2 flux file written by post + export PGB1F={tools.YES_NO(doc.post.PGB1F)} + if [ $RUN_ENVIR = "nco" ]; then + export PGB1F=NO + export GTGF=YES + fi + + if [[ "$CDUMP" == gdas ]] ; then + export nth_postgp={doc.partition_common.resources.run_gdaspost[0].OMP_NUM_THREADS} + else + export nth_postgp={doc.partition_common.resources.run_gfspost[0].OMP_NUM_THREADS} + fi + export GFS_DOWNSTREAM={tools.YES_NO(doc.post.GFS_DOWNSTREAM)} + export downset={doc.post.downset} + export npe_dwn={doc.partition_common.resources.run_dwn[0].mpi_ranks} + + export GRIBVERSION='grib2' + export SENDCOM="YES" + + export CRTM_FIX="{doc.platform.RTMFIX}" + + echo "END: config.post" diff --git a/workflow/config/postsnd.yaml b/workflow/config/postsnd.yaml new file mode 100644 index 00000000000..2541188dd6a --- /dev/null +++ b/workflow/config/postsnd.yaml @@ -0,0 +1,52 @@ +# This file is used to generate config.postsnd, which has settings for the GFS postsnd step + +config_postsnd: + filename: config.postsnd + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.postsnd ########## + # GFS bufr sounding step specific + + echo "BEGIN: config.postsnd" + + export npe_postsnd="{doc.partition_common.resources.run_postsnd[0].mpi_ranks}" + export npe_node_postsnd="{doc.partition_common.resources.run_postsnd[0].max_ppn}" + export npe_postsndcfp="{doc.partition_common.resources.run_postsndcfp[0].mpi_ranks}" + export npe_node_postsndcfp="{doc.partition_common.resources.run_postsndcfp[0].max_ppn}" + if [[ "$machine" == "WCOSS_DELL_P3" ]]; then + export npe_node_postsnd=4 + export npe_postsndcfp=9 + fi + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_postsnd="254M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_postsnd="254" + fi + + export CDATEm1=$($NDATE -24 $CDATE) + export PDYm1=$(echo $CDATEm1 | cut -c1-8) + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export DATAROOT="$RUNDIR/$CDATE/$CDUMP/postsnd" + + export job="jgfs_postsnd_$cyc" + export jlogfile="$ROTDIR/logs/$CDATE/$job.log" + export DATA="$DATAROOT/$job" + export SENDCOM="YES" + export SENDDBN="YES" + export HOMEbufrsnd=$HOMEgfs + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export pcom="$COMOUT/wmo" + export COMAWP="$COMOUT/nawips" + + export POSTSNDSH=$HOMEgfs/jobs/JGFS_POSTSND + + if [ $machine = "JET" ]; then + export POSTSNDSH=$HOMEgfs/jobs/JGFS_POSTSND_JET + fi + + echo "END: config.postsnd" diff --git a/workflow/config/prep.yaml b/workflow/config/prep.yaml new file mode 100644 index 00000000000..6dab4d3c48b --- /dev/null +++ b/workflow/config/prep.yaml @@ -0,0 +1,67 @@ +# This file is used to generate config.prep, which controls +# observation pre-processing. The output of this is sent into the GFS +# or GDAS analysis. The observation processing system is not part of +# this public release, so this file is ignored. + +config_prep: + filename: config.prep + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.prep ########## + # Prep step specific + + echo "BEGIN: config.prep" + + export DO_MAKEPREPBUFR="{tools.YES_NO(doc.data_assimilation.DO_MAKEPREPBUFR)}" # if NO, will copy prepbufr from globaldump + + # Relocation and syndata QC + export PROCESS_TROPCY="{tools.YES_NO(doc.data_assimilation.PROCESS_TROPCY)}" + [[ $RUN_ENVIR == "nco" && $envir == "prod" ]] && export PROCESS_TROPCY="YES" + export DO_RELOCATE="{tools.YES_NO(doc.data_assimilation.DO_RELOCATE)}" + export TROPCYQCRELOSH="$HOMEgfs/scripts/extropcy_qc_reloc.sh.ecf" + export SENDCOM="YES" + + export HOMERELO=$HOMEgfs + export EXECRELO=${{HOMERELO}}/exec + export FIXRELO=${{HOMERELO}}/fix/fix_am + export USHRELO=${{HOMERELO}}/ush + + export npe_node_prep="{doc.partition_common.resources.run_prep[0].max_ppn}" + export npe_prep="{doc.partition_common.resources.run_prep[0].mpi_ranks}" + export nth_prep="{doc.partition_common.resources.run_prep[0].OMP_NUM_THREADS}" + + export cycle="t\"$cyc\"z" + export OPREFIX="{doc.data_assimilation.OPREFIX}" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + [[ ! -d $COMOUT ]] && mkdir -p $COMOUT + + export DATAROOT={doc.places.DATAROOT}/prepbufr + export COMIN_OBS={doc.data_assimilation.COMIN_OBS} + export COMSP=$COMIN_OBS/$OPREFIX + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMINgdas="$ROTDIR/gdas.$PDY/$cyc" + export COMINgfs="$ROTDIR/gfs.$PDY/$cyc" + + ############################################################### + # For running real-time parallels on WCOSS_C, execute tropcy_qc and + # copy files from operational syndata directory to a local directory. + # Otherwise, copy existing tcvital data from globaldump. + + if [ $PROCESS_TROPCY = "NO" ]; then + cp $DMPDIR/$CDATE/$CDUMP/$CDUMP.$cycle.syndata.tcvitals.tm00 $COMOUT/. + fi + ############################################################### + # Generate prepbufr files from dumps or copy from OPS + if [ $DO_MAKEPREPBUFR = "NO" ]; then + $NCP $DMPDIR/$CDATE/$CDUMP/$CDUMP.$cycle.prepbufr $COMOUT/$CDUMP.$cycle.prepbufr + $NCP $DMPDIR/$CDATE/$CDUMP/$CDUMP.$cycle.prepbufr.acft_profiles $COMOUT/$CDUMP.$cycle.prepbufr.acft_profiles + [[ $DONST = "YES" ]] && $NCP $DMPDIR/$CDATE/$CDUMP/$CDUMP.$cycle.nsstbufr $COMOUT/$CDUMP.$cycle.nsstbufr + fi + + echo "END: config.prep" + diff --git a/workflow/config/prepbufr.yaml b/workflow/config/prepbufr.yaml new file mode 100644 index 00000000000..91bc3ae0d3a --- /dev/null +++ b/workflow/config/prepbufr.yaml @@ -0,0 +1,39 @@ +# This file is used to generate config.prepbufr, which is used to +# generate the prepbuf file that contains some of the observations +# sent into the GFS and GDAS analysis. The observation processing +# suite is not included in this release, so this file is ignored. + +config_prepbufr: + filename: config.prepbufr + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.prepbufr ########## + # PREPBUFR specific configuration + + echo "BEGIN: config.prepbufr" + + # Set variables + #------------------------------ + if [ $machine = "THEIA" ]; then + #------------------------------ + + export GESROOT=/scratch4/NCEPDEV/rstprod/nwges/prod # set by module prod_envir on WCOSS_C + + elif [ $machine = "JET" ]; then + + export GESROOT=/mnt/lfs3/projects/hfv3gfs/glopara/nwges/prod # this is a dummy dir + + fi + + export npe_prepbufr={doc.partition_common.resources.run_prepbufr[0].mpi_ranks} + export npe_node_prepbufr={doc.partition_common.resources.run_prepbufr[0].max_ppn} + export nth_prepbufr="{doc.partition_common.resources.run_prepbufr[0].OMP_NUM_THREADS}" + + + echo "END: config.prepbufr" + diff --git a/workflow/config/relocate.yaml b/workflow/config/relocate.yaml new file mode 100644 index 00000000000..52ef9dd70cd --- /dev/null +++ b/workflow/config/relocate.yaml @@ -0,0 +1,50 @@ +# This file is used to generate config.relocate, which controls +# observation pre-processing. The output of this is sent into the GFS +# or GDAS analysis. The observation processing system is not part of +# this public release, so this file is ignored. + +config_relocate: + filename: config.relocate + content: !expand | + #!/bin/ksh -x + + ########## config.relocate ########## + # Prep step specific + + echo "BEGIN: config.relocate" + + set -x + # Relocation and syndata QC + export PROCESS_TROPCY="{tools.YES_NO(doc.data_assimilation.PROCESS_TROPCY)}" + [[ $RUN_ENVIR == "nco" && $envir == "prod" ]] && export PROCESS_TROPCY="YES" + export DO_RELOCATE="{tools.YES_NO(doc.data_assimilation.DO_RELOCATE)}" + export TROPCYQCRELOSH="$HOMEgfs/scripts/extropcy_qc_reloc.sh.ecf" + export SENDCOM="YES" + + export HOMERELO=$HOMEgfs + export EXECRELO=${{HOMERELO}}/exec + export FIXRELO=${{HOMERELO}}/fix/fix_am + export USHRELO=${{HOMERELO}}/ush + + export cycle="t\"$cyc\"z" + export OPREFIX="{doc.data_assimilation.OPREFIX}" + + ################################### + # Only used when DO_RELOCATE=YES + # ignore them for now + ################################### + #export machine=${{machine:-WCOSS_C}} + #if [ $machine = WCOSS_C ] ; then + # export APRNGETTX="time aprun -q -j1 -n1 -N1 -d1 -cc depth" + # export APRNRELOC="time aprun -q -j1 -n7 -N1 -d24 -cc depth " + # export APRNSYNDX="time aprun -q -j1 -n1 -N1 -d1 -cc depth" + #fi + + # If PROCESS_TROPCY=NO, copy over tcvitals files to COMROT + if [ $PROCESS_TROPCY = "NO" ]; then + cp $DMPDIR/$CDATE/$CDUMP/{doc.data_assimilation.OPREFIX}syndata.tcvitals.tm00 $COMOUT/. + fi + set +x + + echo "END: config.relocate" + diff --git a/workflow/config/vrfy.yaml b/workflow/config/vrfy.yaml new file mode 100644 index 00000000000..2c798bacae0 --- /dev/null +++ b/workflow/config/vrfy.yaml @@ -0,0 +1,241 @@ +# This file is used to generate config.vrfy which runs some of the +# post-processing and the VSDB. That functionality is not included in +# this release, so this file is ignored. + +config_vrfy: + filename: config.vrfy + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.vrfy ########## + # Verification step specific + + echo "BEGIN: config.vrfy" + + export npe_vrfy=1 + export npe_node_vrfy=1 + export npe_vrfy_gfs=1 + export npe_node_vrfy_gfs=1 + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_vrfy="3072M" + elif [[ "$machine" == "GAEA" ]]; then + export memory_vrfy="3072M" + elif [[ "$machine" == "THEIA" ]]; then + export memory_vrfy="16384M" + fi + + export VDUMP="{doc.downstream.VDUMP}" # verifying dump + export CDUMPFCST="{doc.downstream.CDUMPFCST}" # Fit-to-obs with GDAS/GFS prepbufr + export CDFNL="{doc.downstream.CDFNL}" # Scores verification against GDAS/GFS analysis + + export MKPGB4PRCP="{tools.YES_NO(doc.downstream.MKPGB4PRCP)}" # make 0.25-deg pgb files in ARCDIR for precip verification + export VRFYFITS="{tools.YES_NO(doc.downstream.VRFYFITS)}" # Fit to observations + export VSDB_STEP1="{tools.YES_NO(doc.downstream.VSDB_STEP1)}" # populate VSDB database + export VSDB_STEP2="{tools.YES_NO(doc.downstream.VSDB_STEP2)}" + export VRFYG2OBS="{tools.YES_NO(doc.downstream.VRFYG2OBS)}" # Grid to observations, see note below if turning ON + export VRFYPRCP="{tools.YES_NO(doc.downstream.VRFYPRCP)}" # Precip threat scores + export VRFYRAD="{tools.YES_NO(doc.downstream.VRFYRAD)}" # Radiance data assimilation monitoring + export VRFYOZN="{tools.YES_NO(doc.downstream.VRFYOZN)}" # Ozone data assimilation monitoring + export VRFYMINMON="{tools.YES_NO(doc.downstream.VRFYMINMON)}" # GSI minimization monitoring + export VRFYTRAK="{tools.YES_NO(doc.downstream.VRFYTRAK)}" # Hurricane track verification + export VRFYGENESIS="{tools.YES_NO(doc.downstream.VRFYGENESIS)}" # Cyclone genesis verification + export RUNMOS="{tools.YES_NO(doc.downstream.RUNMOS)}" # whether to run entire MOS package + + #------------------------------------------------- + # Fit to Observations + #------------------------------------------------- + + if [ $VRFYFITS = "YES" ]; then + + export PRVT=$HOMEgfs/fix/fix_gsi/prepobs_errtable.global + export HYBLEVS=$HOMEgfs/fix/fix_am/global_hyblev.l65.txt + export CUE2RUN=$QUEUE + + if [ $machine = "WCOSS_C" ]; then + export fitdir="$BASE_SVN/verif/global/parafits.fv3nems/batrun" + export PREPQFITSH="$fitdir/subfits_cray_nems" + elif [ $machine = "WCOSS_DELL_P3" ]; then + export fitdir="$BASE_SVN/verif/global/parafits.fv3nems/batrun" + export PREPQFITSH="$fitdir/subfits_dell_nems" + elif [ $machine = "THEIA" ]; then + export fitdir="$BASE_SVN/verif/global/parafits.fv3nems/batrun" + export PREPQFITSH="$fitdir/subfits_theia_nems" + elif [ $machine = "JET" ]; then + export fitdir="$BASE_GIT/verif/global/parafits.fv3nems.dell/batrun" + export PREPQFITSH="$fitdir/subfits_jet_nems" + fi + + fi + + + #---------------------------------------------------------- + # VSDB STEP1, Verify Precipipation and Grid To Obs options + #---------------------------------------------------------- + # All these call $VSDBSH + + if [[ "$CDUMP" = "gfs" ]] ; then + ddd=`echo $CDATE |cut -c 1-8` + #if [ $ddd -eq 5 -o $ddd -eq 10 ]; then export VSDB_STEP2 = "YES" ;fi + + if [ $VSDB_STEP1 = "YES" -o $VSDB_STEP2 = "YES" -o $VRFYPRCP = "YES" -o $VRFYG2OBS = "YES" ]; then + export BACKDATEVSDB=24 # execute vsdbjob for the previous day + export VBACKUP_PRCP=24 # back up for QPF verification data + export vsdbsave="$NOSCRUB/archive/vsdb_data" # place to save vsdb database + export vsdbhome=$BASE_VERIF # location of global verification scripts + export VSDBSH="$vsdbhome/vsdbjob.sh" # VSDB job script + export vlength=$FHMAX_GFS # verification length + export vhr_rain=$FHMAX_GFS # verification length for precip + export ftyplist="pgbq" # verif. files used for computing QPF ETS scores + export ptyplist="PRATE" # precip types in GRIB: PRATE or APCP + export anltype="gfs" # default=gfs, analysis type (gfs or gdas) for verification + export rain_bucket=6 # prate in pgb files is 6-hr accumulated + + export VSDB_START_DATE="$SDATE" # starting date for vsdb maps + export webhost="emcrzdm.ncep.noaa.gov" # webhost(rzdm) computer + export webhostid="$USER" # webhost(rzdm) user name + export SEND2WEB="NO" # whether or not to send maps to webhost + export WEBDIR="/home/people/emc/www/htdocs/gmb/${{webhostid}}/vsdb/$PSLOT" + export mdlist="gfs $PSLOT " # exps (up to 10) to compare in maps + fi + fi + + + #---------------------------------------------------------- + # Minimization, Radiance and Ozone Monitoring + #---------------------------------------------------------- + + if [ $VRFYRAD = "YES" -o $VRFYMINMON = "YES" -o $VRFYOZN = "YES" ]; then + + export envir="para" + + # Radiance Monitoring + if [[ "$VRFYRAD" == "YES" && "$CDUMP" == "$CDFNL" ]] ; then + + export RADMON_SUFFIX=$PSLOT + export TANKverf="$NOSCRUB/monitor/radmon" + if [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" -o $machine = "THEIA" -o $machine = "JET" ]; then + export VRFYRADSH="$HOMEgfs/jobs/JGDAS_VERFRAD" + else + echo "WARNING: Radiance monitoring is not enabled on $machine!" + export VRFYRAD="NO" + fi + + fi + + # Minimization Monitoring + if [[ "$VRFYMINMON" = "YES" ]] ; then + + export MINMON_SUFFIX=$PSLOT + export M_TANKverf="$NOSCRUB/monitor/minmon" + if [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" -o $machine = "THEIA" -o $machine = "JET" ]] ; then + if [[ "$CDUMP" = "gdas" ]] ; then + export VRFYMINSH="$HOMEgfs/jobs/JGDAS_VMINMON" + elif [[ "$CDUMP" = "gfs" ]] ; then + export VRFYMINSH="$HOMEgfs/jobs/JGFS_VMINMON" + fi + else + echo "WARNING: Minimization monitoring is not enabled on $machine!" + export VRFYMINMON="NO" + fi + + fi + + # Ozone Monitoring + if [[ "$VRFYOZN" == "YES" && "$CDUMP" == "$CDFNL" ]] ; then + + export HOMEgfs_ozn="$HOMEgfs" + export OZNMON_SUFFIX=$PSLOT + export TANKverf_ozn="$NOSCRUB/monitor/oznmon" + if [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" -o $machine = "THEIA" -o $machine = "JET" ]; then + export VRFYOZNSH="$HOMEgfs/jobs/JGDAS_VERFOZN" + else + echo "WARNING: Ozone monitoring is not enabled on $machine!" + export VRFYOZN="NO" + fi + + fi + + fi + + + #------------------------------------------------- + # Cyclone genesis and cyclone track verification + #------------------------------------------------- + + if [ $machine = "WCOSS_C" ] ; then + export ens_tracker_ver=v1.1.13.1 + export HOMEens_tracker=/gpfs/hps3/emc/global/noscrub/emc.glopara/git/ens_tracker.${{ens_tracker_ver}} + elif [ $machine = "WCOSS_DELL_P3" ] ; then + export ens_tracker_ver=v1.1.13.1 + export HOMEens_tracker=/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/ens_tracker.${{ens_tracker_ver}} + elif [ $machine = "THEIA" ] ; then + export ens_tracker_ver=v2.0.1 + export HOMEens_tracker=/scratch4/NCEPDEV/global/save/glopara/svn/gfs/branches/gfs_q3fy17/ens_tracker.${{ens_tracker_ver}} + elif [ $machine = "JET" ]; then + export ens_tracker_ver=v1.1.13.2 + export HOMEens_tracker=/mnt/lfs3/projects/hfv3gfs/glopara/git/tracker/ens_tracker.${{ens_tracker_ver}} + else + echo "WARNING: Genesis and cyclone verifications are not enabled on $machine!" + export VRFYGENESIS="NO" + export VRFYTRAK="NO" + fi + + + if [ "$VRFYTRAK" = "YES" ]; then + + export TRACKERSH="$HOMEgfs/jobs/JGFS_CYCLONE_TRACKER" + if [ $CDUMP = "gdas" ]; then + export FHOUT_CYCLONE={doc.downstream.FHOUT_CYCLONE_GDAS} + export FHMAX_CYCLONE=$FHMAX + else + export FHOUT_CYCLONE={doc.downstream.FHOUT_CYCLONE_GFS} + export FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 )) + fi + if [ $machine = "THEIA" ]; then + export COMROOTp1="/scratch4/NCEPDEV/rstprod/com" + export COMINsyn="$COMROOTp1/arch/prod/syndat" + elif [ $machine = "JET" ]; then + export TRACKERSH="$HOMEens_tracker/jobs/JGFS_CYCLONE_TRACKER" + export COMINsyn="/mnt/lfs3/projects/hwrf-data/hwrf-input/SYNDAT" + fi + fi + + + if [[ "$VRFYGENESIS" == "YES" && "$CDUMP" == "gfs" ]]; then + + if [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" ] ; then + export GENESISSH="$HOMEgfs/jobs/JGFS_CYCLONE_GENESIS" + elif [ $machine = "THEIA" ] ; then + export GENESISSH="$HOMEens_tracker/gfs_genesis_para_fv3gfs.sh" + export UTILROOT="/scratch4/NCEPDEV/global/save/glopara/nwpara/prod_util.v1.0.15" + export GETTX_GEN="$NWROOTGENESIS/ens_tracker.v2.0.1/exec/gettrk_gen_g2_theia" + elif [ $machine = "JET" ] ; then + export GENESISSH="$HOMEens_tracker/jobs/JGFS_CYCLONE_GENESIS" + export UTILROOT="/lfs3/projects/hwrf-vd/soft/prod_util-1.0.18" + export GETTX_GEN="$HOMEens_tracker/exec/gettrk_gen_g2" + else + echo "WARNING: Genesis verification is not enabled on $machine!" + export VRFYGENESIS="NO" + export GENESISSH="" + fi + fi + + if [[ "$RUNMOS" == "YES" && "$CDUMP" == "gfs" ]]; then + + if [ $machine = "WCOSS_C" ] ; then + export RUNGFSMOSSH="$HOMEgfs/scripts/run_gfsmos_master.sh.cray" + elif [ $machine = "WCOSS_DELL_P3" ] ; then + export RUNGFSMOSSH="$HOMEgfs/scripts/run_gfsmos_master.sh.dell" + else + echo "WARNING: MOS package is not enabled on $machine!" + export RUNMOS="NO" + export RUNGFSMOSSH="" + fi + fi + + echo "END: config.vrfy" + diff --git a/workflow/config/wafs.yaml b/workflow/config/wafs.yaml new file mode 100644 index 00000000000..bc9633033ef --- /dev/null +++ b/workflow/config/wafs.yaml @@ -0,0 +1,38 @@ +# This file is used to generate config.wafs, which has settings for the GFS wafs step + +config_wafs: + filename: config.wafs + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.wafs ########## + # GFS wafs step specific + + echo "BEGIN: config.wafs" + + export SENDCOM="YES" + + export FIXgfs=$HOMEgfs/fix/wafs + export USHgfs=$HOMEgfs/gempak/ush + export PARMgfs=$HOMEgfs/parm/wafs + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMOUTwmo="$COMOUT/wmo" + export PCOM="$COMOUT/wmo" + + export DATAROOT="$RUNDIR/$CDATE/$CDUMP" + export DATA="$DATAROOT/$job" + + ########################################## + ## Source machine runtime environment + ########################################### + . $HOMEgfs/env/{doc.platform.name}.env wafs + status=$? + [[ $status -ne 0 ]] && exit $status + + echo "END: config.wafs" diff --git a/workflow/config/wafs_gcip.yaml b/workflow/config/wafs_gcip.yaml new file mode 100644 index 00000000000..09c9bdaf0cc --- /dev/null +++ b/workflow/config/wafs_gcip.yaml @@ -0,0 +1,33 @@ +# This file is used to generate config.wafs_gcip, which has settings for the GFS wafs_gcip step + +config_wafs_gcip: + filename: config.wafs_gcip + content: !expand | + #!/bin/ksh -x + + # This file is automatically generated from the YAML-based system + # in ecf/ecfutils/. Any changes will be overwritten if + # setup_case.sh is rerun. + + ########## config.wafs_gcip ########## + # GFS wafs_gcip step specific + + echo "BEGIN: config.wafs_gcip" + + export SENDCOM="YES" + + export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc" + export COMINgfs="$ROTDIR/gfs.$PDY/$cyc" + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc" + export TMPDIR=$STMP + export DATAROOT="$RUNDIR/$CDATE/$CDUMP" + export DATA="$DATAROOT/$job" + + ########################################## + ## Source machine runtime environment + ########################################### + . $HOMEgfs/env/{doc.platform.name}.env wafs + status=$? + [[ $status -ne 0 ]] && exit $status + + echo "END: config.wafs_gcip" diff --git a/workflow/defaults/case.yaml b/workflow/defaults/case.yaml new file mode 100644 index 00000000000..62ce0422923 --- /dev/null +++ b/workflow/defaults/case.yaml @@ -0,0 +1,145 @@ +# This file has the top-level logic to merge other YAML data +# structures into the document-level settings. It merges the contents +# of the case files, default files, platform file, and everywhere +# else, and applies any validation from the schema/ directory. + +fv3_settings: !Immediate + - !MergeMapping + - CDUMP: gfs + Template: *fv3_settings_template + - !calc doc.fv3_defaults + - !calc doc.case.get('fv3_settings',{}) + +ocn_settings: !Immediate + - !MergeMapping + - CDUMP: gfs + Template: *ocn_settings_template + - !calc doc.ocn_defaults + - !calc doc.case.get('ocn_settings',{}) + +ice_settings: !Immediate + - !MergeMapping + - CDUMP: gfs + Template: *ice_settings_template + - !calc doc.ice_defaults + - !calc doc.case.get('ice_settings',{}) + +schedvar: !Immediate + - !MergeMapping + - !calc doc.case.get('schedvar',{}) + - Template: *schedvar_schema + - !calc doc.get('schedvar_defaults',{}) + +fv3_gfs_settings: !Immediate + - !MergeMapping + - !calc doc.fv3_settings + - CDUMP: gfs + Template: *fv3_settings_template + - !calc doc.fv3_gfs_defaults + - !calc doc.case.get('fv3_gfs_settings',{}) + - !calc doc.get('user_fv3_gfs_settings',{}) + +fv3_enkf_settings: !Immediate + - !MergeMapping + - !calc doc.fv3_settings + - !calc doc.fv3_enkf_defaults + - CDUMP: gdas + Template: *fv3_settings_template + - !calc doc.case.get('fv3_enkf_settings',{}) + - !calc doc.get('user_fv3_enkf_settings',{}) + +fv3_gdas_settings: !Immediate + - !MergeMapping + - !calc doc.fv3_settings + - CDUMP: gdas + Template: *fv3_settings_template + - !calc doc.fv3_gdas_defaults + - !calc doc.case.get('fv3_gdas_settings',{}) + - !calc doc.get('user_fv3_gfs_settings',{}) + +gfs_output_settings: !Immediate + - !MergeMapping + - !calc doc.gfs_output_settings_defaults + - { Template: *gfs_output_settings_template } + - !calc doc.case.get('gfs_output_settings',{}) + - !calc doc.get('user_gfs_output_settings',{}) + +data_assimilation: !Immediate + - !MergeMapping + - { Template: *data_assimilation_template } + - !calc doc.case.get('data_assimilation',{}) + - !calc doc.get('user_data_assimilation',{}) + +post: !Immediate + - !MergeMapping + - !calc doc.post_defaults + - { Template: *post_schema } + - !calc doc.case.get('post',{}) + - !calc doc.get('user_post',{}) + +ocnpost: !Immediate + - !MergeMapping + - !calc doc.ocnpost_defaults + - { Template: *post_schema } + - !calc doc.case.get('ocnpost',{}) + - !calc doc.get('user_ocnpost',{}) + +downstream: !Immediate + - !MergeMapping + - !calc doc.downstream_defaults + - { Template: *downstream_schema } + - !calc doc.case.get('downstream',{}) + - !calc doc.get('user_downstream',{}) + +places: !Immediate + - !MergeMapping + - Template: *places_schema + - !calc doc.default_places + - !calc doc.case.get('places',{}) + - !calc doc.user_places + +nsst: !Immediate + - !MergeMapping + - Template: *nsst_schema + - !calc doc.case.get('nsst',{}) + - !calc doc.get('user_nsst',{}) + +exclusive_resources: !calc doc.platform.partitions.default_exclusive.resources +shared_resources: !calc doc.platform.partitions.default_shared.resources +service_resources: !calc doc.platform.partitions.default_service.resources + +settings: !Immediate + - !MergeMapping + - Template: *settings_schema + - !calc doc.default_settings + - !calc doc.case.settings + - !calc doc.get('user_settings',{}) + - Template: *settings_schema + +archiving: !Immediate + - !MergeMapping + - Template: *archive_settings_template + - !calc doc.case.get('archiving',{}) + - !calc doc.get('user_archiving',{}) + +suite_overrides: !Immediate + - !AppendSequence + - !calc doc.platform.get('suite_overrides',[]) + - !calc doc.case.get('suite_overrides',[]) + - !calc doc.get('user_suite_overrides',[]) + +validate_me: + - fv3_settings + - fv3_gfs_settings + - fv3_enkf_settings + - fv3_gdas_settings + - gfs_output_settings + - data_assimilation + - post + - downstream + - settings + - places + - nsst + - archiving + - schedvar + diff --git a/workflow/defaults/default_resources.yaml b/workflow/defaults/default_resources.yaml new file mode 100644 index 00000000000..e14f0cb296b --- /dev/null +++ b/workflow/defaults/default_resources.yaml @@ -0,0 +1,440 @@ +# This file specifies the resource requirements for all jobs. This +# includes jobs from the full DA-cycled workflow, as well as the jobs +# in the public release workflow. + +# Note on threads: +# max = use the largest number of threads possible for the platform, +# ranks, and processors per node +# null = do not specify threading settings. The underlying scripts +# and batch system will fill in settings. +# a number = use this many threads per MPI rank + +# Resource matrix; VALUES of default resource settings +gfs_resource_table: !Select + select: !icalc doc.fv3_gfs_settings.CASE + otherwise: !error "Unknown FV3 deterministic grid: {doc.fv3_gfs_settings.CASE}" + cases: + C192: + # ranks ppn wallclock threads MB_per_rank + prep: [ 12, 12, !timedelta "00:15:00", null, null ] + anal: [ 144, 6, !timedelta "01:30:00", max, null ] + gdaspost: [ 72, 12, !timedelta "00:30:00", 1, null ] + gfspost: [ 72, 12, !timedelta "00:10:00", 1, null ] + gdasvrfy: [ 1, 1, !timedelta "03:00:00", null, null ] + gfsvrfy: [ 5, 1, !timedelta "01:00:00", null, null ] + gdasfcst_mem_per_rank: 3.124e+3 + gfsfcst_mem_per_rank: 3.124e+3 + gdasfcst_wall: !timedelta "00:15:00" + gfsfcst_wall: !timedelta "01:00:00" + coupfcst_medcold_wall: !timedelta "00:30:00" + coupfcst_wall: !timedelta "02:00:00" + gdasfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + gfsfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + post_manager_wallclock_extra: !timedelta "00:15:00" + C384: + # ranks ppn wallclock threads MB_per_rank + prep: [ 12, 12, !timedelta "00:15:00", null, null ] + anal: [ 144, 6, !timedelta "02:30:00", max, null ] + gdaspost: [ 72, 12, !timedelta "00:30:00", 1, null ] + gfspost: [ 72, 12, !timedelta "00:10:00", 1, null ] + gdasvrfy: [ 1, 1, !timedelta "03:00:00", null, null ] + gfsvrfy: [ 5, 1, !timedelta "03:00:00", null, null ] + gdasfcst_wall: !timedelta "00:15:00" + gfsfcst_wall: !timedelta "06:00:00" + coupfcst_medcold_wall: !timedelta "00:30:00" + coupfcst_wall: !timedelta "02:00:00" + gdasfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + gfsfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + post_manager_wallclock_extra: !timedelta "00:15:00" + C768: + # ranks ppn wallclock threads MB_per_rank + prep: [ 4, 4, !timedelta "00:45:00", max, null ] + anal: [ 360, 6, !timedelta "05:30:00", max, 7000 ] + gdaspost: [ 84, 12, !timedelta "00:45:00", null, 3770 ] + gfspost: [ 84, 12, !timedelta "00:30:00", null, 3770 ] + gdasvrfy: [ 1, 1, !timedelta "03:00:00", null, null ] + gfsvrfy: [ 5, null, !timedelta "06:00:00", null, null ] + gdasfcst_mem_per_rank: 3.124e+3 + gfsfcst_mem_per_rank: 3.124e+3 + gdasfcst_wall: !timedelta "01:00:00" + gfsfcst_wall: !timedelta "06:00:00" + coupfcst_medcold_wall: !timedelta "00:30:00" + coupfcst_wall: !timedelta "02:00:00" + gdasfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + gfsfcst_ppn: !icalc doc.platform.partitions.default_exclusive.scheduler_settings.physical_cores_per_node + post_manager_wallclock_extra: !timedelta "00:15:00" + +enkf_resource_table: !Select + select: !icalc doc.fv3_enkf_settings.CASE + otherwise: !error "Unknown FV3 ENKF grid: {doc.fv3_enkf_settings.CASE}" + cases: + C192: + # ranks ppn wallclock threads MB_per_rank + ecen: [ 84, 12, !timedelta "00:30:00", 2, null ] + eobs: [ 72, 6, !timedelta "00:15:00", 4, null ] + eomg: [ 72, 6, !timedelta "00:30:00", 2, null ] + eupd: [ 120, 12, !timedelta "00:30:00", 4, null ] + epos: [ 84, 12, !timedelta "00:30:00", 2, null ] + efcs_wall: !timedelta "00:30:00" + efcs_ppn: 40 + C384: + # ranks ppn wallclock threads MB_per_rank + eobs: [ 140, 14, !timedelta "00:20:00", max, 4.2e+3 ] + eomg: [ 140, 14, !timedelta "00:45:00", max, 4.0e+3 ] + eupd: [ 360, 4, !timedelta "00:30:00", max, 1.5e+3 ] + ecen: [ 80, 4, !timedelta "01:00:00", max, 15.7e+3 ] + epos: [ 80, 4, !timedelta "02:00:00", max, 8.6e+3 ] + efcs_wall: !timedelta "00:45:00" + efcs_ppn: 40 + C768: + # ranks ppn wallclock threads MB_per_rank + eobs: [ 144, 12, !timedelta "00:30:00", 2, null ] + eomg: [ 144, 12, !timedelta "01:00:00", 2, null ] + eupd: [ 240, 4, !timedelta "00:30:00", 4, null ] + ecen: [ 80, 4, !timedelta "01:00:00", 2, null ] + epos: [ 80, 3, !timedelta "02:00:00", 2, null ] + efcs_wall: !timedelta "01:00:00" + efcs_ppn: 40 + +downstream_resource_table: + # ranks ppn wallclock threads MB_batch MB_per_rank + big_downstream: [ 2, 1, !timedelta "00:30:00", 1, 3072M, null ] + small_downstream: [ 2, 2, !timedelta "00:02:00", 1, 3072M, null ] + awips: [ 4, 4, !timedelta "02:00:00", 2, 254M, null ] + postsnd: [ 12, 3, !timedelta "04:00:00", max, 254M, null ] + postsndcfp: [ 10, 3, !timedelta "02:00:00", max, 254M, null ] + gempak: [ 20, 4, !timedelta "02:00:00", 3, 254M, null ] + verfrad: [ 1, 1, !timedelta "00:20:00", 1, 3072M, null ] + vminmon: [ 1, 1, !timedelta "00:10:00", 1, 3072M, null ] + gfs_gempak: [ 28, 4, !timedelta "03:00:00", 3, 3072M, null ] + gdas_gempak: [ 2, 1, !timedelta "00:30:00", 1, 3072M, null ] + gdas_gempak_meta_ncdc: [ 2, 1, !timedelta "00:30:00", 1, 3072M, null ] + one_node_downstream: [ 1, 1, !timedelta "00:30:00", 1, 3072M, null ] + big_downstream: [ 2, 1, !timedelta "00:30:00", 1, 3072M, null ] + small_downstream: [ 2, 2, !timedelta "00:02:00", 1, 3072M, null ] + +util_resource_table: + # ranks ppn wallclock threads MB_batch MB_per_rank + prepbufr: [ 4, 4, !timedelta "00:15:00", 1, 3072M, null ] + fv3ic: [ 24, 24, !timedelta "00:30:00", 1, 3072M, null ] + dump_waiter: [ 1, 1, !timedelta "01:00:00", 1, 300M, null ] + make_next_cycles: [ 1, 1, !timedelta "00:15:00", 1, 600M, null ] + one_hour_exclusive: [ 2, 1, !timedelta "04:00:00", 1, 300M, null ] + nothing: [ 1, 1, !timedelta "00:02:00", 1, 300M, null ] + getic: [ 1, 1, !timedelta "06:00:00", 1, 3072M, null ] + arch: [ 1, 1, !timedelta "06:00:00", 1, 3072M, null ] + earc: [ 1, 1, !timedelta "06:00:00", 1, 3072M, null ] + sfc_prep: [ 1, 1, !timedelta "00:02:00", 1, 1024M, null ] + final: [ 1, 1, !timedelta "00:01:00", 1, 1024M, null ] + dwn: [ 24, 24, !timedelta "00:02:00", 1, 3072M, null ] + +# !!! Model developers should NOT changing these !!! +# + +default_resources: &default_resources + + run_dwn: !JobRequest + - mpi_ranks: !icalc doc.util_resource_table.dwn[0] + OMP_NUM_THREADS: !icalc doc.util_resource_table.dwn[3] + + run_sfc_prep: !JobRequest + - batch_memory: !icalc doc.util_resource_table.sfc_prep[4] + mpi_ranks: !icalc doc.util_resource_table.sfc_prep[0] + walltime: !icalc doc.util_resource_table.sfc_prep[2] + max_ppn: !icalc doc.util_resource_table.sfc_prep[1] + + run_final: !JobRequest + - batch_memory: !icalc doc.util_resource_table.final[4] + mpi_ranks: !icalc doc.util_resource_table.final[0] + walltime: !icalc doc.util_resource_table.final[2] + max_ppn: !icalc doc.util_resource_table.final[1] + + run_getic: !JobRequest + - batch_memory: !icalc doc.util_resource_table.getic[4] + mpi_ranks: !icalc doc.util_resource_table.getic[0] + walltime: !icalc doc.util_resource_table.getic[2] + max_ppn: !icalc doc.util_resource_table.getic[1] + + run_arch: !JobRequest + - batch_memory: !icalc doc.util_resource_table.arch[4] + mpi_ranks: !icalc doc.util_resource_table.arch[0] + walltime: !icalc doc.util_resource_table.arch[2] + max_ppn: !icalc doc.util_resource_table.arch[1] + + run_earc: !JobRequest + - batch_memory: !icalc doc.util_resource_table.earc[4] + mpi_ranks: !icalc doc.util_resource_table.earc[0] + walltime: !icalc doc.util_resource_table.earc[2] + max_ppn: !icalc doc.util_resource_table.earc[1] + + run_make_next_cycles: !JobRequest + - batch_memory: !icalc doc.util_resource_table.make_next_cycles[4] + walltime: !icalc doc.util_resource_table.make_next_cycles[2] + + run_one_hour_exclusive: !JobRequest # Placeholder for one node jobs + - batch_memory: !icalc doc.util_resource_table.one_hour_exclusive[4] + mpi_ranks: !icalc doc.util_resource_table.one_hour_exclusive[0] + walltime: !icalc doc.util_resource_table.one_hour_exclusive[2] + + run_nothing: !JobRequest # Special placeholder for "do nothing" + - batch_memory: !icalc doc.util_resource_table.nothing[4] + walltime: !icalc doc.util_resource_table.nothing[2] + + run_prepbufr: !JobRequest # never used + - batch_memory: !icalc doc.util_resource_table.prepbufr[4] + walltime: !icalc doc.util_resource_table.prepbufr[2] + max_ppn: !icalc doc.util_resource_table.prepbufr[1] + mpi_ranks: !icalc doc.util_resource_table.prepbufr[0] + + run_fv3ic: !JobRequest + - batch_memory: !icalc doc.util_resource_table.fv3ic[4] + mpi_ranks: !icalc doc.util_resource_table.fv3ic[0] + max_ppn: !icalc doc.util_resource_table.fv3ic[1] + walltime: !icalc doc.util_resource_table.fv3ic[2] + + run_dump_waiter: !JobRequest + - batch_memory: !icalc doc.util_resource_table.dump_waiter[4] + walltime: !icalc doc.util_resource_table.dump_waiter[2] + mpi_ranks: !icalc doc.util_resource_table.dump_waiter[0] + max_ppn: !icalc doc.util_resource_table.dump_waiter[1] + + run_big_downstream: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.big_downstream[0] + max_ppn: !icalc doc.downstream_resource_table.big_downstream[1] + walltime: !icalc doc.downstream_resource_table.big_downstream[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.big_downstream[3] + batch_memory: !icalc doc.downstream_resource_table.big_downstream[4] + + run_small_downstream: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.small_downstream[0] + max_ppn: !icalc doc.downstream_resource_table.small_downstream[1] + walltime: !icalc doc.downstream_resource_table.small_downstream[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.small_downstream[3] + batch_memory: !icalc doc.downstream_resource_table.small_downstream[4] + + run_one_node_downstream: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.one_node_downstream[0] + max_ppn: !icalc doc.downstream_resource_table.one_node_downstream[1] + walltime: !icalc doc.downstream_resource_table.one_node_downstream[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.one_node_downstream[3] + batch_memory: !icalc doc.downstream_resource_table.one_node_downstream[4] + + run_gdas_gempak_meta_ncdc: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.gdas_gempak_meta_ncdc[0] + max_ppn: !icalc doc.downstream_resource_table.gdas_gempak_meta_ncdc[1] + walltime: !icalc doc.downstream_resource_table.gdas_gempak_meta_ncdc[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.gdas_gempak_meta_ncdc[3] + batch_memory: !icalc doc.downstream_resource_table.gdas_gempak_meta_ncdc[4] + + run_gfs_gempak: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.gfs_gempak[0] + max_ppn: !icalc doc.downstream_resource_table.gfs_gempak[1] + walltime: !icalc doc.downstream_resource_table.gfs_gempak[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.gfs_gempak[3] + batch_memory: !icalc doc.downstream_resource_table.gfs_gempak[4] + + run_gdas_gempak: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.gdas_gempak[0] + max_ppn: !icalc doc.downstream_resource_table.gdas_gempak[1] + walltime: !icalc doc.downstream_resource_table.gdas_gempak[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.gdas_gempak[3] + batch_memory: !icalc doc.downstream_resource_table.gdas_gempak[4] + + run_vminmon: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.vminmon[0] + max_ppn: !icalc doc.downstream_resource_table.vminmon[1] + walltime: !icalc doc.downstream_resource_table.vminmon[2] + batch_memory: !icalc doc.downstream_resource_table.vminmon[4] + + run_awips: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.awips[0] + max_ppn: !icalc doc.downstream_resource_table.awips[1] + walltime: !icalc doc.downstream_resource_table.awips[2] + batch_memory: !icalc doc.downstream_resource_table.awips[4] + + run_postsnd: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.postsnd[0] + max_ppn: !icalc doc.downstream_resource_table.postsnd[1] + walltime: !icalc doc.downstream_resource_table.postsnd[2] + batch_memory: !icalc doc.downstream_resource_table.postsnd[4] + + run_postsndcfp: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.postsndcfp[0] + max_ppn: !icalc doc.downstream_resource_table.postsndcfp[1] + walltime: !icalc doc.downstream_resource_table.postsndcfp[2] + batch_memory: !icalc doc.downstream_resource_table.postsndcfp[4] + + run_gempak: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.gempak[0] + max_ppn: !icalc doc.downstream_resource_table.gempak[1] + walltime: !icalc doc.downstream_resource_table.gempak[2] + OMP_NUM_THREADS: !icalc doc.downstream_resource_table.gempak[3] + batch_memory: !icalc doc.downstream_resource_table.gempak[4] + + run_verfrad: !JobRequest + - mpi_ranks: !icalc doc.downstream_resource_table.verfrad[0] + max_ppn: !icalc doc.downstream_resource_table.verfrad[1] + walltime: !icalc doc.downstream_resource_table.verfrad[2] + batch_memory: !icalc doc.downstream_resource_table.verfrad[4] + + # Calculated resources; ones that can be determined entirely from + # other variables throughout the document. + + run_gdasfcst: !JobRequest + - batch_memory: "1024M" + mpi_ranks: !icalc doc.fv3_gdas_settings.ATMPETS + max_ppn: !icalc doc.gfs_resource_table.gdasfcst_ppn + walltime: !icalc doc.gfs_resource_table.gdasfcst_wall + + remap_resource_template: &remap_resource_template + mpi_ranks: !calc >- + min(240,resources.total_ranks()) + OMP_NUM_THREADS: 2 + max_ppn: !calc partition.nodes.max_ranks_per_node(resources[0]) + + run_efcs: !JobRequest + - batch_memory: "254M" + mpi_ranks: !icalc doc.fv3_enkf_settings.ATMPETS + max_ppn: !icalc doc.enkf_resource_table.efcs_ppn + walltime: !icalc doc.enkf_resource_table.efcs_wall + + run_gfsfcst: !JobRequest + - batch_memory: "1024M" + mpi_ranks: !icalc doc.fv3_gfs_settings.ATMPETS + max_ppn: !icalc doc.gfs_resource_table.gfsfcst_ppn + walltime: !icalc doc.gfs_resource_table.gfsfcst_wall + + run_coupled_fcst: !JobRequest + - batch_memory: "1024M" + mpi_ranks: !icalc doc.fv3_gfs_settings.ATMPETS+doc.fv3_gfs_settings.OCNPETS+doc.fv3_gfs_settings.ICEPETS + max_ppn: !icalc doc.fv3_gfs_settings.get('fcst_max_ppn',None) + OMP_NUM_THREADS: !icalc doc.fv3_gfs_settings.fv3_threads + walltime: !icalc doc.gfs_resource_table.coupfcst_wall + memory_per_rank: !icalc doc.gfs_resource_table.get('gfsfcst_mem_per_rank',None) + + run_coupled_medcold: !JobRequest + - batch_memory: "1024M" + mpi_ranks: !icalc doc.fv3_gfs_settings.ATMPETS+doc.fv3_gfs_settings.OCNPETS+doc.fv3_gfs_settings.ICEPETS + max_ppn: !icalc doc.fv3_gfs_settings.get('fcst_max_ppn',None) + OMP_NUM_THREADS: !icalc doc.fv3_gfs_settings.fv3_threads + walltime: !icalc doc.gfs_resource_table.coupfcst_medcold_wall + memory_per_rank: !icalc doc.gfs_resource_table.get('gfsfcst_mem_per_rank',None) + + fallback_run_gfsremap: !JobRequest + # Used to generate the config files if the gfs remap is not run in the workflow. + - mpi_ranks: !calc >- + min(240,doc.exclusive_resources.run_gfsfcst.total_ranks()) + OMP_NUM_THREADS: 2 + max_ppn: !calc >- + doc.platform.partitions.default_exclusive.nodes.max_ranks_per_node( + doc.exclusive_resources.run_gfsfcst[0]) + + run_gdas_post_manager: !JobRequest + - memory: "300M" + exe: placeholder + walltime: !icalc doc.gfs_resource_table.gdasfcst_wall+doc.gfs_resource_table.post_manager_wallclock_extra + + run_gfs_post_manager: !JobRequest + - memory: "300M" + exe: placeholder + walltime: !icalc doc.gfs_resource_table.gfsfcst_wall+doc.gfs_resource_table.post_manager_wallclock_extra + + run_ecen: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.enkf_resource_table.ecen[0] + max_ppn: !icalc doc.enkf_resource_table.ecen[1] + walltime: !icalc doc.enkf_resource_table.ecen[2] + OMP_NUM_THREADS: !icalc doc.enkf_resource_table.ecen[3] + memory_per_rank: !icalc doc.enkf_resource_table.ecen[4] + + run_eobs: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.enkf_resource_table.eobs[0] + max_ppn: !icalc doc.enkf_resource_table.eobs[1] + walltime: !icalc doc.enkf_resource_table.eobs[2] + OMP_NUM_THREADS: !icalc doc.enkf_resource_table.eobs[3] + memory_per_rank: !icalc doc.enkf_resource_table.eobs[4] + + run_eomg: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.enkf_resource_table.eomg[0] + max_ppn: !icalc doc.enkf_resource_table.eomg[1] + walltime: !icalc doc.enkf_resource_table.eomg[2] + OMP_NUM_THREADS: !icalc doc.enkf_resource_table.eomg[3] + memory_per_rank: !icalc doc.enkf_resource_table.eomg[4] + + run_eupd: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.enkf_resource_table.eupd[0] + max_ppn: !icalc doc.enkf_resource_table.eupd[1] + walltime: !icalc doc.enkf_resource_table.eupd[2] + OMP_NUM_THREADS: !icalc doc.enkf_resource_table.eupd[3] + memory_per_rank: !icalc doc.enkf_resource_table.eupd[4] + + run_epos: !JobRequest + - batch_memory: "254M" + exe: placeholder + mpi_ranks: !icalc doc.enkf_resource_table.epos[0] + max_ppn: !icalc doc.enkf_resource_table.epos[1] + walltime: !icalc doc.enkf_resource_table.epos[2] + OMP_NUM_THREADS: !icalc doc.enkf_resource_table.epos[3] + memory_per_rank: !icalc doc.enkf_resource_table.epos[4] + + run_prep: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.prep[0] + max_ppn: !icalc doc.gfs_resource_table.prep[1] + walltime: !icalc doc.gfs_resource_table.prep[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.prep[3] + memory_per_rank: !icalc doc.gfs_resource_table.prep[4] + + run_anal: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.anal[0] + max_ppn: !icalc doc.gfs_resource_table.anal[1] + walltime: !icalc doc.gfs_resource_table.anal[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.anal[3] + memory_per_rank: !icalc doc.gfs_resource_table.anal[4] + + run_gdaspost: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.gdaspost[0] + max_ppn: !icalc doc.gfs_resource_table.gdaspost[1] + walltime: !icalc doc.gfs_resource_table.gdaspost[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.gdaspost[3] + memory_per_rank: !icalc doc.gfs_resource_table.gdaspost[4] + + run_gfspost: !JobRequest + - batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.gfspost[0] + max_ppn: !icalc doc.gfs_resource_table.gfspost[1] + walltime: !icalc doc.gfs_resource_table.gfspost[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.gfspost[3] + memory_per_rank: !icalc doc.gfs_resource_table.gfspost[4] + + run_gfsvrfy: !JobRequest + - compute_memory: "16384M" + batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.gfsvrfy[0] + max_ppn: !icalc doc.gfs_resource_table.gfsvrfy[1] + walltime: !icalc doc.gfs_resource_table.gfsvrfy[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.gfsvrfy[3] + + run_gdasvrfy: !JobRequest + - compute_memory: "16384M" + batch_memory: "3072M" + exe: placeholder + mpi_ranks: !icalc doc.gfs_resource_table.gdasvrfy[0] + max_ppn: !icalc doc.gfs_resource_table.gdasvrfy[1] + walltime: !icalc doc.gfs_resource_table.gdasvrfy[2] + OMP_NUM_THREADS: !icalc doc.gfs_resource_table.gdasvrfy[3] + diff --git a/workflow/defaults/downstream.yaml b/workflow/defaults/downstream.yaml new file mode 100644 index 00000000000..d9ba91a7ebf --- /dev/null +++ b/workflow/defaults/downstream.yaml @@ -0,0 +1,20 @@ +downstream_defaults: &downstream_defaults + DO_POST_PROCESSING: NO + DO_BUFRSND: NO + DO_GEMPAK: NO + DO_AWIPS: NO + DO_FAX: NO + DO_WAFS: NO + DO_BULLETINS: NO + VRFYFITS: NO + VSDB_STEP1: NO + VSDB_STEP2: NO + VRFYG2OBS: NO + VRFYPRCP: NO + VRFYRAD: NO + VRFYOZN: NO + VRFYMINMON: NO + VRFYTRAK: NO + VRFYGENESIS: NO + RUNMOS: NO + RMOLDSTD: 144 diff --git a/workflow/defaults/fv3.yaml b/workflow/defaults/fv3.yaml new file mode 100644 index 00000000000..44ac888ed2c --- /dev/null +++ b/workflow/defaults/fv3.yaml @@ -0,0 +1,13 @@ +# default settings for fv3 core + +fv3_defaults: &fv3_defaults + CASE: C384 + LEVS: 65 + layout_x: 6 + layout_y: 6 + fv3_threads: 2 + WRITE_GROUP: 1 + WRTTASK_PER_GROUP: 24 + WRTIOBUF: "32M" + OCNPETS: 120 + ICEPETS: 48 diff --git a/workflow/defaults/fv3_enkf.yaml b/workflow/defaults/fv3_enkf.yaml new file mode 100644 index 00000000000..85e53a6fe81 --- /dev/null +++ b/workflow/defaults/fv3_enkf.yaml @@ -0,0 +1,53 @@ +# This file is used to configure the ENKF-related config files. The +# ENKF is not included in this release, so these settings are not +# needed. However, they do need to be set to valid values because +# there is validation of the fv3_enkf_settings variables elsewhere in +# the YAML documents. + +fv3_enkf_defaults: &fv3_enkf_defaults + QUILTING: True + layout_x: 4 + layout_y: 6 + ATMPETS: !FirstTrue + - when: !calc doc.fv3_enkf_settings.QUILTING + do: !calc layout_x * layout_y * 6 + WRITE_GROUP * WRTTASK_PER_GROUP + - otherwise: !calc layout_x * layout_y * 6 + + FHSWR: 3600. + FHLWR: 3600. + IEMS: 1 + ISOL: 2 + IAER: 111 + ICO2: 2 + dspheat: YES + shal_cnv: NO + FHZER: 6 + FHCYC: 1 + restart_interval: 6 + FHMAX: 6 + +# Stochastic physics parameters (only for ensemble forecasts) + DO_SKEB: YES + SKEB: 0.3 + SKEB_TAU: 21600. + SKEB_LSCALE: 250000. + SKEBNORM: 0 + SKEB_NPASS: 30 + SKEB_VDOF: 5 + DO_SHUM: YES + SHUM: 0.005 + SHUM_TAU: 21600. + SHUM_LSCALE: 500000. + DO_SPPT: YES + DOIAU_ENKF: NO + IAUFHRS_ENKF: 6 + IAU_DELTHRS_ENKF: 6 + SPPT: 0.5 + SPPT_TAU: 21600. + SPPT_LSCALE: 500000. + SPPT_LOGIT: YES + SPPT_SFCLIMIT: YES + + k_split: 1 + n_split: 12 + consv_te: 1. diff --git a/workflow/defaults/fv3_gdas.yaml b/workflow/defaults/fv3_gdas.yaml new file mode 100644 index 00000000000..5cf5c4ad9bd --- /dev/null +++ b/workflow/defaults/fv3_gdas.yaml @@ -0,0 +1,8 @@ +fv3_gdas_defaults: &fv3_gdas_defaults + QUILTING: True + hord_mt_nh_nonmono: 5 + hord_xx_nh_nonmono: 5 + ATMPETS: !FirstTrue + - when: !calc doc.fv3_gdas_settings.QUILTING + do: !calc layout_x * layout_y * 6 + WRITE_GROUP * WRTTASK_PER_GROUP + - otherwise: !calc layout_x * layout_y * 6 diff --git a/workflow/defaults/fv3_gfs.yaml b/workflow/defaults/fv3_gfs.yaml new file mode 100644 index 00000000000..e29cd925978 --- /dev/null +++ b/workflow/defaults/fv3_gfs.yaml @@ -0,0 +1,23 @@ +fv3_gfs_defaults: &fv3_gfs_defaults + QUILTING: True + ATMPETS: !FirstTrue + - when: !calc doc.fv3_gfs_settings.QUILTING + do: !calc layout_x * layout_y * 6 + WRITE_GROUP * WRTTASK_PER_GROUP + - otherwise: !calc layout_x * layout_y * 6 + cdmbgwd: !FirstTrue + - when: !calc CASE=="C48" + do: "0.071,2.1,1.0,1.0" + - when: !calc CASE=="C96" + do: "0.14,1.8,1.0,1.0" + - when: !calc CASE=="C192" + do: "0.23,1.5,1.0,1.0" + - when: !calc CASE=="C384" + do: "1.1,0.72,1.0,1.0" + - when: !calc CASE=="C768" + do: "4.0,0.15,1.0,1.0" + - when: !calc CASE=="C1152" + do: "4.0,0.10,1.0,1.0" + - when: !calc CASE=="C3072" + do: "4.0,0.05,1.0,1.0" + - otherwise: "" + message: "Newly defined CASE, please make sure cdmbgwd is set correctly in case file under fv3_gfs_settings!" diff --git a/workflow/defaults/gfs_output_settings.yaml b/workflow/defaults/gfs_output_settings.yaml new file mode 100644 index 00000000000..7b04d97dc28 --- /dev/null +++ b/workflow/defaults/gfs_output_settings.yaml @@ -0,0 +1,7 @@ +# This file sets default values for the GFS output settings. See +# schema/output.yaml for documentation. + +gfs_output_settings_defaults: &gfs_output_settings_defaults + FHMAX_GFS: 384 + FHOUT_GFS: 3 + OUTPUT_GRID: gaussian_grid diff --git a/workflow/defaults/ice.yaml b/workflow/defaults/ice.yaml new file mode 100644 index 00000000000..f885d70ce89 --- /dev/null +++ b/workflow/defaults/ice.yaml @@ -0,0 +1,5 @@ +# default settings for fv3 core + +ice_defaults: &ice_defaults + MOD: cice5 + ICEPETS: 48 diff --git a/workflow/defaults/ocn.yaml b/workflow/defaults/ocn.yaml new file mode 100644 index 00000000000..5ccb468d527 --- /dev/null +++ b/workflow/defaults/ocn.yaml @@ -0,0 +1,5 @@ +# default settings for ocean model core + +ocn_defaults: &ocn_defaults + MOD: mom6 + OCNPETS: 120 diff --git a/workflow/defaults/places.yaml b/workflow/defaults/places.yaml new file mode 100644 index 00000000000..9d0dcd1b7a7 --- /dev/null +++ b/workflow/defaults/places.yaml @@ -0,0 +1,31 @@ +# This file sets default values for data and executable locations. +# See the schema/places.yaml for details. +default_places: &default_places + EXPROOT: !error Please select a EXPROOT directory. + HOMEgfs: !calc ( tools.realpath(tools.abspath("../../")) ) + HOMEcrow: !calc tools.crow_install_dir() + HOMEobsproc_prep: !expand "{doc.platform.BASE_GIT}/obsproc/obsproc_global.v3.2.3" + HOMEobsproc_network: !expand "{doc.platform.BASE_GIT}/obsproc/obsproc_global.v3.2.3" + + NWPROD: !calc doc.platform.NWPROD + DMPDIR: !calc doc.platform.DMPDIR + RTMFIX: !calc doc.platform.RTMFIX + + EXPDIR: !expand "{doc.platform.EXPROOT}/{doc.names.experiment}" + ROTDIR: !expand "{doc.platform.COMROOT}/{doc.names.experiment}" + ICSDIR: !expand "{ROTDIR}/FV3ICS" + MOM6IC: /scratch4/NCEPDEV/nems/noscrub/Bin.Li/FROM_HPSS + DATAROOT: !expand "{doc.platform.DATAROOT}/{doc.names.experiment}" +# SHORT_TERM_TEMP: !calc doc.platform.short_term_temp +# LONG_TERM_TEMP: !calc doc.platform.long_term_temp +# SHORT_TERM_TEMP: !error GOT HERE +# LONG_TERM_TEMP: !error GOT HERE TOO + + ECF_ROOT: !calc tools.env("ECF_ROOT","/dev/null/ECF_ROOT") + ECF_HOME: !calc tools.env("ECF_HOME","/dev/null/ECF_HOME") + ecflow_def_dir: !expand '{ECF_ROOT}/defs' + ECF_INCLUDE: !expand '{ECF_ROOT}/include' + ECF_OUT: !expand '{ECF_ROOT}/output' + + YAMLDIR: !calc EXPDIR + rocoto_workflow_xml: !expand '{YAMLDIR}/workflow.xml' diff --git a/workflow/defaults/post.yaml b/workflow/defaults/post.yaml new file mode 100644 index 00000000000..4bb05e63b07 --- /dev/null +++ b/workflow/defaults/post.yaml @@ -0,0 +1,7 @@ +# default settings for post jobs + +post_defaults: &post_defaults + NPOSTGRP: 5 + +ocnpost_defaults: &ocnpost_defaults + NPOSTGRP: 1 diff --git a/workflow/defaults/settings.yaml b/workflow/defaults/settings.yaml new file mode 100644 index 00000000000..b09e8b86ee2 --- /dev/null +++ b/workflow/defaults/settings.yaml @@ -0,0 +1,79 @@ +# Do not change this file unless you know what you're doing. It sets +# default values for the "settings" section. You probably want to +# override the settings in the case file instead. + +default_settings: &default_settings + ecf_log_path: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + # in four cycle mode ,the log name is the last path component, + # with the leading "j" removed. + do: !calc ( '%COM%/output/%ENVIR%/today/' + task_path_list[-1][1:] + '_%CYC%.o%J' ) + - otherwise: !calc ( '%COM%/logs/%PDY%%CYC%/'+ task_path_var + '.log.%J') + + # rocoto_log_path: sets the stdout/stderr log path for batch jobs + # when using Rocoto. This is inserted in the tag of the + # task's tag. + rocoto_log_path: !calc ("&LOG_DIR;/@Y@m@d@H/"+task_path_var+".log") + rocoto_task_throttle: 16 + rocoto_cycle_throttle: 2 + + max_job_tries: 2 + + # REALTIME and realtime - an alias that should be removed once we're + # sure it is consistent among the yaml files: + realtime: NO + + four_cycle_mode: NO + nco_mode: NO + use_nco_ecflow_headers: NO + print_esmf: .false. + + gfs_cyc: 4 + nems_temp: 'atm' + nems_temp_cold: 'atm' + + cplflx: .false. + cplice: !calc cplflx + cpl: .false. + + DOIAU: NO + IAUFHRS: 6 + IAU_DELTHRS: 6 + + KEEPDATA: NO + DUMP_SUFFIX: "" + run_gsi: No + chgres_and_convert_ics: Yes + + # awips setting + NAWIPSGRP: 21 + + mkdir_before_running_ecflow: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: null + - otherwise: !expand '{doc.places.ROTDIR}/logs/%Y%m%d%H' + + ecflow_module: !calc doc.platform.get("ecflow_module","ecflow") + + ecflow_real_clock: no + ecflow_virtual_clock: no + ecflow_hybrid_clock: no + ecflow_totality_limit: yes + + ecflow_rocoto_cdate_workaround: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: | + # When sourcing config files, the $PDY must be set: + datedir=/tmp/date.$$.$RANDOM + mkdir -p "$datedir" + pushd "$datedir" + export cyc="%CYC%" + export cycle="t%CYC%z" + setpdy.sh + . ./PDY + set -u + export CDATE="$PDY$cyc" + set +u + popd + rm -rf "$datedir" + - otherwise: '# date variables will be set by header file.' diff --git a/workflow/layout/cycled_gfs.yaml b/workflow/layout/cycled_gfs.yaml new file mode 100644 index 00000000000..f7cf895216d --- /dev/null +++ b/workflow/layout/cycled_gfs.yaml @@ -0,0 +1,776 @@ +suite: !Cycle + <<: *suite_defaults + + Alarms: + <<: *cycled_suite_alarms + + ###################################################################### + ## GDAS FAMILY ####################################################### + ###################################################################### + + gdas: !Family + RUN: gdas + ecflow_def: "edit RUN 'gdas'" + + jgdas_emc_dump_waiter: !Task + <<: *dump_waiter_task + Disable: !calc metasched.type != 'ecflow' + + #Time: !FirstTrue + # - when: !calc doc.settings.four_cycle_mode + # do: !timedelta +5:50:00 + # - otherwise: null + + realtime_logic: !FirstTrue + - when: !calc doc.settings.realtime + do: "edit ECF_TRIES 72" + - otherwise: "" + ecflow_def: !expand | + {realtime_logic} + edit DUMPDIR '{doc.places.DMPDIR}' + Trigger: !Depend forecast.at('-6:00:00') + + dump: !Family + ecflow_def: "# NCO will need to add NCO dump job here" + Dummy: true ## <-- Node is not implemented yet + Trigger: !Depend forecast.at('-6:00:00') + jgdas_ics: !Task + <<: *exclusive_task_template + release_gdas00_ics: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + + #jgdas_dump_post: !Task + # Trigger: !Depend jgdas_dump + # release_sfcprep: !DataEvent {file="/dev/null"} + # release_gdas00_dump_alert: !DataEvent {file="/dev/null"} + # ecf_file: *ecf_file_template + # resources: !calc partition.resources.run_nothing + # accounting: *exclusive_accounting + # J_JOB: nothing + + jgdas_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Trigger: !Depend jgdas_dump + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:45:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + + #Replaced by emc version of dump job + #This dump job should be using NCO version when delivery to NCO + jgdas_dump: !Task + <<: *exclusive_task_template + release_sfcprep: !DataEvent {file="/dev/null"} + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:50:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + #endfamily dump + + prep: !Family + Trigger: !Depend | + (jgdas_emc_dump_waiter.updated_status | dump.jgdas_dump.release_sfcprep ) & up.gdas.forecast.at('-6:00:00') + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + jgdas_emcsfc_sfc_prep: !Task + <<: *exclusive_task_template + Disable: !calc not doc.data_assimilation.DO_EMCSFC + Trigger: !Depend 'up.dump.jgdas_dump.release_sfcprep | up.jgdas_emc_dump_waiter.updated_status' + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + resources: !calc partition.resources.run_prep + config_list: [ base, prep, prepbufr, emcsfc ] + J_JOB: JGLOBAL_EMCSFC_SFC_PREP + + jgdas_prep: !Task + <<: *exclusive_task_template + Trigger: !Depend ( ( up.dump.jgdas_dump & up.jgdas_emc_dump_waiter ) & up.up.gdas.post.at('-6:00:00') ) + ecflow_def: + edit model 'obsproc_global' + resources: !calc partition.resources.run_prep + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/prep.sh +# model: obsproc_global # for four cycle mode +# J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + jgdas_tropcy_cp: !Task + <<: *exclusive_task_template + Dummy: True + Trigger: !Depend prep.jgdas_prep + resources: !calc partition.resources.run_one_hour_exclusive + J_JOB: rocoto/tropcy_cp.sh + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + + analysis: !Family + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + jgdas_analysis_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & up.prep.jgdas_emcsfc_sfc_prep & up.enkf.post.at('-6:00:00') ) + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + + forecast: !Family + jgdas_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.analysis.jgdas_analysis_high & up.enkf.innovate ) | ~ suite.has_cycle('-6:00:00') + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gdasfcst + J_JOB: JGLOBAL_FORECAST + + post_processing: !Family + Disable: !calc not doc.downstream.DO_POST_PROCESSING + Complete: !Depend ~ suite.has_cycle('-6:00:00') + bulletins: !Family + Disable: !calc not doc.downstream.DO_BULLETINS + jgdas_bulls_navy: !Task + <<: *exclusive_task_template + Dummy: True # job is broken + AlarmName: gfs_00_12 + Trigger: !Depend ( up.up.prep.jgdas_prep ) + resources: !calc partition.resources.run_big_downstream + J_JOB: JGDAS_BULLS_NAVY + + gempak: !Family + Disable: !calc not doc.downstream.DO_GEMPAK + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgdas_gempak: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gdas_gempak + J_JOB: JGDAS_GEMPAK + Trigger: !Depend '( up.post if doc.settings.dev_safeguards else up.forecast.jgdas_forecast_high )' + + jgdas_gempak_meta_ncdc: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gdas_gempak_meta_ncdc + Trigger: !Depend jgdas_gempak + J_JOB: JGDAS_GEMPAK_META_NCDC + + #endfamily gempak + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.gfs_output_settings.gdas_forecast_hours + post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend ( up.forecast.is_running() ) + - otherwise: !Depend "up.forecast.is_running() | up.forecast" + Disable: !calc metasched.type=='rocoto' + Foreach: [] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgdas_post_manager + resources: !calc partition.resources.run_gdas_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand > + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match the logic in scripts/exglobal_pmgr.sh.ecf + + # The post_anl has to be a !TaskElement to ensure it shows up between + # the post manager and forecast hour posts: + jgdas_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: 'anl' + Name: jgdas_post_anl + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgdas_post_manager.release_postanl + - otherwise: !Depend jgdas_post_manager.release_postanl | up.forecast + release_pgrb2_anl: !DataEvent {file="/dev/null"} + config_list: [ base, post ] + resources: !calc partition.resources.run_gdaspost + ecflow_def: !expand "edit FHR '{FHR}'" + J_JOB: rocoto/post.sh + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgdas_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gdaspost + Name: !expand jgdas_post_f{dimval.fhr:03d} + FHR: !expand '{dimval.fhr:03d}' + config_list: [ base, post ] + J_JOB: rocoto/post.sh + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgdas_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + - otherwise: !Depend jgdas_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) | up.forecast + #endfamily post + + jgdas_emc_vrfy: !Task + <<: *exclusive_task_template + Disable: !calc not doc.settings.run_vrfy_jobs + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + Trigger: !Depend post + resources: !calc partition.resources.run_gdasvrfy + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/vrfy.sh + ecf_module_commands: "# vrfy.sh will load modules instead" + + enkf: !Family + jgdas_enkf_select_obs: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & post.at('-6:00:00') & up.jgdas_tropcy_cp ) + resources: !calc partition.resources.run_eobs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + J_JOB: JGLOBAL_ENKF_SELECT_OBS + + innovate: !TaskArray + Trigger: !Depend jgdas_enkf_select_obs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: !calc tools.seq(1,doc.data_assimilation.ENKF_INNOVATE_GROUPS,1) + jgdas_enkf_innovate_obs_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_innovate_obs_grp{dimval.groupid}" + resources: !calc partition.resources.run_eomg + J_JOB: JGLOBAL_ENKF_INNOVATE_OBS + ENSGRP: !expand "{dimval.groupid:02d}" + # Lin wants ENSGRP to be outside the ecflow suite definition: + #rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB} + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + + jgdas_enkf_update: !Task + <<: *exclusive_task_template + Trigger: !Depend innovate + ecflow_def: "edit ECF_PASS 'FREE'" + Complete: !Depend ~ suite.has_cycle('-6:00:00') + resources: !calc partition.resources.run_eupd + J_JOB: JGLOBAL_ENKF_UPDATE + + jgdas_enkf_inflate_recenter: !Task + <<: *exclusive_task_template + Trigger: !Depend ( jgdas_enkf_update & up.analysis.jgdas_analysis_high ) + resources: !calc partition.resources.run_ecen + J_JOB: JGDAS_ENKF_RECENTER + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !TaskArray + + Trigger: !Depend up.enkf.jgdas_enkf_inflate_recenter | ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: !calc tools.seq(1,doc.data_assimilation.ENKF_FORECAST_GROUPS,1) + jgdas_enkf_fcst_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_fcst_grp{dimval.groupid}" + resources: !calc partition.resources.run_efcs + J_JOB: JGDAS_ENKF_FCST + ENSGRP: !expand "{dimval.groupid:02d}" + + # Lin wants ENSGRP to be outside the ecflow suite definition: + #rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB} + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + +# Create epos task array +# export FHRLST="f003 f004 f005 f006 f007 f008 f009" +# fhrlst=$(echo $FHRLST | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +# FHMIN_EPOS=$fhr +# FHMAX_EPOS=$fhr +# FHOUT_EPOS=$fhr +# job=epos${fhr} + post: !TaskArray + Trigger: !Depend forecast + Dimensions: + fhr: !calc doc.gfs_output_settings.enkf_epos_fhr + jgdas_enkf_post_fhr: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgdas_enkf_post_f{dimval.fhr:03d}" + FHMIN_EPOS: !expand '{dimval.fhr:03d}' + FHMAX_EPOS: !expand '{dimval.fhr:03d}' + FHOUT_EPOS: !expand '{dimval.fhr:03d}' + ecflow_command: !expand | + export FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + /usr/bin/env FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} &HOMEgfs;/jobs/{J_JOB} + resources: !calc partition.resources.run_epos + J_JOB: JGDAS_ENKF_POST + + #endfamily enkf + #endfamily gdas + + ###################################################################### + ## GFS FAMILY ######################################################## + ###################################################################### + + gfs: !Family + Disable: !calc doc.settings.gfs_cyc==0 + Complete: !Depend ~ suite.has_cycle('-6:00:00') + AlarmName: gfs + RUN: 'gfs' + ecflow_def: "edit RUN 'gfs'" + + jgfs_emc_dump_waiter: !Task + <<: *dump_waiter_task + Disable: !calc metasched.type != 'ecflow' + realtime_logic: !FirstTrue + - when: !calc doc.settings.realtime + do: "edit ECF_TRIES 72" + - otherwise: "" + #Time: !FirstTrue + # - when: !calc doc.settings.four_cycle_mode + # do: !timedelta +2:47:00 + # - otherwise: null + ecflow_def: !expand | + {realtime_logic} + edit DUMPDIR '{doc.places.DMPDIR}' + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + + dump: !Family + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + Complete: !Depend ~ suite.has_cycle('-6:00:00') + Dummy: true ## <-- Node is not implemented yet + ecflow_def: "# NCO will need to add NCO dump job here" + jgfs_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_dump + jtwc_bull_email: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:41:00 + - otherwise: null + + jgfs_dump: !Task + <<: *exclusive_task_template + ecflow_def: '#NCO needs to replace this with the real dump job' + release_sfcprep: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:47:00 + - otherwise: null + + prep: !Family + Trigger: !Depend ( ( jgfs_emc_dump_waiter.updated_status | dump.jgfs_dump.release_sfcprep ) & up.gdas.forecast.at('-6:00:00') ) + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_emcsfc_sfc_prep: !Task + <<: *exclusive_task_template + Disable: !calc not doc.data_assimilation.DO_EMCSFC + Trigger: !Depend 'up.dump.jgfs_dump.release_sfcprep | up.jgfs_emc_dump_waiter.updated_status' + resources: !calc partition.resources.run_sfc_prep + J_JOB: JGLOBAL_EMCSFC_SFC_PREP + ecf_environment_settings: !expand | + {doc.settings.ecflow_rocoto_cdate_workaround} + export DATAROOT="$DATAROOT.$job" + config_list: [ base, prep, prepbufr, emcsfc ] + + jgfs_prep: !Task + <<: *exclusive_task_template + ecflow_def: + edit model 'obsproc_global' + Trigger: !Depend up.dump + resources: !calc partition.resources.run_prep + ecf_environment_settings: !expand | + {doc.settings.ecflow_rocoto_cdate_workaround} + export DATAROOT="$DATAROOT.$job" + J_JOB: rocoto/prep.sh +# model: obsproc_global # for four cycle mode +# J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + jgfs_tropcy_cp: !Task + <<: *exclusive_task_template + Dummy: True + Trigger: !Depend prep.jgfs_prep + resources: !calc partition.resources.run_one_hour_exclusive + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/tropcy_cp.sh + + jgfs_analysis: !Task + <<: *exclusive_task_template + Trigger: !Depend ( prep.jgfs_prep & prep.jgfs_emcsfc_sfc_prep & up.gdas.enkf.post.at('-6:00:00') ) + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !Family + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfsfcst + J_JOB: JGLOBAL_FORECAST + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.gfs_output_settings.gfs_forecast_hours + jgfs_post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Disable: !calc metasched.type == 'rocoto' + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend "up.forecast.is_running()" + - otherwise: !Depend "up.forecast.is_running() | up.forecast" + Foreach: [ ] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgfs_post_manager + resources: !calc partition.resources.run_gfs_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match post_manager.yaml + + jgfs_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: anl + Name: jgfs_post_anl + Trigger: !FirstTrue + - when: !calc doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.release_postanl + - otherwise: !Depend jgfs_post_manager.release_postanl | up.forecast + + ecflow_def: !expand "edit FHR '{FHR}'" + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gfspost + config_list: [ base, post ] + J_JOB: rocoto/post.sh + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + /usr/bin/env post_times=anl &HOMEgfs;/jobs/{J_JOB} + + jgfs_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gfspost + Name: !expand jgfs_post_f{dimval.fhr:03d} + FHR: !expand "{dimval.fhr:03d}" + config_list: [ base, post ] + J_JOB: rocoto/post.sh + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + - otherwise: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) | up.forecast + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + /usr/bin/env post_times={FHR} FHRLST={FHR} FHRGRP={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgfs_pgrb2_spec_post: !FirstTrue + - when: !calc max(doc.gfs_output_settings.gfs_forecast_hours)>=384 + do: !Task + <<: *exclusive_task_template + Dummy: true ## <-- Node is not implemented yet + Trigger: !Depend ( jgfs_post_f336 & jgfs_post_f348 & jgfs_post_f360 & jgfs_post_f372 & jgfs_post_f384 ) + resources: !calc partition.resources.run_gfspost + - otherwise: null + #endfamily post + + post_processing: !Family + Disable: !calc not doc.downstream.DO_POST_PROCESSING + fax: !Family + Disable: !calc not doc.downstream.DO_FAX + jgfs_fax_f000: !Task + <<: *fax_task_template + FCSTHR: '00' + Trigger: !Depend ( up.up.post.jgfs_post_f000 & up.up.post.jgfs_post_anl ) + + jgfs_fax_anl: !Task + <<: *fax_task_template + FCSTHR: 'anl' + Trigger: !Depend up.up.post.jgfs_post_anl + + jgfs_fax_wafs_f012: !Task + <<: *fax_wafs_task_template + FCSTHR: '12' + fhr_list: '06 12' + Trigger: !Depend up.up.post.jgfs_post_f012 + + jgfs_fax_wafs_f024: !Task + <<: *fax_wafs_task_template + FCSTHR: '24' + fhr_list: '18 24' + Trigger: !Depend up.up.post.jgfs_post_f024 + + jgfs_fax_wafs_f036: !Task + <<: *fax_wafs_task_template + FCSTHR: '36' + fhr_list: '30 36' + Trigger: !Depend up.up.post.jgfs_post_f036 + + grib_wafs: !TaskArray + Disable: !calc not doc.downstream.DO_WAFS + Dimensions: + fhr: !calc tools.seq(0,doc.gfs_output_settings.wafs_last_hour,6) + jgfs_wafs: !TaskElement + <<: *exclusive_task_template + Name: !expand 'jgfs_wafs_f{dimval.fhr:03d}' + resources: !ref partition.resources.run_one_node_downstream + Trigger: !FirstTrue + - when: !calc dimval.fhr == 0 + do: !Depend ( up.up.post.jgfs_post_f000 & up.up.post.jgfs_post_f120 & up.grib2_wafs.jgfs_wafs_grib2 ) + - otherwise: !Depend >- + up.up.post.depend("jgfs_post_f{N:03d}",N=[ dimval.fhr ]) + & up.grib_wafs.depend("jgfs_wafs_f{N:03d}",N=[ dimval.fhr-6 ]) + ecflow_command: !expand | + export fcsthrs=%FCSTHR% + $HOMEgfs/jobs/{J_JOB} + FCSTHR: !expand '{dimval.fhr:02d}' + J_JOB: JGFS_WAFS + more_vars: { fcsthrs: FCSTHR } + config_list: [ base, wafs ] + + bufr_sounding: !Family + Disable: !calc not doc.downstream.DO_BUFRSND + jgfs_postsnd: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f144 + resources: !calc partition.resources.run_postsnd + + bulletins: !Family + Disable: !calc not doc.downstream.DO_BULLETINS + jgfs_fbwind: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.up.post.jgfs_post_f006 & up.up.post.jgfs_post_f012 & up.up.post.jgfs_post_f024 ) + resources: !calc partition.resources.run_small_downstream + + grib2_wafs: !Family + Disable: !calc not doc.downstream.DO_WAFS + jgfs_wafs_grib2: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f000 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_blending: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_wafs_grib2 + Time: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + do: !timedelta +4:33:00 + - otherwise: null + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_gcip: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f003 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs_gcip ] + J_JOB: JGFS_WAFS_GCIP + + awips_20km_1p0: !TaskArray + Disable: !calc not doc.downstream.DO_AWIPS + Dimensions: + fhr: !calc " sorted(list(set(doc.gfs_output_settings.gfs_forecast_hours)&set(doc.gfs_output_settings.awips_20km_1p0_hours))) " + # tasks every 6 hours till f240 + jgfs_awips: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgfs_awips_f{dimval.fhr:03d}" + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + Trigger: !Depend up.up.post.depend('jgfs_post_f{N:03d}',N=dimval.fhr) + resources: !calc partition.resources.run_awips + TRDRUN: !calc ('YES' if (dimval.fhr==0 or dimval.fhr%6!=0 or dimval.fhr>=84) else 'NO') + J_JOB: JGFS_AWIPS_20KM_1P0DEG + FHR: !expand '{dimval.fhr:03d}' + more_vars: { fcsthrs: FHR } + + awips_g2: !TaskArray + Disable: !calc not doc.downstream.DO_AWIPS + Dimensions: + fhr: !calc " sorted(list(set(doc.gfs_output_settings.gfs_forecast_hours)&set(doc.gfs_output_settings.awips_g2_hours))) " + jgfs_awips: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgfs_awips_g2_f{dimval.fhr:03d}" + Trigger: !Depend up.up.post.depend('jgfs_post_f{N:03d}',N=dimval.fhr) + resources: !calc partition.resources.run_awips + FHR: !expand '{dimval.fhr:03d}' + J_JOB: JGFS_AWIPS_G2 + more_vars: { fcsthrs: FHR } + #endfamily post_processing + + gempak: !Family + Disable: !calc not doc.downstream.DO_GEMPAK + jgfs_gempak: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gfs_gempak + Trigger: !Depend up.jgfs_analysis + + jgfs_gempak_meta: !Task + <<: *exclusive_task_template + Dummy: True # job is broken + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfs_gempak + + jgfs_pgrb2_spec_npoess: !Task + <<: *exclusive_task_template + Trigger: !Depend 'up.post.jgfs_post_anl.is_running() | up.post.jgfs_post_anl' + resources: !calc partition.resources.run_one_node_downstream + + jgfs_pgrb2_spec_gempak: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_pgrb2_spec_npoess + resources: !calc partition.resources.run_big_downstream + + jgfs_gempak_ncdc_upapgif: !Task # NOTE: twelve-hourly + <<: *exclusive_task_template + Dummy: True # job is broken + Trigger: !Depend 'jgfs_gempak.is_running() | jgfs_gempak' + resources: !calc partition.resources.run_one_node_downstream + + jgfs_emc_vrfy: !Task + <<: *exclusive_task_template + Trigger: !Depend post + Disable: !calc not doc.settings.run_vrfy_jobs + resources: !calc partition.resources.run_gfsvrfy + J_JOB: rocoto/vrfy.sh + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + ecf_module_commands: "# vrfy.sh will load modules instead" + + #endfamily gempak + #endfamily gfs + + ###################################################################### + ## ARCHIVE FAMILY #################################################### + ###################################################################### + + archive: !Family + Disable: !calc not doc.archiving.archive_to_hpss + ecflow_def: + edit ECF_TRIES '3' + jgdas_archive: !Task + <<: *service_task_template + AlarmName: gdas + Trigger: !Depend up.gdas + #Trigger: !Depend up.gdas.jgdas_emc_vrfy + resources: !calc partition.resources.run_arch + Disable: !calc not doc.archiving.archive_to_hpss + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/arch.sh + ecf_module_commands: "# arch.sh will load modules instead" + RUN: gdas + ecflow_def: !FirstTrue + - when: !calc not doc.settings.four_cycle_mode + take: "edit RUN 'gdas'" + - otherwise: | + time +00:10 + edit RUN 'gdas' + + jgdas_enkf_archive: !TaskArray + ecflow_def: "edit RUN 'gdas'" + AlarmName: gdas + Trigger: !Depend up.gdas.enkf.post + Dimensions: + groupid: !calc tools.seq(0,doc.data_assimilation.ENKF_ARCHIVE_GROUPS,1) + grp: !TaskElement + <<: *service_task_template + resources: !calc partition.resources.run_arch + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/earc.sh + ecf_module_commands: "# arch.sh will load modules instead" + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_archive_grp{dimval.groupid:02d}" + RUN: gdas + ENSGRP: !expand '{dimval.groupid:02d}' + more_vars: { ENSGRP: ENSGRP } + ecflow_def: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: !expand "edit ENSGRP '{ENSGRP}'" + - otherwise: !expand | + time +00:10 + edit ENSGRP '{ENSGRP}' + + jgfs_archive: !Task + <<: *service_task_template + Disable: !calc doc.settings.gfs_cyc == 0 + Complete: !Depend ~ suite.has_cycle('-6:00:00') + AlarmName: gfs + Trigger: !Depend up.gfs + resources: !calc partition.resources.run_arch + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/arch.sh + ecf_module_commands: "# arch.sh will load modules instead" + RUN: gfs + ecflow_def: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: "edit RUN 'gfs'" + - otherwise: | + time +00:10 + edit RUN 'gfs' + + ###################################################################### + ## CYCLE COMPLETION TASKS ############################################ + ###################################################################### + + cycle_end: *cycle_end_task + + make_next_cycles: !Task + <<: *make_next_cycles_task + Disable: !calc ( not metasched.type=="ecflow" or doc.settings.four_cycle_mode ) + Trigger: !Depend gdas + ecflow_def: !expand | + # These variables are used by the make_next_cycles job to generate + # suites for later cycles once earlier cycles have finished. + edit WORKFLOW_FIRST_CYCLE '{tools.strftime(suite.Clock.start,"%Y%m%d%H")}' + edit WORKFLOW_LAST_CYCLE '{tools.strftime(suite.Clock.end,"%Y%m%d%H")}' + edit WORKFLOW_CYCLES_TO_GENERATE '5' + edit WORKFLOW_CROW_HOME '{doc.places.HOMEcrow}' + edit WORKFLOW_EXPDIR '{doc.places.EXPDIR}' + edit RUN 'gdas' # useless but required + + final: !Task + <<: *service_task_template + Disable: !calc not metasched.type=="rocoto" + resources: !calc partition.resources.run_final + rocoto_command: /bin/true + RUN: gfs # useless but required + diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml new file mode 100644 index 00000000000..1b6e6677496 --- /dev/null +++ b/workflow/layout/free_forecast_gfs.yaml @@ -0,0 +1,211 @@ +suite: !Cycle + <<: *suite_defaults + + Alarms: + <<: *cycled_suite_alarms + + ##################################################################### + ## ICS FAMILY ####################################################### + ##################################################################### + + ics: !Family + Disable: !calc >- + doc.settings.get("IC_CDUMP","") and not doc.settings.chgres_and_convert_ics +# RUN: 'gfs' + jgfs_emc_getics: !Task + <<: *service_task_template + RUN: 'gfs' +# Disable: !calc doc.settings.get("IC_CDUMP","") + Disable: !calc doc.settings.get("IC_CDUMP","") or doc.settings.cpl + resources: !calc partition.resources.run_getic + J_JOB: rocoto/getic.sh + ecf_module_commands: "# getics.sh will load modules instead" + model: gfs + + jgfs_emc_fv3ic: !Task + <<: *exclusive_task_template + RUN: 'gfs' +# Disable: !calc not doc.settings.chgres_and_convert_ics + Disable: !calc not doc.settings.chgres_and_convert_ics or doc.settings.cpl + Trigger: !Depend jgfs_emc_getics + resources: !calc partition.resources.run_fv3ic + J_JOB: rocoto/crow_fv3ic.sh + ecf_module_commands: "# fv3ic.sh will load modules instead" + model: gfs + + jgfs_emc_mom6ic: !Task + <<: *exclusive_task_template + RUN: 'gfs' + Disable: !calc doc.settings.cplflx=='.false.' or not doc.settings.chgres_and_convert_ics + Trigger: !Depend jgfs_emc_getics + resources: !calc partition.resources.run_fv3ic + J_JOB: rocoto/coupled_ic.sh + ecf_module_commands: "# fv3ic.sh will load modules instead" + model: gfs + + ##################################################################### + ## GFS FAMILY ####################################################### + ##################################################################### + + gfs: !Family +# AlarmName: gdas ##!!"gdas" mean to run every 6 hrs from sdate to edate!!" + RUN: 'gfs' + ecflow_def: "edit RUN 'gfs'" + + forecast: !Family + medcold: !Task + <<: *exclusive_task_template + Disable: !calc doc.settings.cplflx=='.false.' + Trigger: !Depend up.up.ics.jgfs_emc_mom6ic + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + resources: !calc partition.resources.run_coupled_medcold + J_JOB: JGLOBAL_FORECAST_MEDCOLD + + highres: !Task + <<: *exclusive_task_template + Trigger: !FirstTrue + - when: !calc doc.settings.cplflx=='.true.' + do: !Depend medcold + - otherwise: !Depend up.up.ics.jgfs_emc_fv3ic + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + resources: !FirstTrue + - when: !calc doc.settings.cplflx=='.true.' + do: !calc partition.resources.run_coupled_fcst + - otherwise: !calc partition.resources.run_gfsfcst + J_JOB: JGLOBAL_FORECAST + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.gfs_output_settings.gfs_forecast_hours + jgfs_post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Disable: !calc metasched.type == 'rocoto' + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend ( up.forecast.is_running() ) + - otherwise: !Depend "up.forecast.is_running() | up.forecast" + Foreach: [ ] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgfs_post_manager + resources: !calc partition.resources.run_gfs_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match post_manager.yaml + + jgfs_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: anl + Name: p_anl + Trigger: !FirstTrue + - when: !calc doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.release_postanl + - otherwise: !Depend jgfs_post_manager.release_postanl | up.forecast + + ecflow_def: !expand "edit FHR '{FHR}'" + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gfspost + J_JOB: JGLOBAL_NCEPPOST + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgfs_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gfspost + Name: !expand p_{dimval.fhr:03d} + FHR: !expand "{dimval.fhr:03d}" + J_JOB: JGLOBAL_NCEPPOST + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + - otherwise: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) | up.forecast + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env post_times={FHR} FHRLST={FHR} FHRGRP={FHR} &HOMEgfs;/jobs/{J_JOB} + #endfamily post + + ocnpost: !TaskArray + RUN: !calc up.RUN + Disable: !calc not doc.settings.cplflx + Dimensions: + fhr: !calc doc.gfs_output_settings.ocnpost_hours + jgfs_ocnpost_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_one_node_downstream + Name: !expand p_{dimval.fhr:03d} + FHR: !expand "{dimval.fhr:03d}" + OCN_FHRLST: !calc "tools.seq(dimval.fhr, dimval.fhr+doc.gfs_output_settings.OCN_INTERVAL, 6)" + J_JOB: rocoto/ocnpost.sh + Trigger: !Depend up.forecast + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + export OCN_FHRLST="{OCN_FHRLST}"; $HOMEgfs/jobs/{J_JOB} + #endfamily ocnpost + + jgfs_emc_vrfy: !Task + <<: *exclusive_task_template + Trigger: !Depend post + Disable: !calc not doc.settings.run_vrfy_jobs + resources: !calc partition.resources.run_gfsvrfy + J_JOB: rocoto/vrfy.sh + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + ecf_module_commands: "# vrfy.sh will load modules instead" + + #endfamily gfs + + + ####################################################################### + ## ARCHIVE FAMILY ##################################################### + ####################################################################### + + archive: !Family + Disable: !calc not doc.archiving.archive_to_hpss + ecflow_def: + edit ECF_TRIES '3' + + jgfs_archive: !Task + <<: *service_task_template + Disable: !calc doc.settings.gfs_cyc == 0 + AlarmName: gfs + Trigger: !Depend up.gfs + resources: !calc partition.resources.run_arch + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/arch.sh + ecf_module_commands: "# arch.sh will load modules instead" + RUN: gfs + ecflow_def: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: "edit RUN 'gfs'" + - otherwise: | + time +00:10 + edit RUN 'gfs' + + final: !Task + <<: *service_task_template + Disable: !calc not metasched.type=="rocoto" + resources: !calc partition.resources.run_final + rocoto_command: /bin/true + model: gfs # useless but required + diff --git a/workflow/layout/manual_temp.yaml b/workflow/layout/manual_temp.yaml new file mode 100644 index 00000000000..58696fbf712 --- /dev/null +++ b/workflow/layout/manual_temp.yaml @@ -0,0 +1,548 @@ +suite: !Cycle + <<: *suite_defaults + + Alarms: + <<: *cycled_suite_alarms + + ###################################################################### + ## GDAS FAMILY ####################################################### + ###################################################################### + + gdas: !Family + RUN: gdas + ecflow_def: "edit RUN 'gdas'" + + dump: !Family + ecflow_def: "# NCO will need to add NCO dump job here" + Dummy: true ## <-- Node is not implemented yet + Trigger: !Depend forecast.at('-6:00:00') + jgdas_ics: !Task + <<: *exclusive_task_template + release_gdas00_ics: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + + jgdas_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Trigger: !Depend jgdas_dump + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:45:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + + #This dump job should be using NCO version when delivery to NCO + jgdas_dump: !Task + <<: *exclusive_task_template + release_sfcprep: !DataEvent {file="/dev/null"} + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:50:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + #endfamily dump + + prep: !Family + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + + jgdas_prep: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.dump.jgdas_dump & up.up.gdas.post.at('-6:00:00') ) + ecflow_def: + edit model 'obsproc_global' + resources: !calc partition.resources.run_prep + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: JGLOBAL_PREP + model: obsproc_global # for four cycle mode + J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + jgdas_tropcy_cp: !Task + <<: *exclusive_task_template + Dummy: True + Trigger: !Depend prep.jgdas_prep + resources: !calc partition.resources.run_one_hour_exclusive + J_JOB: rocoto/tropcy_cp.sh + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + + analysis: !Family + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + jgdas_analysis_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & up.enkf.post.at('-6:00:00') ) + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + + forecast: !Family + jgdas_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.analysis.jgdas_analysis_high & up.enkf.innovate ) | ~ suite.has_cycle('-6:00:00') + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gdasfcst + J_JOB: JGLOBAL_FORECAST + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.settings.gdas_forecast_hours + post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Trigger: !Depend ( up.forecast.is_running() ) + Disable: !calc metasched.type=='rocoto' + Foreach: [] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgdas_post_manager + resources: !calc partition.resources.run_gdas_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand > + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match the logic in scripts/exglobal_pmgr.sh.ecf + + # The post_anl has to be a !TaskElement to ensure it shows up between + # the post manager and forecast hour posts: + jgdas_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: 'anl' + Name: jgdas_post_anl + Trigger: !Depend jgdas_post_manager.release_postanl + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gdaspost + ecflow_def: !expand "edit FHR '{FHR}'" + J_JOB: JGLOBAL_NCEPPOST + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgdas_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gdaspost + Name: !expand jgdas_post_f{dimval.fhr:03d} + FHR: !expand '{dimval.fhr:03d}' + J_JOB: JGLOBAL_NCEPPOST + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + Trigger: !Depend jgdas_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + #endfamily post + + enkf: !Family + jgdas_enkf_select_obs: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & post.at('-6:00:00') & up.jgdas_tropcy_cp ) + resources: !calc partition.resources.run_eobs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + J_JOB: JGLOBAL_ENKF_SELECT_OBS + + innovate: !TaskArray + Trigger: !Depend jgdas_enkf_select_obs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: [ 1, 2, 3, 4, 5, 6 ] + jgdas_enkf_innovate_obs_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_innovate_obs_grp{dimval.groupid}" + resources: !calc partition.resources.run_eomg + J_JOB: JGLOBAL_ENKF_INNOVATE_OBS + ENSGRP: !expand "{dimval.groupid:02d}" + # Lin wants ENSGRP to be outside the ecflow suite definition: + #rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB} + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + + jgdas_enkf_update: !Task + <<: *exclusive_task_template + Trigger: !Depend innovate + ecflow_def: "edit ECF_PASS 'FREE'" + Complete: !Depend ~ suite.has_cycle('-6:00:00') + resources: !calc partition.resources.run_eupd + J_JOB: JGLOBAL_ENKF_UPDATE + + jgdas_enkf_inflate_recenter: !Task + <<: *exclusive_task_template + Trigger: !Depend ( jgdas_enkf_update & up.analysis.jgdas_analysis_high ) + resources: !calc partition.resources.run_ecen + J_JOB: JGDAS_ENKF_RECENTER + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !TaskArray + + Trigger: !Depend up.enkf.jgdas_enkf_inflate_recenter | ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] + jgdas_enkf_fcst_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_fcst_grp{dimval.groupid}" + resources: !calc partition.resources.run_efcs + J_JOB: JGDAS_ENKF_FCST + ENSGRP: !expand "{dimval.groupid:02d}" + + # Lin wants ENSGRP to be outside the ecflow suite definition: + #rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB} + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + +# Create epos task array +# export FHRLST="f003 f004 f005 f006 f007 f008 f009" +# fhrlst=$(echo $FHRLST | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +# FHMIN_EPOS=$fhr +# FHMAX_EPOS=$fhr +# FHOUT_EPOS=$fhr +# job=epos${fhr} + post: !TaskArray + Trigger: !Depend forecast + Dimensions: + fhr: !calc doc.settings.enkf_epos_fhr + jgdas_enkf_post_fhr: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgdas_enkf_post_f{dimval.fhr:03d}" + FHMIN_EPOS: !expand '{dimval.fhr:03d}' + FHMAX_EPOS: !expand '{dimval.fhr:03d}' + FHOUT_EPOS: !expand '{dimval.fhr:03d}' + ecflow_command: !expand | + export FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} &HOMEgfs;/jobs/{J_JOB} + resources: !calc partition.resources.run_epos + J_JOB: JGDAS_ENKF_POST + + #endfamily enkf + #endfamily gdas + + ###################################################################### + ## GFS FAMILY ######################################################## + ###################################################################### + + gfs: !Family + Disable: !calc doc.settings.gfs_cyc==0 + Complete: !Depend ~ suite.has_cycle('-6:00:00') + AlarmName: gfs + RUN: 'gfs' + ecflow_def: "edit RUN 'gfs'" + + jgfs_emc_dump_waiter: !Task + <<: *dump_waiter_task + Disable: !calc metasched.type != 'ecflow' + realtime_logic: !FirstTrue + - when: !calc doc.settings.realtime + do: "edit ECF_TRIES 72" + - otherwise: "" + #Time: !FirstTrue + # - when: !calc doc.settings.four_cycle_mode + # do: !timedelta +2:47:00 + # - otherwise: null + ecflow_def: !expand | + {realtime_logic} + edit DUMPDIR '{doc.places.DMPDIR}' + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + + dump: !Family + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + Complete: !Depend ~ suite.has_cycle('-6:00:00') + Dummy: true ## <-- Node is not implemented yet + ecflow_def: "# NCO will need to add NCO dump job here" + jgfs_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_dump + jtwc_bull_email: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:41:00 + - otherwise: null + + jgfs_dump: !Task + <<: *exclusive_task_template + ecflow_def: '#NCO needs to replace this with the real dump job' + release_sfcprep: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:47:00 + - otherwise: null + + prep: !Family + Trigger: !Depend ( ( jgfs_emc_dump_waiter.updated_status | dump.jgfs_dump.release_sfcprep ) & up.gdas.forecast.at('-6:00:00') ) + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_prep: !Task + <<: *exclusive_task_template + ecflow_def: + edit model 'obsproc_global' + Trigger: !Depend up.dump + resources: !calc partition.resources.run_prep + ecf_environment_settings: !expand | + {doc.settings.ecflow_rocoto_cdate_workaround} + export DATAROOT="$DATAROOT.$job" + J_JOB: JGLOBAL_PREP + model: obsproc_global # for four cycle mode + J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + jgfs_tropcy_cp: !Task + <<: *exclusive_task_template + Dummy: True + Trigger: !Depend prep.jgfs_prep + resources: !calc partition.resources.run_one_hour_exclusive + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/tropcy_cp.sh + + jgfs_analysis: !Task + <<: *exclusive_task_template + Trigger: !Depend ( prep.jgfs_prep & up.gdas.enkf.post.at('-6:00:00') ) + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !Family + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfsfcst + J_JOB: JGLOBAL_FORECAST + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.settings.gfs_forecast_hours + jgfs_post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Disable: !calc metasched.type == 'rocoto' + Trigger: !Depend "up.forecast.is_running()" + Foreach: [ ] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgfs_post_manager + resources: !calc partition.resources.run_gfs_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match post_manager.yaml + + jgfs_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: anl + Name: jgfs_post_anl + Trigger: !Depend jgfs_post_manager.release_postanl + + ecflow_def: !expand "edit FHR '{FHR}'" + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gfspost + J_JOB: JGLOBAL_NCEPPOST + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env post_times=anl &HOMEgfs;/jobs/{J_JOB} + + jgfs_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gfspost + Name: !expand jgfs_post_f{dimval.fhr:03d} + FHR: !expand "{dimval.fhr:03d}" + J_JOB: JGLOBAL_NCEPPOST + Trigger: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env post_times={FHR} FHRLST={FHR} FHRGRP={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgfs_pgrb2_spec_post: !FirstTrue + - when: !calc max(doc.settings.gfs_forecast_hours)>=384 + do: !Task + <<: *exclusive_task_template + Dummy: true ## <-- Node is not implemented yet + Trigger: !Depend ( jgfs_post_f336 & jgfs_post_f348 & jgfs_post_f360 & jgfs_post_f372 & jgfs_post_f384 ) + resources: !calc partition.resources.run_gfspost + - otherwise: null + #endfamily post + + post_processing: !Family + grib2_wafs: !Family + jgfs_wafs_grib2: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f000 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_blending: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_wafs_grib2 + Time: !timedelta +4:33:00 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_gcip: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f003 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs_gcip ] + J_JOB: JGFS_WAFS_GCIP + + gempak: !Family + jgfs_gempak: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gfs_gempak + Trigger: !Depend up.jgfs_analysis + + jgfs_gempak_meta: !Task + <<: *exclusive_task_template + Dummy: True # job is broken + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfs_gempak + + jgfs_pgrb2_spec_npoess: !Task + <<: *exclusive_task_template + Trigger: !Depend 'up.post.jgfs_post_anl.is_running() | up.post.jgfs_post_anl' + resources: !calc partition.resources.run_npoess + + jgfs_pgrb2_spec_gempak: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_pgrb2_spec_npoess + resources: !calc partition.resources.run_big_downstream + + jgfs_gempak_ncdc_upapgif: !Task # NOTE: twelve-hourly + <<: *exclusive_task_template + Dummy: True # job is broken + Trigger: !Depend 'jgfs_gempak.is_running() | jgfs_gempak' + resources: !calc partition.resources.run_one_node_downstream + + #endfamily gempak + #endfamily gfs + + ###################################################################### + ## ARCHIVE FAMILY #################################################### + ###################################################################### + + archive: !Family +# Disable: !calc not doc.archiving.archive_to_hpss + ecflow_def: + edit ECF_TRIES '3' + jgdas_archive: !Task + <<: *service_task_template + AlarmName: gdas + Trigger: !Depend up.gdas + #Trigger: !Depend up.gdas.jgdas_emc_vrfy + resources: !calc partition.resources.run_arch +# Disable: !calc not doc.archiving.archive_to_hpss + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/arch.sh + ecf_module_commands: "# arch.sh will load modules instead" + RUN: gdas + ecflow_def: !FirstTrue + - when: !calc not doc.settings.four_cycle_mode + take: "edit RUN 'gdas'" + - otherwise: | + time +00:10 + edit RUN 'gdas' + + jgdas_enkf_archive: !TaskArray + ecflow_def: "edit RUN 'gdas'" + AlarmName: gdas + Trigger: !Depend up.gdas.enkf.post + Dimensions: + groupid: [ 1, 2, 3, 4, 5, 6 ] + grp: !TaskElement + <<: *service_task_template + resources: !calc partition.resources.run_arch + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/earc.sh + ecf_module_commands: "# arch.sh will load modules instead" + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_archive_grp{dimval.groupid:02d}" + RUN: gdas + ENSGRP: !expand '{dimval.groupid:02d}' + more_vars: { ENSGRP: ENSGRP } + ecflow_def: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: !expand "edit ENSGRP '{ENSGRP}'" + - otherwise: !expand | + time +00:10 + edit ENSGRP '{ENSGRP}' + + jgfs_archive: !Task + <<: *service_task_template + Disable: !calc doc.settings.gfs_cyc == 0 + Complete: !Depend ~ suite.has_cycle('-6:00:00') + AlarmName: gfs + Trigger: !Depend up.gfs + resources: !calc partition.resources.run_arch + ecf_environment_settings: !calc doc.settings.ecflow_rocoto_cdate_workaround + J_JOB: rocoto/arch.sh + ecf_module_commands: "# arch.sh will load modules instead" + RUN: gfs + ecflow_def: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: "edit RUN 'gfs'" + - otherwise: | + time +00:10 + edit RUN 'gfs' + + ###################################################################### + ## CYCLE COMPLETION TASKS ############################################ + ###################################################################### + + cycle_end: *cycle_end_task + +# make_next_cycles: !Task +# <<: *make_next_cycles_task +# Disable: !calc ( not metasched.type=="ecflow" or doc.settings.four_cycle_mode ) +# Trigger: !Depend gdas +# ecflow_def: !expand | +# # These variables are used by the make_next_cycles job to generate +# # suites for later cycles once earlier cycles have finished. +# edit WORKFLOW_FIRST_CYCLE '{tools.strftime(suite.Clock.start,"%Y%m%d%H")}' +# edit WORKFLOW_LAST_CYCLE '{tools.strftime(suite.Clock.end,"%Y%m%d%H")}' +# edit WORKFLOW_CYCLES_TO_GENERATE '5' +# edit WORKFLOW_CROW_HOME '{doc.places.HOMEcrow}' +# edit WORKFLOW_EXPDIR '{doc.places.EXPDIR}' +# edit RUN 'gdas' # useless but required + + final: !Task + <<: *service_task_template + Disable: !calc not metasched.type=="rocoto" + resources: !calc partition.resources.run_nothing + rocoto_command: /bin/true + RUN: gfs # useless but required + diff --git a/workflow/layout/nco.yaml b/workflow/layout/nco.yaml new file mode 100644 index 00000000000..b199e543a5e --- /dev/null +++ b/workflow/layout/nco.yaml @@ -0,0 +1,597 @@ +suite: !Cycle + <<: *suite_defaults + + Alarms: + <<: *cycled_suite_alarms + + ###################################################################### + ## GDAS FAMILY ####################################################### + ###################################################################### + + gdas: !Family + RUN: gdas + ecflow_def: "edit RUN 'gdas'" + + jgdas_verfrad: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_verfrad + + jgdas_vminmon: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_vminmon + + dump: !Family + + #jgdas_dump_post: !Task + # Trigger: !Depend jgdas_dump + # release_sfcprep: !DataEvent {file="/dev/null"} + # release_gdas00_dump_alert: !DataEvent {file="/dev/null"} + # ecf_file: *ecf_file_template + # resources: !calc partition.resources.run_nothing + # accounting: *exclusive_accounting + # J_JOB: nothing + + jgdas_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:45:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + + #Replaced by emc version of dump job + #This dump job should be using NCO version when delivery to NCO + jgdas_dump: !Task + <<: *exclusive_task_template + release_sfcprep: !DataEvent {file="/dev/null"} + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +5:50:00 + - otherwise: null + resources: !calc partition.resources.run_one_hour_exclusive + #endfamily dump + + prep: !Family + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + jgdas_emcsfc_sfc_prep: !Task + <<: *exclusive_task_template + Trigger: !Depend 'up.dump.jgdas_dump.release_sfcprep' + resources: !calc partition.resources.run_prep + config_list: [ base, prep, prepbufr ] + J_JOB: JGLOBAL_EMCSFC_SFC_PREP + + jgdas_prep: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.dump.jgdas_dump & up.up.gdas.post.at('-6:00:00') ) + ecflow_def: + edit model 'obsproc_global' + resources: !calc partition.resources.run_prep + J_JOB: JGLOBAL_PREP + model: obsproc_global # for four cycle mode + J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + analysis: !Family + Complete: !Depend ( ~ suite.has_cycle('-6:00:00') ) + jgdas_analysis_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & up.prep.jgdas_emcsfc_sfc_prep & up.enkf.post.at('-6:00:00') ) + #Trigger: !Depend ( up.prep.jgdas_prep & up.prep.jgdas_emcsfc_sfc_prep ) + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + + forecast: !Family + jgdas_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.analysis.jgdas_analysis_high & up.enkf.innovate ) | ~ suite.has_cycle('-6:00:00') + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + #release_fcst: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gdasfcst + J_JOB: JGLOBAL_FORECAST + + post_processing: !Family + Complete: !Depend ~ suite.has_cycle('-6:00:00') + bulletins: !Family + jgdas_bulls_navy: !Task + <<: *exclusive_task_template + AlarmName: gfs_00_12 + Trigger: !Depend ( up.up.dump.jgdas_dump ) + resources: !calc partition.resources.run_big_downstream + J_JOB: JGDAS_BULLS_NAVY + + gempak: !Family + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgdas_gempak: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gdas_gempak + J_JOB: JGDAS_GEMPAK + Trigger: !Depend '( up.post if doc.settings.dev_safeguards else up.forecast.jgdas_forecast_high )' + + jgdas_gempak_meta_ncdc: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gdas_gempak_meta_ncdc + Trigger: !Depend jgdas_gempak + J_JOB: JGDAS_GEMPAK_META_NCDC + + #endfamily gempak + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.gfs_output_settings.gdas_forecast_hours + post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend ( up.forecast.is_running() ) + - otherwise: !Depend "up.forecast.is_running() | up.forecast" + Disable: !calc metasched.type=='rocoto' + Foreach: [] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgdas_post_manager + resources: !calc partition.resources.run_gdas_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand > + {metasched.varref("COM")}/gdas.{metasched.datestring("%Y%m%d/%H/")}gdas.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match the logic in scripts/exglobal_pmgr.sh.ecf + + # The post_anl has to be a !TaskElement to ensure it shows up between + # the post manager and forecast hour posts: + jgdas_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: 'anl' + Name: jgdas_post_anl + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgdas_post_manager.release_postanl + - otherwise: !Depend jgdas_post_manager.release_postanl | up.forecast + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gdaspost + ecflow_def: !expand "edit FHR '{FHR}'" + J_JOB: JGLOBAL_NCEPPOST + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgdas_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gdaspost + Name: !expand jgdas_post_f{dimval.fhr:03d} + FHR: !expand '{dimval.fhr:03d}' + J_JOB: JGLOBAL_NCEPPOST + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHRGRP={FHR} post_times={FHR} FHRLST={FHR} &HOMEgfs;/jobs/{J_JOB} + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgdas_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + - otherwise: !Depend jgdas_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) | up.forecast + #endfamily post + + enkf: !Family + jgdas_enkf_select_obs: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.prep.jgdas_prep & post.at('-6:00:00') ) + resources: !calc partition.resources.run_eobs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + J_JOB: JGLOBAL_ENKF_SELECT_OBS + + innovate: !TaskArray + Trigger: !Depend jgdas_enkf_select_obs + Complete: !Depend ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: !calc tools.seq(1,doc.data_assimilation.ENKF_INNOVATE_GROUPS,1) + jgdas_enkf_innovate_obs_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_innovate_obs_grp{dimval.groupid}" + resources: !calc partition.resources.run_eomg + J_JOB: JGLOBAL_ENKF_INNOVATE_OBS + ENSGRP: !expand "{dimval.groupid:02d}" + # Lin wants ENSGRP to be outside the ecflow suite definition: + rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + + jgdas_enkf_update: !Task + <<: *exclusive_task_template + Trigger: !Depend innovate + ecflow_def: "edit ECF_PASS 'FREE'" + Complete: !Depend ~ suite.has_cycle('-6:00:00') + resources: !calc partition.resources.run_eupd + J_JOB: JGLOBAL_ENKF_UPDATE + + jgdas_enkf_inflate_recenter: !Task + <<: *exclusive_task_template + Trigger: !Depend ( jgdas_enkf_update & up.analysis.jgdas_analysis_high ) + resources: !calc partition.resources.run_ecen + J_JOB: JGDAS_ENKF_RECENTER + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !TaskArray + + Trigger: !Depend up.enkf.jgdas_enkf_inflate_recenter | ~ suite.has_cycle('-6:00:00') + Dimensions: + groupid: !calc tools.seq(1,doc.data_assimilation.ENKF_FORECAST_GROUPS,1) + jgdas_enkf_fcst_grp: !TaskElement + <<: *exclusive_task_template + Foreach: [ groupid ] + Name: !expand "jgdas_enkf_fcst_grp{dimval.groupid}" + resources: !calc partition.resources.run_efcs + J_JOB: JGDAS_ENKF_FCST + ENSGRP: !expand "{dimval.groupid:02d}" + + # Lin wants ENSGRP to be outside the ecflow suite definition: + rocoto_command: !expand "/usr/bin/env ENSGRP={ENSGRP} &HOMEgfs;/jobs/{J_JOB}" + ecflow_command: !expand | + export ENSGRP={ENSGRP} + $HOMEgfs/jobs/{J_JOB} + +# Create epos task array +# export FHRLST="f003 f004 f005 f006 f007 f008 f009" +# fhrlst=$(echo $FHRLST | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +# FHMIN_EPOS=$fhr +# FHMAX_EPOS=$fhr +# FHOUT_EPOS=$fhr +# job=epos${fhr} + post: !TaskArray + Trigger: !Depend forecast + Dimensions: + fhr: !calc doc.gfs_output_settings.enkf_epos_fhr + jgdas_enkf_post_fhr: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgdas_enkf_post_f{dimval.fhr:03d}" + FHMIN_EPOS: !expand '{dimval.fhr:03d}' + FHMAX_EPOS: !expand '{dimval.fhr:03d}' + FHOUT_EPOS: !expand '{dimval.fhr:03d}' + job: !expand "epos{dimval.fhr:03d}" + ecflow_command: !expand | + export FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env FHMIN_EPOS={FHMIN_EPOS} FHMAX_EPOS={FHMAX_EPOS} FHOUT_EPOS={FHOUT_EPOS} job={job} &HOMEgfs;/jobs/{J_JOB} + resources: !calc partition.resources.run_epos + J_JOB: JGDAS_ENKF_POST + + #endfamily enkf + #endfamily gdas + + ###################################################################### + ## GFS FAMILY ######################################################## + ###################################################################### + + gfs: !Family + Disable: !calc doc.settings.gfs_cyc==0 + Complete: !Depend ~ suite.has_cycle('-6:00:00') + AlarmName: gfs + RUN: 'gfs' + ecflow_def: "edit RUN 'gfs'" + + dump: !Family + Trigger: !Depend up.gdas.forecast.at('-6:00:00') + Complete: !Depend ~ suite.has_cycle('-6:00:00') + ecflow_def: "# NCO will need to add NCO dump job here" + jgfs_tropcy_qc_reloc: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_dump + jtwc_bull_email: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:41:00 + - otherwise: null + + jgfs_dump: !Task + <<: *exclusive_task_template + ecflow_def: '#NCO needs to replace this with the real dump job' + release_sfcprep: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_one_hour_exclusive + Time: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !timedelta +2:47:00 + - otherwise: null + + prep: !Family + Trigger: !Depend "( dump.jgfs_dump.release_sfcprep ) & up.gdas.forecast.at('-6:00:00')" + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_emcsfc_sfc_prep: !Task + <<: *exclusive_task_template + Trigger: !Depend 'up.dump.jgfs_dump.release_sfcprep' + resources: !calc partition.resources.run_sfc_prep + J_JOB: JGLOBAL_EMCSFC_SFC_PREP + config_list: [ base, prep, prepbufr ] + + jgfs_prep: !Task + <<: *exclusive_task_template + ecflow_def: + edit model 'obsproc_global' + Trigger: !Depend up.dump + resources: !calc partition.resources.run_prep + J_JOB: JGLOBAL_PREP + model: obsproc_global # for four cycle mode + J_JOB_PATH: '$HOMEobsproc_global/jobs' + config_list: [ base, prep, prepbufr ] + + jgfs_analysis: !Task + <<: *exclusive_task_template + Trigger: !Depend ( prep.jgfs_prep & prep.jgfs_emcsfc_sfc_prep & up.gdas.enkf.post.at('-6:00:00') ) + #Trigger: !Depend ( prep.jgfs_prep & prep.jgfs_emcsfc_sfc_prep ) + resources: !calc partition.resources.run_anal + J_JOB: JGLOBAL_ANALYSIS + Complete: !Depend ~ suite.has_cycle('-6:00:00') + + forecast: !Family + Complete: !Depend ~ suite.has_cycle('-6:00:00') + jgfs_forecast_high: !Task + <<: *exclusive_task_template + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfsfcst + J_JOB: JGLOBAL_FORECAST + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + + post: !TaskArray + RUN: !calc up.RUN + Dimensions: + fhr: !calc doc.gfs_output_settings.gfs_forecast_hours + jgfs_post_manager_el: !TaskElement + <<: *exclusive_task_template + RUN: !calc up.RUN + Disable: !calc metasched.type == 'rocoto' + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend "up.forecast.is_running()" + - otherwise: !Depend "up.forecast.is_running() | up.forecast" + Foreach: [ ] + J_JOB: JGLOBAL_POST_MANAGER + Name: jgfs_post_manager + resources: !calc partition.resources.run_gfs_post_manager + release_postanl: !DataEvent + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf000.nemsio + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:03d}" + Foreach: [ fhr ] + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match post_manager.yaml + + jgfs_post_anl_el: !TaskElement + <<: *exclusive_task_template + Foreach: [] + FHR: anl + Name: jgfs_post_anl + Trigger: !FirstTrue + - when: !calc doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.release_postanl + - otherwise: !Depend jgfs_post_manager.release_postanl | up.forecast + + ecflow_def: !expand "edit FHR '{FHR}'" + release_pgrb2_anl: !DataEvent {file="/dev/null"} + resources: !calc partition.resources.run_gfspost + J_JOB: JGLOBAL_NCEPPOST + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env post_times=anl &HOMEgfs;/jobs/{J_JOB} + + jgfs_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gfspost + Name: !expand jgfs_post_f{dimval.fhr:03d} + FHR: !expand "{dimval.fhr:03d}" + J_JOB: JGLOBAL_NCEPPOST + Trigger: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + take: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) + - otherwise: !Depend jgfs_post_manager.depend("release_post{F:03d}",F=[dimval.fhr]) | up.forecast + ecflow_def: !expand "edit FHR '{FHR}'" + ecflow_command: !expand | + export post_times=%FHR% FHRLST=%FHR% FHRGRP=%FHR% + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + export post_times={FHR} FHRLST={FHR} FHRGRP={FHR} &HOMEgfs;/jobs/{J_JOB} + + jgfs_pgrb2_spec_post: !FirstTrue + - when: !calc max(doc.gfs_output_settings.gfs_forecast_hours)>=384 + do: !Task + <<: *exclusive_task_template + Trigger: !Depend ( jgfs_post_f336 & jgfs_post_f348 & jgfs_post_f360 & jgfs_post_f372 & jgfs_post_f384 ) + resources: !calc partition.resources.run_gfspost + - otherwise: null + #endfamily post + + post_processing: !Family + fax: !Family + jgfs_fax_f000: !Task + <<: *fax_task_template + FCSTHR: '00' + Trigger: !Depend ( up.up.post.jgfs_post_f000 & up.up.post.jgfs_post_anl ) + + jgfs_fax_anl: !Task + <<: *fax_task_template + FCSTHR: 'anl' + Trigger: !Depend up.up.post.jgfs_post_anl + + jgfs_fax_wafs_f012: !Task + <<: *fax_wafs_task_template + FCSTHR: '12' + fhr_list: '06 12' + Trigger: !Depend up.up.post.jgfs_post_f012 + + jgfs_fax_wafs_f024: !Task + <<: *fax_wafs_task_template + FCSTHR: '24' + fhr_list: '18 24' + Trigger: !Depend up.up.post.jgfs_post_f024 + + jgfs_fax_wafs_f036: !Task + <<: *fax_wafs_task_template + FCSTHR: '36' + fhr_list: '30 36' + Trigger: !Depend up.up.post.jgfs_post_f036 + + grib_wafs: !TaskArray + Dimensions: + fhr: !calc tools.seq(0,doc.gfs_output_settings.wafs_last_hour,6) + jgfs_wafs: !TaskElement + <<: *exclusive_task_template + Name: !expand 'jgfs_wafs_f{dimval.fhr:03d}' + resources: !ref partition.resources.run_one_node_downstream + Trigger: !FirstTrue + - when: !calc dimval.fhr == 0 + do: !Depend ( up.up.post.jgfs_post_f000 & up.up.post.jgfs_post_f120 & up.grib2_wafs.jgfs_wafs_grib2 ) + - otherwise: !Depend >- + up.up.post.depend("jgfs_post_f{N:03d}",N=[ dimval.fhr ]) + & up.grib_wafs.depend("jgfs_wafs_f{N:03d}",N=[ dimval.fhr-6 ]) + ecflow_command: !expand | + export fcsthrs=%FCSTHR% + $HOMEgfs/jobs/{J_JOB} + FCSTHR: !expand '{dimval.fhr:02d}' + J_JOB: JGFS_WAFS + more_vars: { fcsthrs: FCSTHR } + config_list: [ base, wafs ] + + bufr_sounding: !Family + jgfs_postsnd: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f144 + resources: !calc partition.resources.run_postsnd + + bulletins: !Family + jgfs_fbwind: !Task + <<: *exclusive_task_template + Trigger: !Depend ( up.up.post.jgfs_post_f006 & up.up.post.jgfs_post_f012 & up.up.post.jgfs_post_f024 ) + resources: !calc partition.resources.run_small_downstream + + grib2_wafs: !Family + jgfs_wafs_grib2: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f000 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_blending: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_wafs_grib2 + Time: !FirstTrue + - when: !calc not doc.settings.dev_safeguards + do: !timedelta +4:33:00 + - otherwise: null + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs ] + + jgfs_wafs_gcip: !Task + <<: *exclusive_task_template + Trigger: !Depend up.up.post.jgfs_post_f003 + resources: !calc partition.resources.run_one_node_downstream + config_list: [ base, wafs_gcip ] + J_JOB: JGFS_WAFS_GCIP + + awips_20km_1p0: !TaskArray + Dimensions: + fhr: !calc " sorted(list(set(doc.gfs_output_settings.gfs_forecast_hours)&set(doc.gfs_output_settings.awips_20km_1p0_hours))) " + # tasks every 6 hours till f240 + jgfs_awips: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgfs_awips_f{dimval.fhr:03d}" + Trigger: !Depend up.up.post.depend('jgfs_post_f{N:03d}',N=dimval.fhr) + resources: !calc partition.resources.run_awips_20km_1p0 + TRDRUN: !calc ('YES' if (dimval.fhr==0 or dimval.fhr%6!=0 or dimval.fhr>=84) else 'NO') + J_JOB: JGFS_AWIPS_20KM_1P0DEG + FHR: !expand '{dimval.fhr:03d}' + more_vars: { fcsthrs: FHR } + + awips_g2: !TaskArray + Dimensions: + fhr: !calc " sorted(list(set(doc.gfs_output_settings.gfs_forecast_hours)&set(doc.gfs_output_settings.awips_g2_hours))) " + jgfs_awips: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + Name: !expand "jgfs_awips_g2_f{dimval.fhr:03d}" + Trigger: !Depend up.up.post.depend('jgfs_post_f{N:03d}',N=dimval.fhr) + resources: !calc partition.resources.run_awips_g2 + FHR: !expand '{dimval.fhr:03d}' + J_JOB: JGFS_AWIPS_G2 + more_vars: { fcsthrs: FHR } + #endfamily post_processing + + gempak: !Family + jgfs_gempak: !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_gfs_gempak + Trigger: !Depend up.jgfs_analysis + + jgfs_gempak_meta: !Task + <<: *exclusive_task_template + Trigger: !Depend up.jgfs_analysis + resources: !calc partition.resources.run_gfs_gempak + + jgfs_pgrb2_spec_npoess: !Task + <<: *exclusive_task_template + Trigger: !Depend 'up.post.jgfs_post_anl.is_running() | up.post.jgfs_post_anl' + resources: !calc partition.resources.run_one_node_downstream + + jgfs_pgrb2_spec_gempak: !Task + <<: *exclusive_task_template + Trigger: !Depend jgfs_pgrb2_spec_npoess + resources: !calc partition.resources.run_big_downstream + + jgfs_gempak_ncdc_upapgif: !Task # NOTE: twelve-hourly + <<: *exclusive_task_template + Trigger: !Depend 'jgfs_gempak.is_running() | jgfs_gempak' + resources: !calc partition.resources.run_one_node_downstream + + #endfamily gempak + #endfamily gfs + + ###################################################################### + ## CYCLE COMPLETION TASKS ############################################ + ###################################################################### + + cycle_end: *cycle_end_task + + make_next_cycles: !Task + <<: *make_next_cycles_task + Disable: !calc ( not metasched.type=="ecflow" or doc.settings.four_cycle_mode ) + Trigger: !Depend gdas + ecflow_def: !expand | + # These variables are used by the make_next_cycles job to generate + # suites for later cycles once earlier cycles have finished. + edit WORKFLOW_FIRST_CYCLE '{tools.strftime(suite.Clock.start,"%Y%m%d%H")}' + edit WORKFLOW_LAST_CYCLE '{tools.strftime(suite.Clock.end,"%Y%m%d%H")}' + edit WORKFLOW_CYCLES_TO_GENERATE '5' + edit WORKFLOW_CROW_HOME '{doc.places.HOMEcrow}' + edit WORKFLOW_EXPDIR '{doc.places.EXPDIR}' + edit RUN 'gdas' # useless but required + + final: !Task + <<: *service_task_template + Disable: !calc not metasched.type=="rocoto" + resources: !calc partition.resources.run_nothing + rocoto_command: /bin/true + RUN: gfs # useless but required + diff --git a/workflow/layout/public_release_v1.yaml b/workflow/layout/public_release_v1.yaml new file mode 100644 index 00000000000..4c6fed50716 --- /dev/null +++ b/workflow/layout/public_release_v1.yaml @@ -0,0 +1,127 @@ +# This file describes the jobs to run and their dependencies. The +# CROW/crow/metascheduler package converts this to a Rocoto XML +# document or an ecFlow suit definition with ecf files. Much of the +# configuration logic is included from the runtime/ directory via YAML +# imports (<<: *anchor_name) + +suite: !Cycle + <<: *suite_defaults + + # The ics family of tasks generates FV3 input conditions. It is only + # enabled if required. + ics: !Family + Disable: !calc >- + doc.settings.get("IC_CDUMP","") and not doc.settings.chgres_and_convert_ics + model: 'gfs' # useless but required + + # The jgfs_emc_getics task pulls GFS GSM data from disk. + jgfs_emc_getics: !Task + <<: *service_task_template + Disable: !calc doc.settings.get("IC_CDUMP","") + resources: !calc partition.resources.run_getic + J_JOB: rocoto/getic.sh + ecf_module_commands: "# getics.sh will load modules instead" + model: gfs + + # The jgfs_emc_fv3ic task convers GFS GSM data to FV3 data + jgfs_emc_fv3ic: !Task + <<: *exclusive_task_template + Disable: !calc not doc.settings.chgres_and_convert_ics + Trigger: !Depend jgfs_emc_getics + resources: !calc partition.resources.run_fv3ic + J_JOB: rocoto/fv3ic.sh + ecf_module_commands: "# fv3ic.sh will load modules instead" + model: gfs + + # The gfs family executes the gfs forecast and post-processing. + # Were this the full workflow, it would also run the observation + # processing, analysis, and varoius downstream (post-processing of + # the post-processed data). + gfs: !Family + Trigger: !Depend ics + model: 'gfs' + + # Forecast family: in the old GSM GFS workflow this contains two + # jobs at two resolutions. We only have one forecast for FV3. + forecast: !Family + # jgfs_forecast_high - the FV3 forecast + jgfs_forecast_high: !Task + <<: *forecast_task_template + resources: !calc partition.resources.run_gfsfcst + J_JOB: JGLOBAL_FORECAST + resources_remap: !JobRequest [ { <<: *remap_resource_template } ] + + # post family - this family runs the UPP to convert FV3 output + # conditions to GRIB files. It runs wgrib2 to convert + # gaussian grid files to lat-lon. + post: !TaskArray + model: !calc up.model + Dimensions: + fhr: !calc doc.gfs_output_settings.gfs_forecast_hours + + # jgfs_post_manager_el - used to generate the jgfs_post_manager + # task. That task is only executed in ecFlow; it watches the + # forecast output directory for output data and alerts ecflow + # when new files become available. + # + # When running Rocoto, the specified data files (!DataEventElement) + # are used to define Rocoto date dependencies in later jobs. + jgfs_post_manager_el: !TaskElement + <<: *exclusive_task_template + model: !calc up.model + Disable: !calc metasched.type == 'rocoto' + Trigger: !Depend up.forecast.is_running() + Complete: !Depend up.forecast + Foreach: [ ] + ecflow_command: *post_manager_job_contents + J_JOB: post_manager + Name: jgfs_post_manager + resources: !calc partition.resources.run_gfs_post_manager + release_post_fhr: !DataEventElement + Name: !expand "release_post{dimval.fhr:02d}" + Foreach: [ fhr ] + file: !expand >- + {metasched.varref("COM")}/gfs.{metasched.datestring("%Y%m%d/%H/")}gfs.t{metasched.datestring("%H")}z.logf{dimval.fhr:03d}.nemsio + # NOTE: the above files must match post_manager.yaml + + # jgfs_post_fhr_el - generates the numerous jgfs_post jobs, one per + # forecast hour. These jobs run UPP and wgrib2 to convert FV3 + # output to gaussian and lat-lon GRIB files. + jgfs_post_fhr_el: !TaskElement + <<: *exclusive_task_template + Foreach: [ fhr ] + resources: !calc partition.resources.run_gfspost + Name: !expand jgfs_post_f{dimval.fhr:02d} + FHRGRP: !expand "{dimidx.fhr+1:03d}" + FHRLST: !expand "f{dimval.fhr:03d}" + FHR: !expand 'f{dimval.fhr:02d}' + HR: !expand '{dimval.fhr:02d}' + more_vars: [ FHRGRP, FHRLST, FHR, HR ] + J_JOB: JGLOBAL_NCEPPOST + Trigger: !Depend jgfs_post_manager.depend("release_post{F:02d}",F=[dimval.fhr]) | up.forecast + ecflow_command: !expand | + export post_times={dimval.fhr:03d} FHRLST={FHRLST} FHRGRP={FHRGRP} + $HOMEgfs/jobs/{J_JOB} + rocoto_command: !expand >- + {rocoto_load_modules} ; + /usr/bin/env post_times={dimval.fhr:03d} &HOMEgfs;/jobs/{J_JOB} + #endfamily post + + #endfamily gfs + + # final - The special "final" task is used in Rocoto to detect when + # all tasks that have to run in a given forecast cycle are + # completed. This is not needed for this workflow, but is mandatory + # anyway. It is needed when Complete directives are present. Such + # directives tell the metascheduler that a job should be marked as + # completed without submitting it if a certain condition is met. + # Rocoto does not have that capability, so it has to be implemented + # using Rocoto's "final" task capability. + + final: !Task + <<: *service_task_template + Disable: !calc not metasched.type=="rocoto" + resources: !calc partition.resources.run_nothing + rocoto_command: /bin/true + model: gfs # useless but required + diff --git a/workflow/platforms/_common.yaml b/workflow/platforms/_common.yaml new file mode 100644 index 00000000000..c0efaeb6884 --- /dev/null +++ b/workflow/platforms/_common.yaml @@ -0,0 +1,16 @@ +platform_common: &global_platform_common + Evaluate: False + default_resources: {} + +partition_common: &global_partition_common + Evaluate: False + resources: !Immediate + - !MergeMapping + - !calc doc.default_resources + - !calc doc.platform.default_resources + - !calc doc.case.get('resources',{}) + - !calc doc.get('user_resources',{}) + - !calc doc.resources_sum + + default_resources: {} + diff --git a/workflow/platforms/_sandbox.yaml b/workflow/platforms/_sandbox.yaml new file mode 100644 index 00000000000..3251393edba --- /dev/null +++ b/workflow/platforms/_sandbox.yaml @@ -0,0 +1,185 @@ +# This file configures the workflow to run on the cray parts of WCOSS + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: sandbox + + # detect: this is a function that returns true iff the user is on GAEA + # and false otherwise + detect: True + + # skip_if_others_present: if this is true, and at least one other + # platform is detected with this flag set to false, then skip this + # platform + skip_if_others_present: true + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /gpfs/hps3/emc/global/noscrub/emc.glopara/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "/gpfs/hps/nco/ops/nwprod" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: !calc doc.user_places.EXPROOT + #"/Users/jiankuang/Documents/Eclipse_workspace/ecfutils_007" + #DMPDIR: !FirstTrue + # - do: "/gpfs/gp1/emc/globaldump" + # when: !calc tools.isdir(do) + # - do: "/gpfs/tp1/emc/globaldump" + # when: !calc tools.isdir(do) + # - otherwise: !error "Cannot find globaldump directory." + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "$CRTM_FIX" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various subversion repositories. This is used on some platforms + # to find executables for this workflow. + BASE_SVN: "/gpfs/hps3/emc/global/noscrub/emc.glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/gpfs/hps3/emc/global/noscrub/emc.glopara/git" + + # config_base_extras - Additional configuration data to put in the + # config.base file + config_base_extras: "sandbox" + #config_base_extras: | + # if [ -d /gpfs/tp1 ]; then + # export SITE="LUNA" + # elif [ -d /gpfs/gp1 ]; then + # export SITE="SURGE" + # fi + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + partitions: + Evaluate: false + default_exclusive: !calc doc.platform.partitions.sandbox + default_service: !calc doc.platform.partitions.sandbox + default_shared: !calc doc.platform.partitions.sandbox + sandbox: + <<: *global_partition_common + + # specification - string to specify to the batch system to + # request this partition. Not relevant for WCOSS Cray + specification: null + + # shared_accounting_ref - accounting settings for shared jobs + shared_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.shared_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # service_accounting_ref - accounting settings for service jobs (jobs + # that require tape or network access) + service_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.service_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # exclusive_accounting_ref - accounting settings for jobs that require + # exclusive access to a node. + exclusive_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.exclusive_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # Queues to use for each job type. This logic automatically + # switches between development queues on the backup machine and + # development queues on the production machine based on whether the + # /gpfs/hps2/ptmp is writable. + shared_queue: dev + service_queue: dev + exclusive_queue: dev + #shared_queue: !FirstTrue + # - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + # do: dev_shared + # - otherwise: devonprod_shared + #service_queue: dev_transfer + #exclusive_queue: !FirstTrue + # - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + # do: dev + # - otherwise: devonprod + + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: LSFAlps + parallelism_name: LSFAlps + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + # Path to mmlsquota, the program used to get GPFS disk usage information: + #mmlsquota: "/usr/lpp/mmfs/bin/mmlsquota" + + # Automatically detect the least used scrub area the user can access: + #least_used_ptmp: !Immediate + # - !FirstMax + # - do: /gpfs/hps3/ptmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps3-ptmp","hps3",mmlsquota) ) + # message: Use {do} for long-term temp. + # - do: /gpfs/hps/ptmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps-ptmp","hps",mmlsquota) ) + # message: Use {do} for long-term temp. + # - do: /gpfs/hps2/ptmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps2-ptmp","hps2",mmlsquota) ) + # message: Use {do} for long-term temp. + #least_used_stmp: !Immediate + # - !FirstMax + # - do: /gpfs/hps/stmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps-stmp","hps",mmlsquota) ) + # message: Use {do} for short-term temp. + # - do: /gpfs/hps2/stmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps2-stmp","hps2",mmlsquota) ) + # message: Use {do} for short-term temp. + # - do: /gpfs/hps3/stmp + # when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"hps3-stmp","hps3",mmlsquota) ) + # message: Use {do} for short-term temp. + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + long_term_temp: !calc doc.user_places.EXPROOT + #!expand "{least_used_ptmp}/{tools.env('USER')}" + + # short_term_temp - area for data that is only needed within one job: + short_term_temp: !calc doc.user_places.EXPROOT + #!expand "{least_used_stmp}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !calc doc.user_places.EXPROOT + #!expand "{doc.user_places.EXPROOT}/noscrub/{tools.env('USER')}" diff --git a/workflow/platforms/gaea.yaml b/workflow/platforms/gaea.yaml new file mode 100644 index 00000000000..d3ad81f0b10 --- /dev/null +++ b/workflow/platforms/gaea.yaml @@ -0,0 +1,214 @@ +# This file configures the workflow to run on GAEA. + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: GAEA + + # detect: this is a function that returns true iff the user is on GAEA + # and false otherwise + detect: !calc tools.isdir("/lustre/f1") + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /lustre/f1/pdata/ncep_shared/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "//dev/null/global/save/glopara/nwpara" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: "/dev/null/global/noscrub/dump" + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "/dev/null/da/save/Michael.Lueken/nwprod/lib/crtm/2.2.3/fix_update" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various repositories. This is used on some platforms to find + # executables for this workflow. + BASE_SVN: "/dev/null/global/save/glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/dev/null/global/save/glopara/git" + + # rocoto_platform_vars - additional variables sent to the rocoto xml + # file for each job to set platform-specific batch card settings. + rocoto_platform_vars: | + + -S /bin/bash + + # config_base_extras - Additional configuration data to put in the + # config.base file + config_base_extras: | + export POSTGRB2TBL=$PARMgfs/g2tmpl-1.5.0/params_grib2_tbl_new + export WGRIB2="aprun -n 1 -j 1 -N 1 -d 1 -cc depth $EXECgfs/wgrib2" + + partition_common: &partition_common + <<: *global_partition_common + # Settings common to all partitions of this machine + + # shared_accounting_ref - accounting settings for shared jobs + shared_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.shared_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # service_accounting_ref - accounting settings for service jobs (jobs + # that require tape or network access) + service_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.service_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # exclusive_accounting_ref - accounting settings for jobs that require + # exclusive access to a node. + exclusive_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.exclusive_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # Queues to use for each job type + shared_queue: batch + service_queue: rdtn + exclusive_queue: batch + + # Generate the actual Python objects for the scheduler, parallelism, + # and nodes: + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + partitions: + Evaluate: false + c4: + <<: *partition_common + specification: c4 + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: MoabAlps + parallelism_name: AprunCrayMPI + node_type: generic + physical_cores_per_node: 36 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + es: + <<: *partition_common + specification: es + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: MoabAlps + parallelism_name: AprunCrayMPI + node_type: generic + physical_cores_per_node: 2 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + + c3: + <<: *partition_common + specification: c3 + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: MoabAlps + parallelism_name: AprunCrayMPI + node_type: generic + physical_cores_per_node: 32 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + c3_or_c4: + <<: *partition_common + specification: 'c3:c4' + # Details about the scheduler on this cluster. + scheduler_settings: &c4_scheduler_settings + scheduler_name: MoabAlps + parallelism_name: AprunCrayMPI + node_type: generic + physical_cores_per_node: 32 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + # Default partitions for each common job type: + default_shared: !calc doc.platform.partitions.c3_or_c4 + default_exclusive: !calc doc.platform.partitions.c3_or_c4 + default_service: !calc doc.platform.partitions.es + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + {metasched.defvar(doc.schedvar.service_partition, "es")} + + # Automatically detect the least used scrub area the user can access: + user_scrub_area: !Immediate + - !FirstTrue + - do: /lustre/f1 + when: !calc tools.can_write(do) + - do: /lustre/f1/ncep + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.esrl.rocoto + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.bgrp-account + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.ccsp-users + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.cm3 + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.cmip6 + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.decp + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.esm2g + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.esm2m + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.fre_test + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.hrao + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.ogrp-account + when: !calc tools.can_write(do) + - do: /lustre/f1/oar.gfdl.ssam + when: !calc tools.can_write(do) + - otherwise: !error "Cannot find project area. Please set manually." + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + long_term_temp: !expand "{user_scrub_area}/{tools.env('USER')}" + + # short_term_temp - area for data that is only needed within one job: + short_term_temp: !expand "{user_scrub_area}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" diff --git a/workflow/platforms/hera.yaml b/workflow/platforms/hera.yaml new file mode 100644 index 00000000000..8b2f74fc80c --- /dev/null +++ b/workflow/platforms/hera.yaml @@ -0,0 +1,173 @@ +# This file configures the workflow to run on Theia + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: HERA + + # detect: this is a function that returns true iff the user is on GAEA + # and false otherwise + detect: !calc tools.isdir("/scratch1") and tools.isdir("/scratch2") + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: "/scratch1/NCEPDEV/global/glopara/dump" + NWPROD: "/scratch1/NCEPDEV/global/glopara/svn/verif/global/tags/vsdb/nwprod" + RTMFIX: "/scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.2.6" + BASE_SVN: "/scratch1/NCEPDEV/global/glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/scratch1/NCEPDEV/global/glopara/git/" + + config_base_extras: !expand | + echo "place holder for NWPROD and associated utilities, now go through load_gfs_modules.sh" + + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + partitions: + Evaluate: false + default_shared: !calc doc.platform.partitions.hera + default_exclusive: !calc doc.platform.partitions.hera + default_service: !calc doc.platform.partitions.hera_service + default_bigmem: !calc doc.platform.partitions.hera_bigmem + hera: + <<: *global_partition_common + specification: hera + # Queues to use for each job type + shared_queue: hera + exclusive_queue: hera + + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 40 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (96*1024) + + exclusive_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: hera + + shared_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: hera + + # Generate the actual Python objects for the scheduler, parallelism, + # and nodes: + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + hera_service: + <<: *global_partition_common + specification: service + service_queue: service + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 1 + logical_cpus_per_core: 1 + hyperthreading_allowed: false + indent_text: " " + memory_per_node: !calc (64*1024) + + service_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: service + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + hera_bigmem: + <<: *global_partition_common + specification: bigmem + service_queue: bigmem + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 24 + hyperthreading_allowed: false + indent_text: " " + memory_per_node: !calc (64*1024) + + bigmem_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: bigmem + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + # Path to pan_df, the program used to get Panasas disk usage information: + pan_df: pan_df + + # Automatically detect the least used scrub area the user can access: + least_used_temp: !Immediate + - !FirstTrue + - do: !expand "{doc.user_places.DATAROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !FirstMax + - do: /scratch1/NCEPDEV/stmp2 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch1/NCEPDEV/stmp4 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch2/NCEPDEV/stmp1 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch2/NCEPDEV/stmp3 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: !expand "{doc.user_places.EXPROOT}" + when: !calc tools.panasas_gb(do)/4 + message: "{do}: use project directory as scrub space" + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + COMROOT: !FirstTrue + - do: !expand "{doc.user_places.COMROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !expand "{doc.platform.least_used_temp}/{tools.env('USER')}" + + # short_term_temp - area for data that is only needed within one job: + DATAROOT: !FirstTrue + - do: !expand "{doc.user_places.DATAROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !expand "{doc.platform.least_used_temp}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" diff --git a/workflow/platforms/jet.yaml b/workflow/platforms/jet.yaml new file mode 100644 index 00000000000..04c524ccb79 --- /dev/null +++ b/workflow/platforms/jet.yaml @@ -0,0 +1,270 @@ +# This file configures the workflow to run on the xJet partition of Jet + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. Note that the name does not specify the jet partition. + name: JET + + # detect: this is a function that returns true iff the user is on GAEA + # and false otherwise + detect: !calc tools.isdir("/lfs3") + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /lfs3/projects/hfv3gfs/Samuel.Trahan/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "/lfs3/projects/hfv3gfs/nwprod" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: "/lfs3/projects/hfv3gfs/glopara/noscrub/dump" + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "/mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/crtm/v2.2.6/fix" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various repositories. This is used on some platforms to find + # executables for this workflow. + BASE_SVN: "/dev/null/global/save/glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/mnt/lfs3/projects/hfv3gfs/glopara/git" + + # config_base_extras - Additional configuration data to put in the + # config.base file + config_base_extras: | + export POSTGRB2TBL=/lfs3/projects/hfv3gfs/glopara/nwpara/lib/g2tmpl/v1.5.0/src/params_grib2_tbl_new + export NDATE="$NWPROD/util/exec/ndate" + export NHOUR="$NWPROD/util/exec/nhour" + export WGRIB="$NWPROD/util/exec/wgrib" + export WGRIB2="$NWPROD/util/exec/wgrib2" + export COPYGB="$NWPROD/util/exec/copygb" + export COPYGB2="$NWPROD/util/exec/copygb2" + export GRBINDEX="$NWPROD/util/exec/grbindex" + export GRB2INDEX="$NWPROD/util/exec/grb2index" + export GRBINDEX2="$NWPROD/util/exec/grb2index" + export CNVGRIB="$NWPROD/util/exec/cnvgrib" + export CNVGRIB21_GFS="$NWPROD/util/exec/cnvgrib" + + # A CFP wrapper is installed as part of the "mpiserial" module on Jet. + export USE_CFP=YES + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + long_term_temp: !expand "{doc.user_places.EXPROOT}/scrub" + + # short_term_temp - area for data that is only needed within one job: + short_term_temp: !calc long_term_temp + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" + + ###################################################################### + + # Jet partitions: + + partition_common: &partition_common + <<: *global_partition_common + specification: none # to be defined later + + shared_queue: batch + service_queue: batch + exclusive_queue: batch + + shared_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.shared_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + partition: !calc metasched.varref(doc.schedvar.partition) + + # service_accounting_ref - accounting settings for service jobs (jobs + # that require tape or network access) + service_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.service_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + partition: service + + # exclusive_accounting_ref - accounting settings for jobs that require + # exclusive access to a node. + exclusive_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.exclusive_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + partition: !calc metasched.varref(doc.schedvar.partition) + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + partitions: + Evaluate: false + default_shared: !calc doc.platform.partitions.xjet + default_exclusive: !calc doc.platform.partitions.kjet + default_service: !calc doc.platform.partitions.service + + tjet_or_wherever: + <<: *partition_common + specification: "tjet:ujet:sjet:vjet:xjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 12 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (23.5*1024) + + bigmem: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "bigmem" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (250*1024) + + xjet: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "xjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (63.5*1024) + + vjet: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "vjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 16 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (63.5*1024) + + kjet: + <<: *partition_common + specification: "kjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 40 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (96*1024) + + sjet: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "sjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 16 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (31.5*1024) + + sjet_or_vjet: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "sjet:vjet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 16 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (31.5*1024) + + service: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "service" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 1 + logical_cpus_per_core: 1 + hyperthreading_allowed: false + indent_text: " " + memory_per_node: !calc (24*1024) + + tjet_or_ujet: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "tjet:ujet" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 12 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (23.5*1024) + + bigmem: + <<: *partition_common + # Details about the scheduler on this cluster. + specification: "bigmem" + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (251.5*1024) diff --git a/workflow/platforms/theia.yaml b/workflow/platforms/theia.yaml new file mode 100644 index 00000000000..485bb26d30f --- /dev/null +++ b/workflow/platforms/theia.yaml @@ -0,0 +1,176 @@ +# This file configures the workflow to run on Theia + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: THEIA + + # detect: this is a function that returns true iff the user is on GAEA + # and false otherwise + detect: !calc tools.isdir("/scratch4") and tools.isdir("/scratch3") + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /scratch4/NCEPDEV/global/noscrub/glopara/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "/scratch4/NCEPDEV/global/save/glopara/nwpara" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: "/scratch4/NCEPDEV/global/noscrub/dump" + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "/scratch4/NCEPDEV/da/save/Michael.Lueken/nwprod/lib/crtm/2.2.3/fix_update" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various repositories. This is used on some platforms to find + # executables for this workflow. + BASE_SVN: "/scratch4/NCEPDEV/global/save/glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/scratch4/NCEPDEV/global/save/glopara/git" + + # config_base_extras - Additional configuration data to put in the + # config.base file. On Theia, we use this to specify the location + # of some programs. + config_base_extras: !expand | + export NDATE="$NWPROD/util/exec/ndate" + export NHOUR="$NWPROD/util/exec/nhour" + export WGRIB="$NWPROD/util/exec/wgrib" + export WGRIB2="/scratch3/NCEPDEV/nwprod/utils/wgrib2.v2.0.6c/wgrib2/wgrib2" + export COPYGB="$NWPROD/util/exec/copygb" + export COPYGB2="$NWPROD/util/exec/copygb2" + export GRBINDEX="$NWPROD/util/exec/grbindex" + export GRB2INDEX="$NWPROD/util/exec/grb2index" + export GRBINDEX2="$NWPROD/util/exec/grb2index" + export CNVGRIB="/apps/cnvgrib/1.4.0/bin/cnvgrib" + export CNVGRIB21_GFS="/apps/cnvgrib/1.4.0/bin/cnvgrib" + export POSTGRB2TBL="/scratch3/NCEPDEV/nwprod/lib/sorc/g2tmpl/params_grib2_tbl_new" + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + partitions: + Evaluate: false + default_shared: !calc doc.platform.partitions.theia + default_exclusive: !calc doc.platform.partitions.theia + default_service: !calc doc.platform.partitions.theia_service + theia: + <<: *global_partition_common + specification: theia + # Queues to use for each job type + shared_queue: theia + exclusive_queue: theia + + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + exclusive_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: theia + + shared_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: theia + + # Generate the actual Python objects for the scheduler, parallelism, + # and nodes: + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + theia_service: + <<: *global_partition_common + specification: service + service_queue: service + scheduler_settings: + scheduler_name: Slurm + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 1 + logical_cpus_per_core: 1 + hyperthreading_allowed: false + indent_text: " " + memory_per_node: !calc (64*1024) + + service_accounting_ref: + queue: batch + project: !calc doc.accounting.cpu_project + partition: service + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + + # Path to pan_df, the program used to get Panasas disk usage information: + pan_df: pan_df + + # Automatically detect the least used scrub area the user can access: + least_used_temp: !Immediate + - !FirstTrue + - do: !expand "{doc.user_places.DATAROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !FirstMax + - do: /scratch3/NCEPDEV/stmp1 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch3/NCEPDEV/stmp2 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch4/NCEPDEV/stmp3 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: /scratch4/NCEPDEV/stmp4 + when: !calc ( int(tools.can_write(do)) and tools.panasas_gb(do) ) + - do: !expand "{doc.user_places.EXPROOT}" + when: !calc tools.panasas_gb(do)/4 + message: "{do}: use project directory as scrub space" + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + long_term_temp: !expand "{doc.platform.least_used_temp}/{tools.env('USER')}" + + # short_term_temp - area for data that is only needed within one job: + short_term_temp: !expand "{doc.platform.least_used_temp}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" diff --git a/workflow/platforms/wcoss_cray.yaml b/workflow/platforms/wcoss_cray.yaml new file mode 100644 index 00000000000..b68439eecdc --- /dev/null +++ b/workflow/platforms/wcoss_cray.yaml @@ -0,0 +1,191 @@ +# This file configures the workflow to run on the cray parts of WCOSS + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: WCOSS_C + + # detect: this is a function that returns true iff the user is on + # WCOSS and false otherwise. Note that all WCOSSes are detected + # from all WCOSSes. + detect: !calc ( tools.isdir("/usrx/local") and tools.isdir("/gpfs") ) + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /gpfs/hps3/emc/global/noscrub/emc.glopara/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "/gpfs/hps/nco/ops/nwprod" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: !FirstTrue + - do: "/gpfs/gp1/emc/globaldump" + when: !calc tools.isdir(do) + - do: "/gpfs/tp1/emc/globaldump" + when: !calc tools.isdir(do) + - otherwise: !error "Cannot find globaldump directory." + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "$CRTM_FIX" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various subversion repositories. This is used on some platforms + # to find executables for this workflow. + BASE_SVN: "/gpfs/hps3/emc/global/noscrub/emc.glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/gpfs/hps3/emc/global/noscrub/emc.glopara/git" + + # config_base_extras - Additional configuration data to put in the + # config.base file + config_base_extras: | + if [ -d /gpfs/tp1 ]; then + export SITE="LUNA" + elif [ -d /gpfs/gp1 ]; then + export SITE="SURGE" + fi + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + partitions: + Evaluate: false + default_exclusive: !calc doc.platform.partitions.wcoss_cray + default_service: !calc doc.platform.partitions.wcoss_cray + default_shared: !calc doc.platform.partitions.wcoss_cray + wcoss_cray: + <<: *global_partition_common + + # specification - string to specify to the batch system to + # request this partition. Not relevant for WCOSS Cray + specification: null + + # shared_accounting_ref - accounting settings for shared jobs + shared_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.shared_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # service_accounting_ref - accounting settings for service jobs (jobs + # that require tape or network access) + service_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.service_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # exclusive_accounting_ref - accounting settings for jobs that require + # exclusive access to a node. + exclusive_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.exclusive_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # Queues to use for each job type. This logic automatically + # switches between development queues on the backup machine and + # development queues on the production machine based on whether the + # /gpfs/hps2/ptmp is writable. + shared_queue: !FirstTrue + - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + do: dev_shared + - otherwise: devonprod_shared + service_queue: dev_transfer + exclusive_queue: !FirstTrue + - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + do: dev + - otherwise: devonprod + + # Details about the scheduler on this cluster. + scheduler_settings: + scheduler_name: LSFAlps + parallelism_name: LSFAlps + node_type: generic + physical_cores_per_node: 24 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (64*1024) + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + # Path to mmlsquota, the program used to get GPFS disk usage information: + mmlsquota: "/usr/lpp/mmfs/bin/mmlsquota" + + gpfs_hps: !Immediate + - !FirstTrue + - take: hps + when: !calc tools.isfile('/etc/SuSE-release') + - otherwise: gpfs-hps + + # Automatically detect the least used scrub area the user can access: + least_used_ptmp: !Immediate + - !FirstTrue + - do: !expand "{doc.user_places.COMROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !FirstMax + - do: /gpfs/hps3/ptmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps3-ptmp",gpfs_hps+"3",mmlsquota,-1) ) + message: Use {do} for long-term temp. + - do: /gpfs/hps/ptmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps-ptmp",gpfs_hps,mmlsquota,-1) ) + message: Use {do} for long-term temp. + - do: /gpfs/hps2/ptmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps2-ptmp",gpfs_hps+"2",mmlsquota,-1) ) + message: Use {do} for long-term temp. + + least_used_stmp: !Immediate + - !FirstTrue + - do: !expand "{doc.user_places.DATAROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !FirstMax + - do: /gpfs/hps/stmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps-stmp",gpfs_hps,mmlsquota,-1) ) + message: Use {do} for short-term temp. + - do: /gpfs/hps2/stmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps2-stmp",gpfs_hps+"2",mmlsquota,-1) ) + message: Use {do} for short-term temp. + - do: /gpfs/hps3/stmp + when: !calc ( int(tools.can_write(take)) * tools.gpfs_gb(take,"hps3-stmp",gpfs_hps+"3",mmlsquota,-1) ) + message: Use {do} for short-term temp. + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + COMROOT: !FirstTrue + - do: !expand "{doc.user_places.COMROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !expand "{doc.platform.least_used_ptmp}/{tools.env('USER')}" + + DATAROOT: !FirstTrue + - do: !expand "{doc.user_places.DATAROOT}" + when: !calc doc.user_places.FIX_SCRUB + - otherwise: !expand "{doc.platform.least_used_stmp}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" diff --git a/workflow/platforms/wcoss_dell_p3.yaml b/workflow/platforms/wcoss_dell_p3.yaml new file mode 100644 index 00000000000..d1a459a6b51 --- /dev/null +++ b/workflow/platforms/wcoss_dell_p3.yaml @@ -0,0 +1,223 @@ +# This file configures the workflow to run on the dell parts of WCOSS + +platform: !Platform + <<: *global_platform_common + + # Evaluate: this must be "false" to ensure disk space availability logic + # is not run unless this file is for the current platform. + Evaluate: false + + # name: the name of this platform; this must match what the underlying + # scripts expect. + name: WCOSS_DELL_P3 + + # detect: this is a function that returns true iff the user is on + # WCOSS and false otherwise. Note that all WCOSSes are detected + # from all WCOSSes. + detect: !calc ( tools.isdir("/usrx/local") and tools.isdir("/gpfs") ) + + # public_release_ics: location of input conditions that have been + # prepared for the public release. + public_release_ics: /gpfs/hps3/emc/global/noscrub/emc.glopara/FV3GFS_V1_RELEASE/ICs + + # CHGRP_RSTPROD_COMMAND - this specifies the command to use to + # restrict access to NOAA "rstprod" data restriction class. + # This only used for observation processing, data assimilation, and + # data assimilation archiving, which are not in the public release. + CHGRP_RSTPROD_COMMAND: "chgrp rstprod" + + # NWPROD - location of the NCEP operational "nwprod" directory, which + # only has meaning on the NCEP WCOSS machines. It is used to get + # the paths to certain programs and scripts. + NWPROD: "/gpfs/dell1/nco/ops/nwprod" + + # DMPDIR - location of the global dump data. This is used by the observation + # processing scripts, which are not included in the public release. + DMPDIR: !FirstTrue + - do: "/gpfs/gp1/emc/globaldump" + when: !calc tools.isdir(do) + - do: "/gpfs/tp1/emc/globaldump" + when: !calc tools.isdir(do) + - otherwise: !error "Cannot find globaldump directory." + + # RTMFIX - location of the CRTM fixed data files used by the GSI data + # assimilation. The data assimilation is not included in this public release + # so this path is unused. + RTMFIX: "$CRTM_FIX" + + # BASE_SVN - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various subversion repositories. This is used on some platforms + # to find executables for this workflow. + BASE_SVN: "/gpfs/dell2/emc/modeling/noscrub/emc.glopara/svn" + + # BASE_GIT - a directory maintained by emc global model developers + # that contains recent versions of source code and executables for + # various git repositories. This is used on some platforms to find + # executables for this workflow. + BASE_GIT: "/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git" + + # config_base_extras - Additional configuration data to put in the + # config.base file + config_base_extras: | + if [ -d /gpfs/tp1 ]; then + export SITE="MARS" + elif [ -d /gpfs/gp1 ]; then + export SITE="VENUS" + fi + + partition_common: &partition_common + <<: *global_partition_common + # Settings common to all partitions of this machine + + # shared_accounting_ref - accounting settings for shared jobs + shared_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.shared_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # service_accounting_ref - accounting settings for service jobs (jobs + # that require tape or network access) + service_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.service_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # exclusive_accounting_ref - accounting settings for jobs that require + # exclusive access to a node. + exclusive_accounting_ref: + queue: !calc metasched.varref(doc.schedvar.exclusive_queue) + project: !calc metasched.varref(doc.schedvar.cpu_project) + + # Queues to use for each job type. This logic automatically + # switches between development queues on the backup machine and + # development queues on the production machine based on whether the + # /gpfs/hps2/ptmp is writable. + shared_queue: !FirstTrue + - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + do: dev_shared + - otherwise: devonprod_shared + service_queue: dev_transfer + exclusive_queue: !FirstTrue + - when: !calc ( tools.can_write("/gpfs/hps2/ptmp") ) + do: dev + - otherwise: devonprod + + scheduler: !calc | + tools.get_scheduler(scheduler_settings.scheduler_name, scheduler_settings) + parallelism: !calc | + tools.get_parallelism(scheduler_settings.parallelism_name, scheduler_settings) + nodes: !calc | + tools.node_tool_for(scheduler_settings.node_type, scheduler_settings) + + partitions: + Evaluate: false + exclusive_p3: + <<: *partition_common + specification: exclusive_p3 + # Details about the scheduler on this cluster + scheduler_settings: + scheduler_name: LSF + use_task_geometry: False + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 28 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (120*1024) + specify_memory: False + + shared_p3: + <<: *partition_common + specification: shared_p3 + # Details about the scheduler on this cluster + scheduler_settings: + scheduler_name: LSF + use_task_geometry: False + parallelism_name: HydraIMPI + node_type: generic + physical_cores_per_node: 1 + logical_cpus_per_core: 2 + hyperthreading_allowed: true + indent_text: " " + memory_per_node: !calc (120*1024) + specify_memory: True + + default_exclusive: !calc doc.platform.partitions.exclusive_p3 + default_service: !calc doc.platform.partitions.shared_p3 + default_shared: !calc doc.platform.partitions.shared_p3 + + # Additional variables to send to Rocoto XML entities or ecflow edits. + metasched_more: !expand | + {metasched.defvar(doc.schedvar.exclusive_queue, doc.accounting.exclusive_partition.exclusive_queue)} + {metasched.defvar(doc.schedvar.shared_queue, doc.accounting.shared_partition.shared_queue)} + {metasched.defvar(doc.schedvar.service_queue, doc.accounting.service_partition.service_queue)} + {metasched.defvar(doc.schedvar.cpu_project, doc.accounting.cpu_project)} + + # Path to mmlsquota, the program used to get GPFS disk usage information: + mmlsquota: "/usr/lpp/mmfs/bin/mmlsquota" + + # Automatically detect the least used scrub area the user can access: + least_used_ptmp: !Immediate + - !FirstMax + - do: /gpfs/dell3/ptmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell3-ptmp","gpfs-dell3",mmlsquota) ) + message: Use {do} for long-term temp. + - do: /gpfs/dell1/ptmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell1-ptmp","gpfs-dell1",mmlsquota) ) + message: Use {do} for long-term temp. + - do: /gpfs/dell2/ptmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell2-ptmp","gpfs-dell2",mmlsquota) ) + message: Use {do} for long-term temp. + least_used_stmp: !Immediate + - !FirstMax + - do: /gpfs/dell1/stmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell1-stmp","gpfs-dell1",mmlsquota) ) + message: Use {do} for short-term temp. + - do: /gpfs/dell2/stmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell2-stmp","gpfs-dell2",mmlsquota) ) + message: Use {do} for short-term temp. + - do: /gpfs/dell3/stmp + when: !calc ( int(tools.can_write(do)) * tools.gpfs_gb(do,"dell3-stmp","gpfs-dell3",mmlsquota) ) + message: Use {do} for short-term temp. + + # long_term_temp - area for storage of data that must be passed + # between jobs or shared with programs external to this workflow. + long_term_temp: !expand "{least_used_ptmp}/{tools.env('USER')}" + + # short_term_temp - area for data that is only needed within one job: + short_term_temp: !expand "{least_used_stmp}/{tools.env('USER')}" + + # EXPROOT - Parent directory of the expdir (experiment directory) + EXPROOT: !expand "{doc.user_places.EXPROOT}" + + ecflow_module: "ecflow/4.7.1" + prod_util_module: "prod_util/1.1.0" + + # four_cycle_mode_modules: this is inserted in every ecf file when + # only four ecflow suites are generated. For all other situations, + # the load_modules.sh is used. + four_cycle_mode_modules: | + module load ips/$ips_ver + module load impi/$impi_ver + module load EnvVars/$EnvVars_ver + + module load g2tmpl/$g2tmpl_ver + module load crtm/$crtm_ver + module load ESMF/$ESMF_ver + module load dev/util_shared/$util_shared_ver # NOTE: remove dev/ after util_shared upgrade + + module unload grib_util + module load dev/grib_util/$grib_util_ver # NOTE: remove dev/ after grib_util upgrade + + module load NCO/$NCO_ver + module load HDF5-serial/$HDF5_serial_ver + module load NetCDF/$NetCDF_ver + module load CFP/$CFP_ver + export USE_CFP=YES + + module use /gpfs/dell1/nco/ops/nwpara/modulefiles/ + module load gempak/$gempak_ver + + module load bufr_dumplist/$bufr_dumplist_ver + module load dumpjb/$dumpjb_ver + module load NCL/$NCL_ver diff --git a/workflow/runtime/_main.yaml b/workflow/runtime/_main.yaml new file mode 100644 index 00000000000..1338819e577 --- /dev/null +++ b/workflow/runtime/_main.yaml @@ -0,0 +1,6 @@ +include: + - rocoto.yaml + - task.yaml + - experiment_include.yaml + - suite.yaml + - "*.yaml" diff --git a/workflow/runtime/cycle_end.yaml b/workflow/runtime/cycle_end.yaml new file mode 100644 index 00000000000..546b2152c8d --- /dev/null +++ b/workflow/runtime/cycle_end.yaml @@ -0,0 +1,41 @@ +cycle_end_task: !Task &cycle_end_task + <<: *shared_task_template + ecflow_def: !expand | + edit ECF_PASS 'FREE' + + ecf_file: | + #! /bin/sh + which ecflow_client + export ECF_PASS=%ECF_PASS% + export ECF_RID=%ECF_RID% + export ECF_HOST=%ECF_HOST% + export ECF_NAME=%ECF_NAME% + ecflow_client --complete + echo Cycle complete at $( date ) + + Disable: !calc ( not doc.settings.four_cycle_mode or not metasched.type=="ecflow" ) + + ecflow_def: !FirstTrue + - when: !calc suite.Clock.now.strftime("%H")=='00' + take: | + edit ECF_JOB_CMD '%ECF_JOB% 1> %ECF_JOBOUT% 2>&1' + edit ECF_KILL_CMD 'kill -15 %ECF_RID%' + time 23:00 + - when: !calc suite.Clock.now.strftime("%H")=='06' + take: | + edit ECF_JOB_CMD '%ECF_JOB% 1> %ECF_JOBOUT% 2>&1' + edit ECF_KILL_CMD 'kill -15 %ECF_RID%' + time 23:01 + - when: !calc suite.Clock.now.strftime("%H")=='12' + take: | + edit ECF_JOB_CMD '%ECF_JOB% 1> %ECF_JOBOUT% 2>&1' + edit ECF_KILL_CMD 'kill -15 %ECF_RID%' + time 11:00 + - when: !calc suite.Clock.now.strftime("%H")=='18' + take: | + edit ECF_JOB_CMD '%ECF_JOB% 1> %ECF_JOBOUT% 2>&1' + edit ECF_KILL_CMD 'kill -15 %ECF_RID%' + time 11:01 + - otherwise: !error "cycle_end only works for hours 0, 6, 12, and 18" + + resources: !calc partition.resources.run_nothing diff --git a/workflow/runtime/dump_waiter.yaml b/workflow/runtime/dump_waiter.yaml new file mode 100644 index 00000000000..f14929d8482 --- /dev/null +++ b/workflow/runtime/dump_waiter.yaml @@ -0,0 +1,48 @@ +dump_waiter: &dump_waiter_task !Task + <<: *exclusive_task_template + manual: | + The dump_waiter task is a special job needed to run this + workflow outside of the production suite. It handles the + external dependency on the dump and the associated + (approximate) time dependency. + In ecflow: + The dump_waiter is started when the cycle is allowed to + begin, and waits for the dump to be available. It sets + the "updated_status" event when the dump is available. + In Rocoto: + The dump_waiter is never run; it is defined so that the + updated_status data event can be used as a data dependency + in the prep job. + + Disable: !calc metasched.type=='rocoto' + + resources: !ref partition.resources.run_dump_waiter + + updated_status: !DataEvent + file: !expand >- + {doc.places.DMPDIR}/@Y@m@d@H/{up.CDUMP}/{up.CDUMP}.t@Hz.updated.status.tm00.bufr_d + + ecflow_WAITFILE: !expand '%DUMPDIR%/${{PDY}}%CYC%/{CDUMP}/{CDUMP}.t%CYC%z.updated.status.tm00.bufr_d' + + # The ecflow_command is dumped into the ecf file where the + # J-Job would normally be called. This job is never run by + # Rocoto, so we don't need the equivalent Rocoto code. + ecflow_command: !expand | + workdir=/tmp/$USER.dump_waiter.$$.$RANDOM + mkdir -p "$workdir" + cd "$workdir" + export cycle="t${{cyc}}z" + export jlogfile=${{jlogfile:-$COMROOT/logs/jlogfiles/jlogfile.${{job}}.${{pid}}}} + export job=${{job:-"anal"}} + setpdy.sh + source ./PDY + WAITFILE={ecflow_WAITFILE} + while [[ ! -s "$WAITFILE" ]] ; do + echo "$WAITFILE: nope" + sleep 37 + done + env | grep ECF_ + echo "$WAITFILE: yup" + ecflow_client --event updated_status + cd / + rm -rf "$workdir" diff --git a/workflow/runtime/experiment_include.yaml b/workflow/runtime/experiment_include.yaml new file mode 100644 index 00000000000..7db4a38021e --- /dev/null +++ b/workflow/runtime/experiment_include.yaml @@ -0,0 +1,36 @@ +ecf_include_experiment: + + # Generates the experiment-XXX.h file that is loaded by the ecf + # files. See the ecf_model_include variable in runtime/task.yaml + + # filename: name of the model-ver.h to make: + filename: !expand "experiment-{doc.names.experiment}.h" + + # disable: if True, this file will not be generated: + disable: False # Maybe should be "true" for NCO? + + # content: what goes in the file + content: !expand | + # These are development overrides. They are intended to set + # variables that would otherwise be set to operational values, and + # cause your workflow to fail. + + # EMC experiment configuration file + export HOMEgfs="{doc.places.HOMEgfs}" + export HOMEobsproc_global="{doc.places.HOMEobsproc_network}" + export HOMEobsproc_network="{doc.places.HOMEobsproc_network}" + export HOMEobsproc_prep="{doc.places.HOMEobsproc_prep}" + export HOMEgfs="{doc.places.HOMEgfs}" + export DATAROOT="{doc.places.DATAROOT}" + export jlogfile="{doc.places.ROTDIR}/jlogfile" + export COMROOT="{doc.places.ROTDIR}" + export CDUMP=$RUN + export cycle=t%CYC%z + export EXPDIR="{doc.places.EXPDIR}" + {date_vars} + + # Send in the CDATE if we're not in four cycle mode: + date_vars: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: '' + - otherwise: "export CDATE=%PDY%%CYC%" diff --git a/workflow/runtime/fax.yaml b/workflow/runtime/fax.yaml new file mode 100644 index 00000000000..7db5073273b --- /dev/null +++ b/workflow/runtime/fax.yaml @@ -0,0 +1,41 @@ +fax_wafs_task_template: &fax_wafs_task_template !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_small_downstream + + # The ecflow_command is dumped into the ecf file where the + # J-Job would normally be called. This job is never run by + # Rocoto, so we don't need the equivalent Rocoto code. + ecflow_command: !expand | + export fhr_list='{fhr_list}' + export fcsthrs=%FCSTHR% + + ########################################################### + # CALL executable job script here + ${{HOMEgfs}}/jobs/JGFS_FAX_WAFS + + rocoto_more_vars: !expand | + fhr_list{fhr_list} + FCSTHR{FCSTHR} + + ecflow_def: !expand "edit FCSTHR '{FCSTHR}'" + +fax_task_template: &fax_task_template !Task + <<: *exclusive_task_template + resources: !calc partition.resources.run_small_downstream + + # The ecflow_command is dumped into the ecf file where the + # J-Job would normally be called. This job is never run by + # Rocoto, so we don't need the equivalent Rocoto code. + ecflow_command: !expand | + export fcsthrs=%FCSTHR% + + ########################################################### + # CALL executable job script here + ${{HOMEgfs}}/jobs/JGFS_FAX + + rocoto_more_vars: !expand | + FCSTHR{FCSTHR} + + ecflow_def: !expand "edit FCSTHR '{FCSTHR}'" + + config_list: [ base, fax, wafs ] diff --git a/workflow/runtime/make_next_cycles.yaml b/workflow/runtime/make_next_cycles.yaml new file mode 100644 index 00000000000..ed4c49d3ee5 --- /dev/null +++ b/workflow/runtime/make_next_cycles.yaml @@ -0,0 +1,72 @@ +make_next_cycles: &make_next_cycles_task !Task + <<: *exclusive_task_template + manual: | + The make_next_cycles task generates and begins the suites for the + next few cycles in the workflow. This job is only used when running + in ecflow. + + Disable: !calc metasched.type=='rocoto' + + resources: !ref partition.resources.run_make_next_cycles + + Trigger: !Depend gdas + + CDUMP: gdas # useless but required + + # The batch_job_command is dumped into the ecf file where the + # J-Job would normally be called. This job is never run by + # Rocoto, so Rocoto never uses this code block. + ecflow_command: !expand | + workdir=/tmp/$USER.dump_waiter.$$.$RANDOM + mkdir -p "$workdir" + cd "$workdir" + if [[ "$ECF_ROOT" == X ]] ; then + export ECF_ROOT=$( cd "$ECF_HOME" ; cd .. ; pwd -P ) + fi + + export PDY=%PDY% + export CYC=%CYC% + export cycle=t%CYC%z + setpdy.sh + source ./PDY + + module load {doc.settings.prod_util_module} + module load {doc.settings.ecflow_module} + + export WORKFLOW_FIRST_CYCLE=%WORKFLOW_FIRST_CYCLE:1970010100% + export WORKFLOW_LAST_CYCLE=%WORKFLOW_LAST_CYCLE:ETERNITY% + export WORKFLOW_CYCLES_TO_GENERATE=%WORKFLOW_CYCLES_TO_GENERATE:5% + export WORKFLOW_CROW_HOME=%WORKFLOW_CROW_HOME% + export WORKFLOW_EXPDIR=%WORKFLOW_EXPDIR% + export ECF_HOME=%ECF_HOME% + export ECF_ROOT=%ECF_ROOT:X% + export ECF_OUT=%ECF_OUT% + export ECF_PORT=%ECF_PORT% + export ECF_HOST=%ECF_HOST% + + set -uex + + # Decide the range of cycles to start + first_cycle=$( $NDATE +6 "%PDY%%CYC%" ) + + if [[ "${{WORKFLOW_LAST_CYCLE:-ETERNITY}}" != ETERNITY && + "$first_cycle" -gt "$WORKFLOW_LAST_CYCLE" ]] ; then + postmsg "Last cycle reached. Not starting any new cycles." + fi + + last_cycle=$first_cycle + for istart in $( seq 1 5 ) ; do + if [[ "${{WORKFLOW_LAST_CYCLE:-ETERNITY}}" != ETERNITY && + "$last_cycle" -gt "$WORKFLOW_LAST_CYCLE" ]] ; then + break + fi + last_cycle=$( $NDATE +6 "$last_cycle" ) + done + + cd "$WORKFLOW_CROW_HOME" + postmsg "Start cycles $first_cycle through $last_cycle." + ./update_ecflow_workflow.sh -v "$WORKFLOW_EXPDIR" "$first_cycle" "$last_cycle" + + + cd / + rm -rf "$workdir" diff --git a/workflow/runtime/post_manager.yaml b/workflow/runtime/post_manager.yaml new file mode 100644 index 00000000000..59c8585437d --- /dev/null +++ b/workflow/runtime/post_manager.yaml @@ -0,0 +1,28 @@ +# This file contains extra shell script contents of the ecf file for +# the post_manager job. That is the job in the workflow that watches +# for post output files to show up, and sets ecflow events every time +# it sees a new file. + +post_manager_job_contents: &post_manager_job_contents !expand | + fhrs='anl {tools.join(up.Dimensions.fhr," ")}' + CDATE=%PDY%%CYC% + for fid in $fhrs ; do + if [[ "$fid" == anl ]] ; then + fhr3=000 + event=release_postanl + else + fhr3=$( printf %%03d $fid ) + fhr2=$( printf %%02d $fid ) + event=release_post$fhr2 + fi + + file=%COM%/$CDUMP.%PDY%/%CYC%/$CDUMP.t%CYC%z.logf$fhr3.nemsio + + while [[ ! -s "$file" ]] ; do + echo "$file: nope" + sleep 33 + done + echo "$file: yup" + ecflow_client --event "$event" + done + echo "done" diff --git a/workflow/runtime/rocoto.yaml b/workflow/runtime/rocoto.yaml new file mode 100644 index 00000000000..b43389fa043 --- /dev/null +++ b/workflow/runtime/rocoto.yaml @@ -0,0 +1,2 @@ +# This file is not used yet. Rocoto settings reside elsewhere in the +# runtime/ and workflow/ directories. diff --git a/workflow/runtime/suite.yaml b/workflow/runtime/suite.yaml new file mode 100644 index 00000000000..fd521f081d5 --- /dev/null +++ b/workflow/runtime/suite.yaml @@ -0,0 +1,201 @@ +# This file contains convenient variables to include in workflow +# suites in the workflow/ directory. Most of these variables are +# specific to the cycled workflow which is not in this release. + +# gfs_clock_1 - run GFS only at 0 UTC daily +gfs_clock_1: !Clock + start: !calc tools.day_of(suite.Clock.start+suite.Clock.step) + step: !timedelta "24:00:00" + +# gfs_clock_2 - run GFS only at 0 UTC and 12 UTC daily +gfs_clock_2: !Clock + start: !calc tools.day_of(suite.Clock.start+suite.Clock.step) + step: !timedelta "12:00:00" + +# gfs_clock_4 - run GFS every cycle +gfs_clock_4: !Clock + start: !FirstTrue + - when: !calc doc.places.workflow_file == "layout/free_forecast_gfs.yaml" + do: !calc suite.Clock.start + - otherwise: !calc suite.Clock.start+suite.Clock.step + step: !timedelta "06:00:00" + +# cycled_suite_alarms - this is used to define which cycles various +# jobs run in. In the workflow suite definition, jobs may have an +# AlarmName defined. That AlarmName refers to one of the clocks in +# this section. +cycled_suite_alarms: &cycled_suite_alarms + # first - first cycle only + first: !Clock + start: !calc suite.Clock.start + end: !calc suite.Clock.start + step: !calc suite.Clock.step + + # gdas - cycles for which the GDAS is run + gdas: !Clock + start: !calc suite.Clock.start + end: !calc suite.Clock.end + step: !calc suite.Clock.step + + # gfs - cycles for which the gfs is run + gfs: !FirstTrue + - when: !calc doc.settings.gfs_cyc == 0 + do: !Clock # required but ignored because gfs is disabled + start: !calc suite.Clock.start + end: !calc suite.Clock.end + step: !calc suite.Clock.step + - when: !calc doc.settings.gfs_cyc == 1 + do: !calc suite.Clock.for_alarm(doc.gfs_clock_1) + - when: !calc doc.settings.gfs_cyc == 2 + do: !calc suite.Clock.for_alarm(doc.gfs_clock_2) + - when: !calc doc.settings.gfs_cyc == 4 + do: !calc suite.Clock.for_alarm(doc.gfs_clock_4) + - otherwise: !error Unknown gfs clock settings {doc.settings.gfs_cyc} + + gfs_00_12: !calc suite.Clock.for_alarm(doc.gfs_clock_2) + + +# suite_defaults - this is inherited by the actual suite definitions +# It sets defaults for various variables that are generally not overridden +suite_defaults: &suite_defaults + # Clock - the list of cycles to run + Clock: !Clock + # NEED TO FIX FOR LESS FREQUENT GFS + start: !calc doc.settings.SDATE + end: !calc doc.settings.EDATE + step: !timedelta "6:00:00" + + Overrides: + allowed: [ 'queue', 'partition', 'resources', 'rocoto_command', 'rocoto_log_path', + 'accounting', 'ecflow_command', 'max_tries', 'resources_remap', 'ecf_log_path', + 'Rocoto', 'ecf_file' ] + rules: !calc doc.suite_overrides + + ecf_file_set: + disk_path: !expand "{doc.places.ECF_ROOT}/scripts/{doc.names.experiment}" + ECF_FILES: !expand "{doc.places.ECF_ROOT}/scripts/{doc.names.experiment}" + + # ecFlow - definition of variables specific to ecFlow suite generation + ecFlow: + # suite_def_filename - file to receive the suite definition + suite_def_filename: !expand "{doc.places.ECF_ROOT}/defs/{doc.names.experiment}/{suite.ecFlow.suite_name}.def" + + # suite_name - name of the suite (the top-level node in the suite definition) + suite_name: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: !expand "{doc.names.experiment}%H" + - otherwise: !expand "{doc.names.experiment}_%Y%m%d%H" + + # dates_in_time_dependencies - flag that controls whether ecflow + # suite definition "date" triggers are to be defined. + dates_in_time_dependencies: !calc (not doc.settings.four_cycle_mode) + + # write_cycles/analyze_cycles: Use these to generate a subset of + # the cycles in ecFlow. Make sure analyze_cycles starts at least + # one cycle before write_cycles so that dependencies are + # processed. These are overridden by update_ecflow_workflow.py + # during its execution. + + # write_cycles: !Clock + # start: 2018-01-02T00:00:00 + # end: 2018-01-02T18:00:00 + # step: !timedelta "6:00:00" + # analyze_cycles: !Clock + # start: 2018-01-02T00:00:00 + # end: 2018-01-02T18:00:00 + # step: !timedelta "6:00:00" + + # ecflow_cycling_logic - used to switch between NCO-style suite + # definitions, with four suites total (one per synoptic time) and + # development-style (one suite per cycle). + ecflow_cycling_logic: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + do: "repeat day 1" + - otherwise: !expand | + autocancel 5 + edit PDY '{tools.strftime(suite.Clock.now,"%Y%m%d")}' + + # Rocoto - contains settings specific to Rocoto workflow generation + Rocoto: &Rocoto + # max_tries - maximum number of times to run a job before giving + # up. Set to 1 to disable automatic resubmission of jobs. + max_tries: !calc doc.settings.max_job_tries + + # workflow_xml - the contents of this variable are written to the Rocoto XML file. + workflow_xml: !expand | + + + + + + + + + + + + + {tools.indent(doc.platform.metasched_more," ")} + ]> + + &COM;/logs/@Y@m@d@H/rocoto.log + {to_rocoto.make_time_xml(indent=1)} + {to_rocoto.make_task_xml(indent=1)} + + + # before_suite_def - when generating an ecFlow suite, the contents + # of this variable are written before anything else in the suite + # definition. The totality limit line sets an external dependency + # on a limit that has the effect of throttling the number of jobs + # the entire ecFlow server can run at a time. + before_suite_def: !FirstTrue + - when: doc.settings.ecflow_totality_limit + do: "extern /totality_limit:TOTALITY" + - otherwise: "" + + ecflow_COM_edit: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: !calc doc.places.COMROOT + - otherwise: !calc doc.places.ROTDIR + + ecflow_def: !expand | + {ecflow_cycling_logic} + edit ECF_TRIES '{doc.settings.max_job_tries}' + # edit ECF_HOME '{doc.places.ECF_HOME}' + edit CYC '{tools.strftime(suite.Clock.now,"%H")}' + edit MACHINE '{doc.accounting.ecflow_machine}' + edit ENVIR 'prod' + edit ECF_OUT '{doc.places.ECF_OUT}' + edit ECF_LOG '{doc.places.ECF_ROOT}/ecf.log' + edit E 'j' + edit ECF_INCLUDE '{doc.places.ECF_INCLUDE}' + edit COM '{ecflow_COM_edit}' + + {"clock real" if doc.settings.ecflow_real_clock else ""} + {"clock virtual" if doc.settings.ecflow_virtual_clock else ""} + {"clock hybrid" if doc.settings.ecflow_hybrid_clock else ""} + {common_metasched_vars} + {doc.platform.metasched_more} + {emc_overrides} + + emc_overrides: !FirstTrue + - when: !calc doc.settings.nco_mode + take: '' + - otherwise: !expand | + # EMC overrides. Should remove before implementation: + edit DATAROOT '{doc.places.DATAROOT}' + edit NWROOT '{tools.dirname(doc.places.HOMEgfs)}' + + {"inlimit /totality_limit:TOTALITY" if doc.settings.ecflow_totality_limit else ""} + + # common_metasched_vars - variables to write to both the ecFlow + # suite definitions ("edit" variables) and Rocoto XML (XML + # entities). The conversion to text is done by the + # CROW/crow/metascheduler/ python package. + + common_metasched_vars: '' + diff --git a/workflow/runtime/task.yaml b/workflow/runtime/task.yaml new file mode 100644 index 00000000000..d3b0967896e --- /dev/null +++ b/workflow/runtime/task.yaml @@ -0,0 +1,327 @@ +# This file defines defaults for tasks in the suite definitions. +# Individual tasks in the suite may override some settings + +eoln: "\n" +parm_config_source_line: "source $HOMEgfs/parm/config/config.%s" +expdir_config_source_line: "source $EXPDIR/parm/config/config.%s" + +task_template: &task_template + + # Template - specifies a schema that is used to validate the variables in this scope. + Template: *task_schema + + # Together, accounting and default_accounting set the accounting + # information, such as queues, projects, and partitions. + + default_accounting: {} + accounting: {} + + # partition_specification - this is passed into the + # rocoto_accounting and batch_accounting functions to set the + # partition to request (-l partition=) if needed. + partition_specification: !calc | + {'partition':partition.get('specification',None)} + + # rocoto_load_modules - commands to run before passing control to + # the j-jobs (jobs/ directory) in Rocoto: + rocoto_load_modules: !expand >- + {doc.platform.get("rocoto_load_modules_extra","")} + source $HOMEgfs/ush/load_fv3gfs_modules.sh {task_type} ; + module list + + # rocoto_command - command rocoto executes to pass control to the + # j-job (jobs/ directory) for this task. + rocoto_command: !expand >- + {rocoto_load_modules} ; + {rocoto_config_source} ; + {J_JOB_PATH}/{J_JOB} + + config_list: [ base ] + + rocoto_config_source: !FirstTrue + - when: !calc not config_list + take: "" + - otherwise: !calc '" ; ".join([ "source $EXPDIR/config.%s"%(x,) for x in config_list ])' + + ecflow_config_source: !FirstTrue + - when: !calc not config_list + take: "" + - when: !calc doc.settings.four_cycle_mode + take: !expand | + export HOMEgfs=${{HOMEgfs:-${{NWROOT:?}}/gfs.${{gfs_ver:?"###FATAL ERROR gfs_ver is not set"}}}} + {doc.eoln.join([ "source $HOMEgfs/parm/config/config.%s"%(x,) for x in config_list ])} + - otherwise: !calc | + doc.eoln.join([ "source $EXPDIR/config.%s"%(x,) for x in config_list ]) + + # ecflow_command - command ecFlow executes from the ecf file to pass + # control to the j-job (jobs/ directory) for this task. + ecflow_command: !expand "{J_JOB_PATH}/{J_JOB}" + + J_JOB_PATH: '$HOMEgfs/jobs' + + # rocoto_log_path: contents of the entry of the which + # sets the stdout and stderr log path in the Rocoto XML. + rocoto_log_path: !ref doc.settings.rocoto_log_path + + # ecflow_log_path: Used in the #BSUB or #PBS lines to specify the + # path. Usually will include %VARS% for the ecflow server to parse. + ecflow_log_path: !ref doc.settings.ecf_log_path + + # Rocoto - contents of the Rocoto tag for this task, + # excluding the tag, which is automatically generated + # in CROW/crow/metascheduler/rocoto.py + Rocoto: !expand | + sh -c '{rocoto_command}' + {partition.scheduler.rocoto_accounting( + partition_specification,default_accounting,accounting, + jobname=task_path_var, + outerr=rocoto_log_path, + partition=partition.specification)} + {partition.scheduler.rocoto_resources(resources)} + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + EXPDIR&EXPDIR; + CDUMP{CDUMP} + RUN_ENVIRemc + DATAROOT&DATAROOT; + HOMEgfs{metasched.varref(doc.schedvar.script_home)} + HOMEobsproc_network{metasched.varref(doc.schedvar.obsproc_network_home)} + HOMEobsproc_global{metasched.varref(doc.schedvar.obsproc_network_home)} + HOMEobsproc_prep{metasched.varref(doc.schedvar.obsproc_prep_home)} + job{task_path_list[-1]}_@H + {rocoto_more_vars} + {rocoto_platform_vars} + + # rocoto_platform_vars - allows the platform definition file + # (platforms/*.yaml) to add tags to the Rocoto + # definition. + rocoto_platform_vars: !calc " doc.platform.get('rocoto_platform_vars','') " + + # RUN - Used in ecf files to specify whether the task is gfs or gdas + RUN: !calc task_path_list[0] + + # CDUMP - alias for RUN expected by development scripts + CDUMP: !calc RUN + + # more_vars - Additional environment variables to pass to the j-job + # through the Rocoto XML or ecFlow ecf files. This variable is only used + # if the task does not override the ecf_more_exports, rocoto_more_vars, + # or ecflow_def_more_vars variables. + more_vars: {} + + # ecf_more_exports - additional shell "export" statements to include in + # this task's ecf file. + ecf_more_exports: !calc | + "\n".join([ f"export {K}=%{V}%\n" for K,V in more_vars.items() ]) + + # rocoto_more_vars - additional variables to set in the Rocoto tag's + # elements. + rocoto_more_vars: !calc | + "\n".join([ metasched.defenvar(K,this[V]) for K,V in more_vars.items() ]) + + # ecflow_def_more_vars - additional variables to set in the ecflow + # suite definition "edit" statements for this task. This is only + # used if the task does not override ecflow_def. + ecflow_def_more_vars: !calc | + "\n".join([ metasched.defenvar(V,this[V]) for K,V in more_vars.items() ]) + + # ecflow_def - the contents of this variable are sent into the task + # definition in the ecflow suite definition file. The + # CROW/crow/metascheduler/ecflow.py copies this variable's contents + ecflow_def: !calc ecflow_def_more_vars + + # ecf_model_include - Name of the ecflow %include file with + # model-specific settings. This is model_ver.h in production or the + # header for the user's experiment. Make sure this matches + # doc.ecf_include_experiment.filename. + ecf_model_include: !FirstTrue + - when: !calc doc.settings.nco_mode + take: '%include ' + - when: !calc doc.settings.use_nco_ecflow_headers + take: | + export NWROOT=%NWROOT% # EMC override + export DATAROOT=%DATAROOT% # EMC override + %include + - otherwise: !calc ecf_experiment_overrides + + ecf_experiment_overrides: !FirstTrue + - when: !calc doc.settings.use_nco_ecflow_headers + take: '' + - otherwise: !expand "%include " + + # ecf_file - the contents of this variable are written to the ecf + # file for each task by the CROW/crow/metascheduler/ecflow.py + # python module. + ecf_file: !expand | + #! /bin/sh + {ecf_batch_resources} + %include + %include <{doc.accounting.ecflow_header}> + + set -x + + {ecf_resource_more} + + export model=%model:gfs% + export NET=%NET:gfs% + export RUN=%RUN% + + {ecf_model_include} + + ############################################################ + # Load modules + ############################################################ + #. $MODULESHOME/init/sh + {ecf_module_commands} + {ecf_after_module_commands} + + ############################################################ + # WCOSS environment settings + ############################################################ + {ecf_experiment_overrides} + {ecf_environment_settings} + + ############################################################ + export cyc=%CYC% + {ecflow_config_source} + {ecf_more_exports} + {ecflow_command} + + %include + %manual + {ecf_manual} + %end + + # ecf_manual - contents of the manual entry for this task in ecflow + ecf_manual: '' + + # ecf_module_commands - these commands are sent to the ecf file for + # this task to load any modules that are required for the job + # ecf_module_commands: !ref doc.settings.ecf_module_commands + ecf_module_commands: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: !calc doc.platform.four_cycle_mode_modules + - otherwise: !expand |- + source "$HOMEgfs/ush/load_fv3gfs_modules.sh" {task_type} + + # ecf_after_module_commands - intended to list the module commands, + # and recover from their consequences. For example, "module purge" + # clears the ecflow module, and "module load ecflow" clears the + # ECF_PORT. + ecf_after_module_commands: !FirstTrue + - when: !calc doc.settings.nco_mode + take: | + module list + + # Synonyms expected by scripts: + export CDUMP="$RUN" + export ROTDIR="$COMROOT" + - when: !calc doc.settings.four_cycle_mode + take: | + module list + export ECF_PORT=%ECF_PORT% # workaround for bug in ecflow module + + export cycle=t%CYC%z + export jlogfile=/%COM%/logs/jlogfile + export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config} # where to get config files + + # Development overrides + export DATAROOT=%DATAROOT% + export COMROOT=/%COM% + + # Development synonyms + export CDUMP="$RUN" + export ROTDIR="$COMROOT" + - otherwise: !expand | + module load {doc.settings.ecflow_module} + module list + export ECF_PORT=%ECF_PORT% # workaround for ecflow module bug + + ecf_environment_settings: !FirstTrue + - when: !calc config_list + take: !calc doc.settings.ecflow_rocoto_cdate_workaround + - otherwise: "# Set tuning variables like KMP_AFFINITY and OMP_STACKSIZE here." + + # ecf_log_path: sets the stdout/stderr log path for the job + ecf_log_path: !ref doc.settings.ecf_log_path + + ecf_job_name: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: !expand '%E%{task_path_list[-1][1:]}_%CYC%' + - otherwise: !calc task_path_var + + # ecf_batch_resources - generates batch card settings for the ecflow ecf file + ecf_batch_resources: !FirstTrue + - when: !calc doc.settings.nco_mode + take: !expand "{partition.scheduler.batch_accounting(partition_specification,default_accounting,accounting,jobname=ecf_job_name,outerr=ecf_log_path)}{partition.scheduler.batch_resources(resources)}#BSUB -cwd /tmp" + - otherwise: !expand "{partition.scheduler.batch_accounting(partition_specification,default_accounting,accounting,jobname=ecf_job_name,outerr=ecf_log_path)}{partition.scheduler.batch_resources(resources)}" + + # ecf_resource_more - generates the ntasks, ptile, and threads variables + # in cases where they are needed + ecf_resource_more: !FirstTrue + - when: !calc doc.settings.four_cycle_mode + take: '' + - otherwise: !expand |- + {ecf_maybe_ntasks} + {ecf_maybe_ptile} + {ecf_maybe_threads} + + # ecf_maybe_ntasks - generates the ntasks variable for any MPI jobs' + # ecf files, which specifies the number of MPI ranks. + ecf_maybe_ntasks: !FirstTrue + - when: !calc resources.total_ranks()>0 + do: !expand 'export ntasks={resources.total_ranks()}' + - otherwise: "# No MPI in use, so I am not setting $ntasks." + + # ecf_maybe_threads - generates the ntasks variable for any OpenMP + # jobs' ecf files. This variable specifies the number of OpenMP threads + ecf_maybe_threads: !FirstTrue + - when: !calc resources.has_threads() + do: !expand 'export threads={resources[0]["OMP_NUM_THREADS"]}' + - otherwise: "# No OpenMP in use, so I am not setting $threads" + + # ecf_maybe_ptile - generates the ptile variable if this is an + # OpenMP or MPI program. The ptile variable specifies the number of + # MPI ranks per node. + ecf_maybe_ptile: !FirstTrue + - when: !calc resources.has_threads() or resources.total_ranks()>0 + do: !expand 'export ptile={partition.nodes.max_ranks_per_node(resources[0])}' + - otherwise: "# Neither OpenMP nor MPI are in use, so I am not setting $ptile" + +# shared_task_template - a convenient alias to define a task that has +# the shared_accounting and passes "shared" to the +# load_fv3gfs_modules.sh +shared_task_template: &shared_task_template + <<: *task_template + partition: !calc doc.accounting.shared_partition + default_accounting: !calc partition.shared_accounting_ref + J_JOB: !expand '{task_path_list[-1].upper()}' + task_type: shared + +# service_task_template - a convenient alias to define a task that has +# the service_accounting and passes "service" to the +# load_fv3gfs_modules.sh +service_task_template: &service_task_template + <<: *task_template + partition: !calc doc.accounting.service_partition + default_accounting: !calc partition.service_accounting_ref + J_JOB: !expand '{task_path_list[-1].upper()}' + task_type: service + +# exclusive_task_template - a convenient alias to define a task that has +# the exclusive_accounting and passes "exclusive" to the +# load_fv3gfs_modules.sh +exclusive_task_template: &exclusive_task_template + <<: *task_template + partition: !calc doc.accounting.exclusive_partition + default_accounting: !calc partition.exclusive_accounting_ref + J_JOB: !expand '{task_path_list[-1].upper()}' + task_type: exclusive + +# exclusive_task_template - a convenient alias for the special case of +# an exclusive_accounting job that runs a forecast. These jobs must +# pass "forecast" to the load_fv3gfs_modules.sh. +forecast_task_template: &forecast_task_template + <<: *exclusive_task_template + task_type: forecast diff --git a/workflow/schema/accounting.yaml b/workflow/schema/accounting.yaml new file mode 100644 index 00000000000..4a8e2412f12 --- /dev/null +++ b/workflow/schema/accounting.yaml @@ -0,0 +1,16 @@ +accounting_template: !Template &accounting_template + user_email: + type: string + description: Which email address to use for automatic status report + + cpu_project: + type: string + description: Which project or accounting code to use for cpu time in the batch queue + + hpss_project: + type: string + description: Which project or accounting code to use for HPSS tape storage. + + noscrub_project: + type: string + description: Which project to use for non-scrubbed disk areas diff --git a/workflow/schema/archiving.yaml b/workflow/schema/archiving.yaml new file mode 100644 index 00000000000..fd89781b2d8 --- /dev/null +++ b/workflow/schema/archiving.yaml @@ -0,0 +1,45 @@ +archive_settings_template: !Template &archive_settings_template + archive_to_hpss: + default: yes + description: save data to HPSS archive + type: bool + + arch_cyc: + default: 18 + description: cycle for archiving files for warm_start capability + type: int + + arch_warmicfreq: + default: 6 + description: archive frequency in days for warm_start capability + type: int + + arch_fcsticfreq: + default: 3 + descripton: archive frequency in days for gdas/gfs forecast-only capability + type: int + + copy_fit2obs_files: + default: yes + description: online archive of nemsio files for fit2obs verification + type: bool + + scrub_in_archive: + type: bool + default: yes + description: "delete GDAS/GFS COM directories at the end of the archive job" + + scrub_in_archive_start: + type: int + default: 144 + description: "starting hour of previous cycles to be removed from rotating directory during archive job" + + scrub_in_archive_end: + type: int + default: 24 + description: "ending hour of previous cycles to be removed from rotating directory during archive job" + + ATARDIR: + type: string + default: "/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT" + description: "Archiving directory on HPSS for the current workflow. Default is a shell expression typically used in config.base." diff --git a/workflow/schema/case.yaml b/workflow/schema/case.yaml new file mode 100644 index 00000000000..3ee39bbffe4 --- /dev/null +++ b/workflow/schema/case.yaml @@ -0,0 +1,31 @@ +case_template: !Template &case_template + gfs_cyc: + type: int + default: 2 + allowed: [ 0, 1, 2, 4 ] + description: | + When to run GFS forecast. Data assimilation is run for every + cycle regardless of these values + * 0: no GFS cycle + * 1: 00Z only + * 2: 00Z and 12Z only + * 4: all 4 cycles (0, 6, 12, 18 Z) + + SDATE: + type: datetime + description: | + First cycle to run. In the cycled workflow, this is a + "half-cycle" in that it only runs the parts of the workflow that + do not require a prior cycle. The first "real" cycle is the + following cycle. + + EDATE: + type: datetime + description: Last cycle to run GDAS. + + experiment_name: + type: string + default: !calc doc.experiment_name + description: | + User-defined experiment name; should be a-z followed by alphanumeric. + Formerly known as PSLOT. diff --git a/workflow/schema/chgres.yaml b/workflow/schema/chgres.yaml new file mode 100644 index 00000000000..4de42b7a6d5 --- /dev/null +++ b/workflow/schema/chgres.yaml @@ -0,0 +1,10 @@ +chgres_template: !Template &chgres_template + CHGRESVARS_ENKF: + type: string + default: "use_ufo=.true.,nopdpvv=.true." + + CHGRESTHREAD: + type: int + default: 12 + description: Number of threads to use for chgres + diff --git a/workflow/schema/data_assimilation.yaml b/workflow/schema/data_assimilation.yaml new file mode 100644 index 00000000000..b141a83691f --- /dev/null +++ b/workflow/schema/data_assimilation.yaml @@ -0,0 +1,117 @@ +# This file configures the data assimilation, which is not included in +# this release. Although the variables are not used, they still must +# be defined to satisfy validation logic elsewhere. + +data_assimilation_template: !Template &data_assimilation_template + DOHYBVAR: + type: bool + default: YES + description: flag (YES or NO) for hybrid ensemble variational option + NMEM_ENKF: + type: int + default: 80 + description: Number of members of the GFS ENKF ensemble. + NMEM_EOMGGRP: { type: int, default: 10 } + NMEM_EFCSGRP: { type: int, default: 10 } + NMEM_EARCGRP: { type: int, default: 10 } + RECENTER_ENKF: { type: bool, default: YES } + SMOOTH_ENKF: { type: bool, default: YES } + assim_freq: { type: int, default: 6 } + l4densvar: { type: bool, default: YES } + lwrite4danl: { type: bool, default: NO } + + NSPLIT: + type: int + default: 4 + description: "Execute prepbufr in parallel. Operational GFS default is 3." + + NAM_ENKF: + type: string + default: "analpertwtnh=0.9,analpertwtsh=0.9,analpertwttr=0.9" + + INCREMENTS_TO_ZERO: + type: string list + default: [ delz_inc, clwmr_inc, icmr_inc ] + + PREP_REALTIME: + type: bool + default: no + + DO_EMCSFC: + type: bool + default: NO + description: "Whether to run EMC SFC job" + + PROCESS_TROPCY: + type: bool + default: no + + DO_RELOCATE: + type: bool + default: no + + DO_MAKEPREPBUFR: + type: bool + default: YES + description: If NO, will copy prepbufr from globaldump. Otherwise, generates prepbufr file anew. + + OPREFIX: + type: string + default: "$CDUMP.$cycle." + description: Format for dump file name prefix + + COM_OBS: + type: string + default: "$DMPDIR" + description: Top folder for COMIN_OBS (DMPDIR or COMROOT) + + COMIN_OBS: + type: string + description: Location to draw observation data from (global dump archive or com) + default: !FirstTrue + - when: !calc ( COM_OBS=="$DMPDIR" ) + do: "$DMPDIR/$CDATE/$CDUMP$DUMP_SUFFIX" + - when: !calc ( COM_OBS=="$COMROOT" ) + do: "$COMROOT/$CDUMP.$PDY/$CYC" + - otherwise: !error "Unknown COM_OBS value: {COM_OBS}" + + RERUN_EFCSGRP: + type: bool + default: NO + description: Rerun failed ensemble forecast group members automatically. + + RERUN_EOMGGRP: + type: bool + default: YES + + GENDIAG: + type: bool + default: YES + + NEPOSGRP: + type: int + default: 7 + + OBSINPUT_INVOBS: + type: string + default: "dmesh(1)=225.0,dmesh(2)=225.0" + OBSQC_INVOBS: + type: string + default: "tcp_width=60.0,tcp_ermin=2.0,tcp_ermax=12.0" + + ENKF_INNOVATE_GROUPS: + type: int + default: !calc ( NMEM_ENKF // NMEM_EOMGGRP ) + + ENKF_FORECAST_GROUPS: + type: int + default: !calc ( NMEM_ENKF // NMEM_EFCSGRP ) + + ENKF_ARCHIVE_GROUPS: + type: int + default: !calc ( NMEM_ENKF // NMEM_EARCGRP ) + +# These variable largely eliminate the need for explicitly setting +# USH directories, FIX files, PARM files, EXECutables below +# The USER can overwrite components that they wish +# e.g. PRVT is used from the GSI diff --git a/workflow/schema/downstream.yaml b/workflow/schema/downstream.yaml new file mode 100644 index 00000000000..bc936f8b6b1 --- /dev/null +++ b/workflow/schema/downstream.yaml @@ -0,0 +1,103 @@ +# This file is used by the downstream product generation. That +# includes all steps of product generation after the post master file +# and lat-lon files are created. For example, the tracker, VSDB, and +# AWIPS files. These are not included in this release, so this file +# should be left unmodified. These variables must still be defined to +# ensure validation logic elsewhere is satisfied. + +downstream_schema: &downstream_schema !Template + VDUMP: + type: string + default: "gfs" + description: verifying dump + CDUMPFCST: + type: string + default: "gdas" + description: Fit-to-obs with GDAS/GFS prepbufr + CDFNL: + type: string + default: "gdas" + description: Scores verification against GDAS/GFS analysis + + MKPGB4PRCP: + type: bool + default: yes + description: make 0.25-deg pgb files in ARCDIR for precip verification + VRFYFITS: + type: bool + default: YES + description: "Fit to observations" + VSDB_STEP1: + type: bool + default: YES + description: "populate VSDB database" + VSDB_STEP2: + type: bool + default: NO + VRFYG2OBS: + type: bool + default: YES + description: "Grid to observations, see note below if turning ON" + VRFYPRCP: + type: bool + default: YES + description: "Precip threat scores" + VRFYRAD: + type: bool + default: YES + description: "Radiance data assimilation monitoring" + VRFYOZN: + type: bool + default: YES + description: "Ozone data assimilation monitoring" + VRFYMINMON: + type: bool + default: YES + description: "GSI minimization monitoring" + VRFYTRAK: + type: bool + default: YES + description: "Hurricane track verification" + VRFYGENESIS: + type: bool + default: YES + description: "Cyclone genesis verification" + RUNMOS: + type: bool + default: YES + description: "Run GFS MOS" + DO_POST_PROCESSING: + type: bool + default: NO + description: "Run post processing tasks (BUFRSND, GEMPAK, AWIPS, FAX, WAFS, BULLETINS" + DO_BUFRSND: + type: bool + default: NO + description: "Run GFS_POSTSND" + DO_GEMPAK: + type: bool + default: NO + description: "Run GEMPAK" + DO_AWIPS: + type: bool + default: NO + description: "Run AWIPS" + DO_FAX: + type: bool + default: NO + description: "Run FAX" + DO_WAFS: + type: bool + default: NO + description: "Run WAFS" + DO_BULLETINS: + type: bool + default: NO + description: "Run BULLETINS" + + FHOUT_CYCLONE_GFS: + type: int + default: 6 + FHOUT_CYCLONE_GDAS: + type: int + default: 3 diff --git a/workflow/schema/fv3.yaml b/workflow/schema/fv3.yaml new file mode 100644 index 00000000000..b6ccc8d240b --- /dev/null +++ b/workflow/schema/fv3.yaml @@ -0,0 +1,307 @@ +# fv3_settings_template - sets the namelist values for the fv3 +# forecast. See the physcs and model documentation for full +# information on these variables. + +fv3_settings_template: !Template &fv3_settings_template + imp_physics: + type: int + allowed: [ 99, 8, 6, 10, 11 ] + default: 11 + description: "Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL" + if_present: !FirstTrue + - when: !calc ( imp_physics==99 ) + do: !Template + ncld: { type: int, default: 1 } + nwat: { type: int, default: 2 } + FIELD_TABLE: + type: string + default: "$HOMEgfs/parm/parm_fv3diag/field_table_zhaocarr" + - when: !calc ( imp_physics == 6 ) + do: !Template + ncld: { type: int, default: 2 } + nwat: { type: int, default: 6 } + FIELD_TABLE: + type: string + default: "$HOMEgfs/parm/parm_fv3diag/field_table_wsm6" + - when: !calc ( imp_physics == 8 ) + do: !Template + ncld: { type: int, default: 2 } + nwat: { type: int, default: 6 } + FIELD_TABLE: + type: string + default: "$HOMEgfs/parm/parm_fv3diag/field_table_thompson" + - when: !calc ( imp_physics == 11 ) + do: !Template + ncld: { type: int, default: 5 } + nwat: { type: int, default: 6 } + FIELD_TABLE: + type: string + default: "$HOMEgfs/parm/parm_fv3diag/field_table_gfdl_satmedmf" + dnats: { type: int, default: 1 } + cal_pre: { type: bool, default: false } + do_sat_adj: { type: bool, default: true } + random_clds: { type: bool, default: false } + hord_mt_nh_nonmono: { type: int, default: 5 } + hord_xx_nh_nonmono: { type: int, default: 5 } + vtdm4_nh_nonmono: { type: float, default: 0.02 } + nord: { type: int, default: 2 } + dddmp: { type: float, default: 0.1 } + d4_bg: { type: float, default: 0.12 } + - otherwise: !error "Unknown imp_physics value: {imp_physics}" + + new_o3force: + type: bool + default: yes + description: Options of stratosphere O3 physics reaction coefficients + h2o_phys: + type: bool + default: yes + description: Use stratosphere h2o physics + do_vort_damp: + type: bool + default: yes + description: vorticity and divergence damping + consv_te: + type: float + default: 1. + description: conserve total energy + fv_sg_adj: + type: int + default: 450 + description: time-scale to remove 2dz instability + dspheat: + type: bool + default: YES + description: dissipative heating + shal_cnv: + type: bool + default: YES + description: shallow convection flag + agrid_vel_rst: + type: bool + default: yes + description: "write velocity restarts on A grid?" + + cal_pre: { type: bool, default: true } + do_sat_adjust: { type: bool, default: false } + random_clds: { type: bool, default: true } + cnvcld: { type: bool, default: true } + dnats: { type: int, default: 0 } + IEMS: { type: int, default: 1 } + IALB: { type: int, default: 1 } + ISOL: { type: int, default: 2 } + IAER: { type: int, default: 111 } + ICO2: { type: int, default: 2 } + warm_start: { type: bool, default: true } + read_increment: { type: bool, default: no } + restart_interval: { type: int, default: 6 } + + LEVS: + type: int + description: Number of vertical levels in FV3 + + FHCYC: + type: int + default: 24 + description: Surface cycle update frequency (gdas=1,gfs=24) + + FHCYC_GDAS: + type: int + default: 1 + description: Surface cycle update frequency for the GDAS + + FHCYC_GFS: + type: int + default: 24 + description: Surface cycle update frequency for the GFS + + QUILTING: + type: bool + default: YES + description: Should output quilting be used? + + WRITE_NEMSIOFILE: + type: bool + default: YES + description: Should nemsio output be used (yes/true) instead of netcdf (no/false)? + + WRITE_NEMSIOFLIP: { type: bool, default: yes } + nst_anl: { type: bool, default: yes } + lprecip_accu: { type: bool, default: yes, + description: "Precipitation accumulation, true--no bucket, false--bucket=FHZER" } + + DONST: { type: bool, default: YES } + + MONO: + type: string + default: "non-mono" + allowed: [ mono, non-mono ] + + MEMBER: + type: int + stage: [ execution ] + default: -1 + description: "-1: control, 0: ensemble mean, >0: ensemble member $MEMBER" + + d4_bg: { type: float, default: 0.15 } + dddmp: { type: float, default: 0.2 } + + ISEED: + type: int + default: 0 + stage: [ execution ] + description: > + Default seed for shum, skeb, and sppt, if specific seeds + are not specified. + + SET_STP_SEED: + type: bool + default: YES + description: > + Automatically set seeds for SKEB, SHUM, and SPPT at execution + time based on simulation date and member. + + DO_SHUM: + type: bool + default: NO + if_present: !FirstTrue + - when: !calc DO_SHUM + do: + ISEED_SHUM: { type: int, optional: true } + SHUM: { type: float, default: -999. } + SHUM_TAU: { type: float, default: -999. } + SHUM_LSCALE: { type: float, default: -999. } + - otherwise: null + + DO_SKEB: + type: bool + default: NO + if_present: !FirstTrue + - when: !calc DO_SKEB + do: + ISEED_SHUM: { type: int, optional: true } + SKEB_TAU: { type: float, default: -999. } + SKEB_LSCALE: { type: float, default: -999. } + SKEBNORM: { type: int, default: 1 } + SKEB: { type: float, default: -999. } + - otherwise: null + + DO_SPPT: + type: bool + default: NO + if_present: !FirstTrue + - when: !calc DO_SPPT==True + do: + ISEED_SHUM: { type: int, optional: true } + SPPT: { type: float } + SPPT_TAU: { type: float, default: -999. } + SPPT_LSCALE: { type: float, default: -999. } + SPPT_SFCLIMIT: { type: bool, default: yes } + - otherwise: !Message "Don't SPPT" + + RUN_EFCSGRP: { type: bool, default: NO } + ncld: { type: int, default: 1 } + nwat: { type: int, default: 2 } + zhao_mic: { type: bool, default: YES } + nh_type: { type: string, default: 'nh' } + USE_COUPLER_RES: { type: bool, default: NO } + cdmbgwd: { type: string, optional: true } + + CDUMP: + type: string + allowed: [ gfs, gdas ] + + CASE: + type: string + allowed: [ 'C48', 'C96', 'C192', 'C384', 'C768', 'C1152', 'C3072' ] + if_present: !FirstTrue + - when: !calc (CASE == "C48") + do: !Template + DELTIM: { type: int, default: 450 } + layout_x: { type: int, default: 2 } + layout_y: { type: int, default: 4 } + fv3_threads: { type: int, default: 1 } + cdmbgwd: + type: string + default: "0.062,3.5,1.0,1.0" + description: "mountain blocking and gravity wave drag" + WRITE_GROUP: { type: int, default: 1 } + WRTTASK_PER_GROUP: { type: int, default: 24 } + WRTIOBUF: { type: string, default: "4M" } + + - when: !calc (CASE == "C96") + do: !Template + DELTIM: { type: int, default: 450 } + layout_x: { type: int, default: 4 } + layout_y: { type: int, default: 4 } + fv3_threads: { type: int, default: 1 } + cdmbgwd: + type: string + default: "0.125,3.0,1.0,1.0" + description: "mountain blocking and gravity wave drag" + WRITE_GROUP: { type: int, default: 1 } + WRTTASK_PER_GROUP: { type: int, default: 24 } + WRTIOBUF: { type: string, default: "4M" } + + - when: !calc (CASE == "C192") + do: !Template + DELTIM: { type: int, default: 450 } + layout_x: { type: int, default: 4 } + layout_y: { type: int, default: 6 } + fv3_threads: { type: int, default: 2 } + cdmbgwd: + type: string + default: "0.23,1.5,1.0,1.0" + description: "mountain blocking and gravity wave drag" + WRITE_GROUP: { type: int, default: 2 } + WRTTASK_PER_GROUP: { type: int, default: 24 } + WRTIOBUF: { type: string, default: "8M" } + + - when: !calc (CASE == "C384") + do: !Template + DELTIM: { type: int, default: 240 } + layout_x: { type: int, default: 6 } + layout_y: { type: int, default: 6 } + fv3_threads: { type: int, default: 2 } + cdmbgwd: + type: string + default: "1.1,0.72,1.0,1.0" + description: "mountain blocking and gravity wave drag" + WRITE_GROUP: { type: int, default: 3 } + WRTTASK_PER_GROUP: { type: int, default: 24 } + WRTIOBUF: { type: string, default: "16M" } + + - when: !calc (CASE == "C768") + do: !Template + DELTIM: { type: int, default: 225 } + layout_x: { type: int, default: 8 } + layout_y: { type: int, default: 16 } + fv3_threads: { type: int, default: 2 } + cdmbgwd: + type: string + default: "3.5,0.25,1.0,1.0" + description: "mountain blocking and gravity wave drag" + WRITE_GROUP: { type: int, default: 4 } + WRTTASK_PER_GROUP: { type: int, default: 60 } + WRTIOBUF: { type: string, default: "32M" } + + - when: !calc (CASE == "C1152") + do: !Template + DELTIM: { type: int, default: 150 } + layout_x: { type: int, default: 8 } + layout_y: { type: int, default: 16 } + fv3_threads: { type: int, default: 4 } + WRITE_GROUP: { type: int, default: 4 } + WRTTASK_PER_GROUP: { type: int, default: 84 } + WRTIOBUF: { type: string, default: "48M" } + + - when: !calc (CASE == "C3072") + do: !Template + DELTIM: { type: int, default: 90 } + layout_x: { type: int, default: 16 } + layout_y: { type: int, default: 32 } + fv3_threads: { type: int, default: 4 } + WRITE_GROUP: { type: int, default: 4 } + WRTTASK_PER_GROUP: { type: int, default: 120 } + WRTIOBUF: { type: string, default: "64M" } + - otherwise: !error "Unknown case: {CASE}" diff --git a/workflow/schema/ice.yaml b/workflow/schema/ice.yaml new file mode 100644 index 00000000000..089e798316e --- /dev/null +++ b/workflow/schema/ice.yaml @@ -0,0 +1,7 @@ + +ice_settings_template: !Template &ice_settings_template + MOD: + type: string + allowed: [ cice5, cice6 ] + default: cice5 + description: "model selection for ice" diff --git a/workflow/schema/nsst.yaml b/workflow/schema/nsst.yaml new file mode 100644 index 00000000000..cf195664ba8 --- /dev/null +++ b/workflow/schema/nsst.yaml @@ -0,0 +1,36 @@ +nsst_schema: &nsst_schema !Template + NST_MODEL: + type: int + allowed: [ 0, 1, 2 ] + default: 2 + description: >- + nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, + 1 = ON but uncoupled, 2 = ON and coupled + NST_RESV: + type: int + default: 0 + allowed: [ 0, 1 ] + ZSEA1: + type: int + default: 0 + ZSEA2: + type: int + default: 0 + NST_GSI: + type: int + allowed: [ 0, 1, 2, 3, 4 ] + default: 3 + description: | + 0: No NST info at all; + 1: Input NST info but not used in GSI; + 2: Input NST info, used in CRTM simulation, no Tr analysis + 3: Input NST info, used in both CRTM simulation and Tr analysis + + NSTINFO: + type: int + default: 0 + description: number of elements added in obs. data array + NST_SPINUP: + type: int + allowed: [ 0, 1] + optional: yes diff --git a/workflow/schema/ocn.yaml b/workflow/schema/ocn.yaml new file mode 100644 index 00000000000..d8f79d48e5f --- /dev/null +++ b/workflow/schema/ocn.yaml @@ -0,0 +1,10 @@ +# fv3_settings_template - sets the namelist values for the fv3 +# forecast. See the physcs and model documentation for full +# information on these variables. + +ocn_settings_template: !Template &ocn_settings_template + MOD: + type: string + allowed: [ mom6, hycom ] + default: mom6 + description: "model selection for ocean" diff --git a/workflow/schema/output.yaml b/workflow/schema/output.yaml new file mode 100644 index 00000000000..42868b81439 --- /dev/null +++ b/workflow/schema/output.yaml @@ -0,0 +1,99 @@ +gfs_output_settings_template: !Template &gfs_output_settings_template + FHOUT_GFS: + type: int + default: 3 + description: GFS forecast output frequency in hours + FHMIN_GFS: + type: int + default: 0 + description: GFS forecast initial hour for output + + FHMIN_ENKF: + type: int + default: 3 + description: GDAS ENKF initial hour for output + FHMAX_ENKF: + type: int + default: 9 + description: GDAS ENKF forecast length + FHOUT_ENKF: + type: int + description: GDAS ENKF output frequency in hours + default: !FirstTrue + - when: doc.data_assimilation.l4densvar + do: 1 + - otherwise: 3 + + FHMIN_GDAS: + type: int + default: 0 + description: GDAS initial hour for output + FHMAX_GDAS: + type: int + default: 9 + description: GDAS forecast length + FHOUT_GDAS: + type: int + default: 3 + description: GDAS output frequency in hours + + FHMAX_HF_GFS: + type: int + default: 0 + description: Last forecast hour with high-frequency output for gfs + + FHOUT_HF_GFS: + type: int + default: 1 + description: Output frequency until FHMAX_HF_GFS hours. + + OCN_INTERVAL: + type: int + default: 120 + description: Interval for separating ocean post job. + + NCO_NAMING_CONV: + type: bool + default: YES + description: "YES = use standard, operational, naming conventions. NO = use non-standard naming conventions" + + OUTPUT_FILE_TYPE: + type: string + default: nemsio + description: "Type of model output file" + + + gfs_forecast_hours: + type: int list + default: !calc >- + tools.uniq(sorted( + tools.seq(FHMIN_GFS,FHMAX_HF_GFS,FHOUT_HF_GFS) + + tools.seq(FHMIN_GFS,FHMAX_GFS, FHOUT_GFS))) + + ocnpost_hours: + type: int list + default: !calc "tools.seq(FHMIN_GFS,FHMAX_GFS-OCN_INTERVAL,OCN_INTERVAL)" + + gdas_forecast_hours: + type: int list + default: !calc "tools.seq(FHMIN_GDAS,FHMAX_GDAS,FHOUT_GDAS)" + + enkf_epos_fhr: + type: int list + default: !calc "tools.seq(FHMIN_ENKF,FHMAX_ENKF,FHOUT_ENKF)" + + wafs_last_hour: + type: int + default: !calc 120 + + awips_g2_hours: + type: int list + default: !calc ( tools.seq(0,240,6) ) + + awips_20km_1p0_hours: + type: int list + default: !calc >- + tools.uniq(sorted( + tools.seq(0,84,3) + + tools.seq(90,240,6))) + diff --git a/workflow/schema/places.yaml b/workflow/schema/places.yaml new file mode 100644 index 00000000000..10d98ef1945 --- /dev/null +++ b/workflow/schema/places.yaml @@ -0,0 +1,67 @@ +# Configures disk locations. See the defaults/places.yaml and +# user.yaml.default file for documentation. + +places_schema: &places_schema !Template + EXPROOT: { type: string } + HOMEgfs: { type: string } + HOMEcrow: { type: string } + NWPROD: { type: string } + DMPDIR: { type: string } + RTMFIX: { type: string } + EXPDIR: { type: string } + ROTDIR: { type: string } + ICSDIR: { type: string } + + COMROOT: + type: string + optional: !calc not doc.settings.four_cycle_mode + +# SHORT_TERM_TEMP: { type: string } +# LONG_TERM_TEMP: { type: string } + + HOMEDIR: + type: string + default: !FirstTrue + - when: !calc doc.platform.get('SAVE_DIR','') + do: !calc doc.platform.SAVE_DIR + - otherwise: !expand "{doc.places.EXPROOT}/save/{tools.env('USER')}" + + NOSCRUB: + type: string + default: !FirstTrue + - when: !calc doc.platform.get('NOSCRUB_DIR','') + do: !calc doc.platform.NOSCRUB_DIR + - otherwise: !expand "{doc.places.EXPROOT}/noscrub/{tools.env('USER')}" + description: "Disk area used to store reduced output, verification statistics, and other small files that may be desired between runs. The workflow will not scrub this." + + FIXgsi: { type: string, optional: true } + HOMEfv3gfs: { type: string, optional: true } + HOMEpost: { type: string, optional: true } + HOMEobsproc_prep: + type: string + optional: true + default: !expand "{doc.platform.BASE_GIT}/obsproc/obsproc_prep_RB-5.2.0" + HOMEobsproc_network: + type: string + optional: true + default: !expand "{doc.platform.BASE_GIT}/obsproc/obsproc_global_RB-3.2.0" + BASE_VERIF: { type: string, optional: true } + BASE_SVN: + type: string + default: !calc doc.platform.BASE_SVN + BASE_GIT: + type: string + default: !calc doc.platform.BASE_GIT + + ics_from: + type: string + default: opsgfs + allowed: [ opsgfs, pargfs ] + description: initial conditions from opsgfs or pargfs + if_present: !Template + parexp: + type: string + default: prnemsrn + HPSS_PAR_PATH: + type: string + default: !expand "/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/{parexp}" diff --git a/workflow/schema/post.yaml b/workflow/schema/post.yaml new file mode 100644 index 00000000000..011336d1e10 --- /dev/null +++ b/workflow/schema/post.yaml @@ -0,0 +1,25 @@ +# This is used to set default values for grib product generation. +# Reconfiguring These variables are not supported in this release. + +post_schema: &post_schema !Template + GOESF: + type: bool + default: yes + description: make synthetic goes imagery + GTGF: + type: bool + default: no + description: make gtg icing product + FLXF: + type: bool + default: yes + description: grib2 flux file written by post + PGB1F: { type: bool, default: yes } + GFS_DOWNSTREAM: { type: bool, default: yes } + downset: { type: int, default: 2 } + NPOSTGRP: { type: int, default: 5 } + master_grid: + type: string + default: "0p25deg" + allowed: [ "1deg", "0p5deg", "0p25deg", "0p125deg" ] + description: Original grid of the post diff --git a/workflow/schema/settings.yaml b/workflow/schema/settings.yaml new file mode 100644 index 00000000000..c4a275db522 --- /dev/null +++ b/workflow/schema/settings.yaml @@ -0,0 +1,138 @@ +settings_schema: !Template &settings_schema + + dev_safeguards: + type: bool + default: true + description: "Add backup triggers in workflow to handle scheduling delays, such as the forecast finishing any posts start. Only turn this off for NCO operational deliveries." + + realtime: + type: bool + default: false + description: "Simulation of an event that is currently happening (ie.: a forecast)" + + run_vrfy_jobs: + type: bool + default: true + description: "Run the EMC product generation jobs." + + four_cycle_mode: + type: bool + description: "Enable NCO-like four cycle suite. Requires a special setup for your ecFlow server, and a prepared directory structure designed to look like NCEP production. Do not use unless you know what you're doing." + default: false + if_present: !FirstTrue + - when: !calc four_cycle_mode + take: !FirstTrue + - when: !calc ( doc.names.experiment in [ "prod", "para", "test" ] ) + message: !expand "Running four cycle mode with envir={doc.names.experiment}" + take: null + - otherwise: !error "In four cycle mode, the experiment must be prod, para, or test, not {doc.names.experiment}." + - otherwise: null + +# ecf_module_commands: +# type: str +# default: !FirstTrue +# - when: !calc four_cycle_mode +# take: !calc doc.platform.four_cycle_mode_modules +# - otherwise: !expand | +# source "$HOMEgfs/ush/load_fv3gfs_modules.sh" {task_type} + + rocoto_cycle_throttle: + type: int + description: "Maximum number of cycles active at once in a Rocoto workflow" + default: 2 + + rocoto_task_throttle: + type: int + description: "Maximum number of tasks active (queued/running) at once in a Rocoto workflow" + default: 5 + + use_nco_ecflow_headers: + type: bool + description: "Use the NCO model-ver.h instead of experiment-specific paths. This will cause you to use the scripts and code in NCEP Operational areas instead of your own scripts and code. Only use this option if you know what you're doing." + default: !calc four_cycle_mode + + prod_util_module: + type: str + default: !calc doc.platform.get("prod_util_module","prod_util") + description: Name of the unix modulefile to load to obtain NCEP production shell utilities. + + ecflow_real_clock: + type: bool + default: no + description: the ecflow suite definition should specify a real clock + + ecflow_virtual_clock: + type: bool + default: no + description: the ecflow suite definition should specify a virtual clock + + ecflow_hybrid_clock: + type: bool + default: no + description: the ecflow suite definition should specify a hybrid clock + + ecflow_totality_limit: + type: bool + default: yes + description: 'Place the entire suite in the "/totality_limit:TOTALITY" limit so that the server can limit the total number of jobs running.' + + run_gsi: + type: bool + description: "Run Gridpoint Statistical Interpolation" + + run_enkf: + type: bool + description: "Enable Ensemble Kalman Filter" + default: !calc run_gsi + + chgres_and_convert_ics: + type: bool + description: "Run the fv3ic job, which changes restart file resolution and converts legacy GFS GSM restart files to FV3 restart files." + + max_job_tries: + type: int + default: 1 + description: "Number of times to try running a job. Set to 1 for no retries." + + IC_CDUMP: + type: string + allowed: [ gfs, gdas ] + optional: true + description: Get initial conditions from gfs or gdas + + gfs_cyc: + type: int + default: 4 + allowed: [ 0, 1, 2, 4 ] + description: | + When to run GFS forecast. Data assimilation is run for every + cycle regardless of these values + * 0: no GFS cycle + * 1: 00Z only + * 2: 00Z and 12Z only + * 4: all 4 cycles (0, 6, 12, 18 Z) + + SDATE: + type: datetime + description: | + First cycle to run. In the cycled workflow, this is a + "half-cycle" in that it only runs the parts of the workflow that + do not require a prior cycle. The first "real" cycle is the + following cycle. + + EDATE: + type: datetime + description: Last cycle to run GDAS. + + ics_from: + type: string + default: opsgfs + allowed: [ opsgfs, pargfs ] + description: initial conditions from opsgfs or pargfs + if_present: !Template + parexp: + type: string + default: prnemsrn + HPSS_PAR_PATH: + type: string + default: !expand "/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/{parexp}" diff --git a/workflow/schema/task.yaml b/workflow/schema/task.yaml new file mode 100644 index 00000000000..71676ae80b7 --- /dev/null +++ b/workflow/schema/task.yaml @@ -0,0 +1,87 @@ +task_schema: &task_schema !Template + + Rocoto: + description: >- + XML to insert in the task definition, excluding the task tag + itself, and the dependencies. + type: string + stages: [ execution ] + + ecf_file: + description: >- + Contents of the ecf file that will be run for this task + type: string + stages: [ execution ] + + rocoto_more_vars: + description: >- + Additional text to include after other envar tags in Rocoto task + definition. This is referenced by the Rocoto variable. + type: string + default: "" + stages: [ execution ] + + ecflow_command: + description: >- + Command to execute for this task when run in ecflow. This is + inserted into the ecf file. + type: string + stages: [ execution ] + + rocoto_command: + description: >- + Command to execute for this task when run in rocoto. This is + inserted into the rocoto command tag for the task. + type: string + stages: [ execution ] + + more_vars: + description: >- + List of variables that should be added to Rocoto envar blocks, + ecflow suite definition edits, and ecf file exports. Overriding + ecflow_def will prevent this from being passed through as edits + in the suite definition. + type: string list + stages: [ execution ] + + ecf_environment_settings: + description: "Tuning variables like KMP_AFFINITY and OMP_STACKSIZE." + type: string + default: "" + stages: [ execution ] + + ecf_module_commands: + description: >- + Any "module" commands: module load, module switch, etc. + type: string + default: "" + stages: [ execution ] + + ecf_resource_more: + description: >- + Generates additional resource specification variables such as + ntasks, threads, and ptile. + type: string + default: "" + stages: [ execution ] + + ecf_maybe_ntasks: + description: "Sets the $ntasks variable (number of MPI ranks) if needed" + type: string + default: "" + stages: [ execution ] + + ecf_maybe_threads: + description: "Sets the $threads variable (number of OpenMP threads) if needed" + type: string + default: "" + stages: [ execution ] + + ecf_maybe_ptile: + description: >- + Sets the $ptile variable (maximum number of MPI ranks per node) if needed + type: string + default: "" + stages: [ execution ] + + diff --git a/workflow/schema/varnames.yaml b/workflow/schema/varnames.yaml new file mode 100644 index 00000000000..a7649aacca7 --- /dev/null +++ b/workflow/schema/varnames.yaml @@ -0,0 +1,57 @@ +schedvar_schema: &schedvar_schema !Template + cpu_project: + type: string + default: CPU_PROJECT + description: name of the metascheduler variable for the project whose CPU allocation should be used + + shared_queue: + type: string + default: QUEUESHARED + description: Name of the metascheduler variable for the queue that should receive jobs that can share nodes with others, but do not need external resources such as network access. + + service_queue: + type: string + default: QUEUESERV + description: Name of the metascheduler variable for the queue that receives jobs which need external resources such as network access. + + exclusive_queue: + type: string + default: QUEUE + description: Name of the metascheduler variable for the queue that receives jobs that are too large to share nodes with other jobs. + + partition: + type: string + default: PARTITION + description: Name of the metascheduler variable that specifies the machine partition for all jobs. + + shared_partition: + type: string + default: SHARED_PARTITION + description: Name of the metascheduler variable that specifies the machine partition for shared_queue jobs. + + service_partition: + type: string + default: SERVICE_PARTITION + description: Name of the metascheduler variable that specifies the machine partition for service_queue jobs. + + exclusive_partition: + type: string + default: EXCLUSIVE_PARTITION + description: Name of the metascheduler variable that specifies the machine partition for exclusive_queue jobs. + + script_home: + type: string + default: HOMEgfs + description: "Installation area of the FV3 GFS. This is the parent directory of ush, scripts, jobs, etc." + + obsproc_network_home: + type: string + default: HOMEobsproc_network + description: "Installation area of the global obsproc." + + obsproc_prep_home: + type: string + default: HOMEobsproc_prep + description: "Installation area of the global obsproc prep." + + diff --git a/workflow/top.yaml b/workflow/top.yaml new file mode 100644 index 00000000000..57d3cdbd1ef --- /dev/null +++ b/workflow/top.yaml @@ -0,0 +1,65 @@ +# This file has the top-level logic to merge other YAML data +# structures into the document-level settings. It merges the contents +# of the case files, default files, platform file, and everywhere +# else, and applies any validation from the schema/ directory. + +fv3_settings: !Immediate + - !calc doc.case.fv3_gfs_settings + +fv3_gfs_settings: !Immediate + - !calc doc.case.fv3_gfs_settings + +fv3_gdas_settings: !Immediate + - !calc doc.case.fv3_gdas_settings + +fv3_enkf_settings: !Immediate + - !calc doc.case.fv3_enkf_settings + +gfs_output_settings: !Immediate + - !calc doc.case.get('gfs_output_settings',{}) + +schedvar: !Immediate + - !MergeMapping + - !calc doc.case.get('schedvar',[]) + - Template: *schedvar_schema + +data_assimilation: !Immediate + - !calc doc.case.get('data_assimilation',{}) + +downstream: !Immediate + - !calc doc.case.get('downstream',{}) + +places: !Immediate + - !MergeMapping + - !calc doc.default_places + - !calc doc.case.places + - !calc doc.user_places + +archiving: !Immediate + - !calc doc.case.get('archiving',{}) + +exclusive_resources: !calc doc.platform.partitions.default_exclusive.resources +shared_resources: !calc doc.platform.partitions.default_shared.resources +service_resources: !calc doc.platform.partitions.default_service.resources + +settings: !Immediate + - !MergeMapping + - !calc doc.default_settings + - !calc doc.case.settings + +suite_overrides: !Immediate + - !AppendSequence + - !calc doc.platform.get('suite_overrides',[]) + - !calc doc.case.get('suite_overrides',[]) + - !calc doc.get('user_suite_overrides',[]) + +validate_me: + - fv3_settings + - fv3_gdas_settings + - fv3_gfs_settings + - fv3_enkf_settings + - settings + - data_assimilation + - downstream + - places + - schedvar diff --git a/workflow/totality_limit.def b/workflow/totality_limit.def new file mode 100644 index 00000000000..2eb08e47b2f --- /dev/null +++ b/workflow/totality_limit.def @@ -0,0 +1,3 @@ +suite totality_limit + limit TOTALITY 12 +endsuite \ No newline at end of file diff --git a/workflow/user.yaml.default b/workflow/user.yaml.default new file mode 100644 index 00000000000..a6c468bdb2c --- /dev/null +++ b/workflow/user.yaml.default @@ -0,0 +1,18 @@ +user_places: &user_places + EXPROOT: !error Please select the EXPROOT directory. + # ie. /scratch4/NCEPDEV/ocean + FIX_SCRUB: False + COMROOT: none # Please select a scrub space when FIX_SCRUB is True + DATAROOT: none + +accounting: &accounting + # Project for CPU accounting. + user_email: none # only when you want automatic status report + cpu_project: !error What accounting code do I use to submit jobs? # ie.: global + hpss_project: !error Where do I put data on HPSS? # ie.: emc-global + ecflow_machine: venus # only relevant on WCOSS using ecFlow + ecflow_header: envir-p3.h + + shared_partition: !calc doc.platform.partitions.default_shared + exclusive_partition: !calc doc.platform.partitions.default_exclusive + service_partition: !calc doc.platform.partitions.default_service diff --git a/workflow/user.yaml.override_example b/workflow/user.yaml.override_example new file mode 100644 index 00000000000..e25d04df702 --- /dev/null +++ b/workflow/user.yaml.override_example @@ -0,0 +1,37 @@ +user_places: &user_places + EXPROOT: /lfs3/projects/hfv3gfs # !error Please select a EXPROOT directory. + # ie. /scratch4/NCEPDEV/ocean + + # Override scrub areas here. Mandatory on Jet. + LONG_TERM_TEMP: !expand "/lfs3/projects/hfv3gfs/{tools.env('USER')}/scrub" + SHORT_TERM_TEMP: !expand "/lfs3/projects/hfv3gfs/{tools.env('USER')}/scrub" + + # Some other interesting directories: + + # COM directory: + # ROTDIR: !expand "/path/to/some/scrub/area/{tools.env('USER')}/comrot/{doc.names.experiment}" + + # Scrub area for individual jobs: + # DATAROOT: !expand "/path/to/some/scrub/area/RUNDIRS/{doc.names.experiment}" + + # Input conditions. This should usually be specified in the case file: + # ICSDIR: !expand "/path/to/your/FV3ICS" + +accounting: &accounting + # Project for CPU accounting. + cpu_project: hfv3gfs # !error What accounting code do I use to submit jobs? # ie.: global + hpss_project: emc-hwrf # !error Where do I put data on HPSS? # ie.: emc-global + ecflow_machine: xc40-dev # only relevant on WCOSS using ecFlow + + # Choose the partition used for each job type. Default is set in + # the platform yaml files. + shared_partition: !calc doc.platform.partitions.xjet + exclusive_partition: !calc doc.platform.partitions.xjet + service_partition: !calc doc.platform.partitions.service + +user_suite_overrides: + - Search: '**/forecast/*_forecast*' + partition: !calc doc.platform.partitions.tjet_or_ujet + - Search: '**/post/j*_post*' + partition: !calc doc.platform.partitions.xjet + max_tries: 5