From 8c2325bb8cdf23c4fa39cdc4613fb0b6de264915 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 7 Mar 2023 02:20:34 +0000 Subject: [PATCH 01/52] Update nco: config section. --- ush/config_defaults.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index b195e694f3..f7e77de215 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -920,6 +920,7 @@ workflow: COLDSTART: true WARMSTART_CYCLE_DIR: "/path/to/warm/start/cycle/dir" + #---------------------------- # NCO specific variables #----------------------------- @@ -954,6 +955,14 @@ nco: # LOGBASEDIR: # Directory in which the log files from the workflow tasks will be placed. # + # NWGES: + # The beginning portion of the directory that will contain the output + # files from the forecast for a given cycle. For a cycle + # that starts on the date specified by yyyymmdd and hour specified by hh + # (where yyyymmdd and hh are as described above), the directory in which + # the forecast output files will be placed will be: + # $NWGES/$NET/$envir/$RUN.$yyyymmdd/$hh + # # For more information on NCO standards # # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf @@ -964,6 +973,7 @@ nco: NET: "rrfs" RUN: "rrfs" model_ver: "v1.0.0" + OPSROOT: '{{ workflow.EXPT_BASEDIR }}/../nco_dirs' COMROOT: '{{ OPSROOT }}/com' PACKAGEROOT: '{{ OPSROOT }}/packages' @@ -974,6 +984,35 @@ nco: COMIN_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' COMOUT_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' + # + # New additions from RRFS_dev1 + # + NWGES: '{{ OPSROOT }}/nwges' + NWGES_BASEDIR: '{{ NWGES }}' + RRFSE_NWGES: '{{ OPSROOT }}/nwges' + RRFSE_NWGES_BASEDIR: '{{ RRFSE_NWGES }}' + ENSCTRL_NWGES: '{{ OPSROOT }}/nwges' + ENSCTRL_NWGES_BASEDIR: '{{ ENSCTRL_NWGES}}' + + ENSCTRL_COMIN: '{{ COMIN_BASEDIR}}' + ENSCTRL_COMOUT: '{{ COMOUT_BASEDIR}}' + ENSCTRL_COMROOT: '{{ ENSCTRL_COMOUT }}' + ENSCTRL_COMIN_BASEDIR: '{{ ENSCTRL_COMIN }}' + ENSCTRL_COMOUT_BASEDIR: '{{ ENSCTRL_COMROOT}}/{{ envir}}' + + # + #----------------------------------------------------------------------- + # + # FG_ROOTDIR: + # First Guess Root Directory, APP will find corresponding first guess + # fields from this directory. RRFS will find FG under NWGES_BASEDIR, + # but we needs to explicitly specify where to find FG for RTMA. + # So this parameter only matters for RTMA + # + #----------------------------------------------------------------------- + # + FG_ROOTDIR: "" + # #----------------------------------------------------------------------- # From 4a2fc0ec98942d590d44e979844843fdcf316d8f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 7 Mar 2023 02:52:44 +0000 Subject: [PATCH 02/52] Add the DO_ parameters from rrfs_dev. --- ush/config_defaults.yaml | 177 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index f7e77de215..d9e1cedfc0 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -5,7 +5,7 @@ metadata: description: >- Default configuration for an experiment. The valid values for most of the parameters are specified in valid_param_vals.yaml - version: !!str '1.0' + version: !!str '2.0' #---------------------------- # USER config parameters #---------------------------- @@ -214,9 +214,12 @@ platform: # #----------------------------------------------------------------------- # + # SCHED_NATIVE_CMD # Allows an extra parameter to be passed to SCHEDULER (SLURM/PBSPRO) via # XML Native command # + #----------------------------------------------------------------------- + # SCHED_NATIVE_CMD: "" # @@ -424,11 +427,16 @@ workflow: # #----------------------------------------------------------------------- # + # WORKFLOW_ID # Unique ID for workflow run that will be set in setup.py # + # TAG + # Prepend all task name with this name if specified + # #----------------------------------------------------------------------- # WORKFLOW_ID: "" + TAG: "" # #----------------------------------------------------------------------- # @@ -1175,6 +1183,173 @@ workflow_switches: RUN_TASK_BIAS_CORRECTION_O3: false RUN_TASK_BIAS_CORRECTION_PM25: false +#---------------------------- +# DO_ parameters. These look like workflow switches since some +# of them are used in FV3LAM_wflow.xml +#----------------------------- +rrfs: + # + #----------------------------------------------------------------------- + # + # Set parameters associated with running retrospective experiments. Definitions: + # + # DO_RRFS_DEV: + # Flag to turn on RRFS dev workflow. Should be removed + # in the future once SRW and RRFS dev workflows are completely integrated + # + # DO_RETRO: + # Flag turn on the retrospective experiments. + # + # DO_SPINUP: + # Flag turn on the spin-up cycle. + # + # DO_POST_SPINUP: + # Flag turn on the UPP for spin-up cycle. + # + # DO_POST_PROD: + # Flag turn on the UPP for prod cycle. + # + # DO_PARALLEL_PRDGEN: + # Flag turn on parallel wgrib2 runs in prdgen . + # + #----------------------------------------------------------------------- + # + DO_RRFS_DEV: false # Turn this on to test RRFS dev workflow + DO_RETRO: true # Turn this on temporarily + DO_POST_PROD: false # Turn this off temporarily + DO_SPINUP: false + DO_POST_SPINUP: false + DO_PARALLEL_PRDGEN: false + # + #----------------------------------------------------------------------- + # + # Set switches associated with ensembles. Definitions: + # + # DO_ENSCONTROL: + # In ensemble mode, whether or not to run member 1 as control member + # + # DO_GSIOBSERVER: + # Decide whether or not to run GSI observer + # + # DO_ENKFUPDATE: + # Decide whether or not to run EnKF update for the ensemble members + # + # DO_ENKF_RADAR_REF: + # Decide whether or not to run Radar Reflectivity EnKF update for the ensemble members + # + # DO_ENVAR_RADAR_REF: + # Decide whether or not to run Radar Reflectivity hybrid analysis + # + # DO_RECENTER: + # Decide whether or not to run recenter for the ensemble members + # + # DO_SAVE_INPUT: + # Decide whether or not to run save_input after the DA analysis + # + # DO_ENS_GRAPHICS: + # Flag to turn on/off ensemble graphics. Turns OFF deterministic + # graphics. + # + # DO_ENSPOST: + # Flag to turn on/off python ensemble postprocessing for WPC testbeds. + # + # DO_ENSINIT: + # Decide whether or not to do ensemble initialization by running 1 timestep ensemble + # forecast and recentering on the deterministic analysis + # + # DO_ENSFCST: + # Flag that determines whether to run ensemble free forecasts (for + # each set of specified cycles). If this is set to "TRUE", NUM_ENS_MEMBERS_FCST + # forecasts are run for each specified cycle, each with a different set of stochastic + # seed values. + # + #----------------------------------------------------------------------- + # + DO_ENSFCST: false + DO_ENSCONTROL: false + DO_GSIOBSERVER: false + DO_ENKFUPDATE: false + DO_ENKF_RADAR_REF: false + DO_ENVAR_RADAR_REF: false + DO_RECENTER: false + DO_ENS_GRAPHICS: false + DO_ENSPOST: false + DO_ENSINIT: false + DO_SAVE_INPUT: false + # + #----------------------------------------------------------------------- + # + # Set switches associated with running data assimilation. Definitions: + # + # DO_DACYCLE: + # Flag that determines whether to run a data assimilation cycle. + # + # DO_SURFACE_CYCLE: + # Flag that determines whether to continue cycle surface fields. + # + # DO_SOIL_ADJUST: + # Flag that determines whether to adjust soil T and Q based on + # the lowest level T/Q analysis increments. + # + # DO_UPDATE_BC: + # Flag that determines whether to update boundary conditions based on the + # analysis results + # + # DO_RADDA: + # Flag that determines whether to assimilate satellite radiance data + # + # DO_BUFRSND: + # Decide whether or not to run EMC BUFR sounding + # + # DO_SMOKE_DUST + # Flag turn on smoke and dust for RRFS-SD + # + #----------------------------------------------------------------------- + # + DO_DACYCLE: false + DO_SURFACE_CYCLE: false + DO_SOIL_ADJUST: false + DO_UPDATE_BC: false + DO_RADDA: false + DO_BUFRSND: false + DO_SMOKE_DUST: false + # + #----------------------------------------------------------------------- + # + # Parameters for JEDI options + # DO_JEDI_ENVAR_IODA: + # Flag turn on the JEDI-IODA converters for EnVAR. It requires GSI + # to produce NetCDF diag files + #----------------------------------------------------------------------- + # + DO_JEDI_ENVAR_IODA: false + # + #----------------------------------------------------------------------- + # + # Parameters for analysis options + # DO_NONVAR_CLDANAL: + # Flag turn on the non-var cloud analysis. + # DO_REFL2TTEN: + # Flag turn on the radar reflectivity to temperature tendenecy. + # DO_NLDN_LGHT + # Flag turn on processing NLDN NetCDF lightning data + # + #----------------------------------------------------------------------- + # + DO_NONVAR_CLDANAL: false + DO_REFL2TTEN: false + DO_NLDN_LGHT: false + DO_SMOKE_DUST: false + # + #----------------------------------------------------------------------- + # + # IS_RTMA: + # If true, some ICs,LBCs,GSI rocoto tasks will be turned off + # + #----------------------------------------------------------------------- + # + IS_RTMA: false + #---------------------------- # MAKE GRID config parameters #----------------------------- From 6a13df3ec7216c4977ce928d9c4548e4558a1560 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 7 Mar 2023 17:59:13 +0000 Subject: [PATCH 03/52] Put values for RRFS_NA_13km predef grid from RRFS_dev1 in comments. --- ush/predef_grid_params.yaml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index 4b1ad0c451..13615606dc 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -877,20 +877,21 @@ #----------------------------------------------------------------------- # # Emulation of GSD's RAP domain with ~13km cell size. +# Note: Close to RRFS_dev1's GSD_RAP13km. Differences are given in comments # #----------------------------------------------------------------------- # "RRFS_NA_13km": GRID_GEN_METHOD: "ESGgrid" - ESGgrid_LON_CTR: -112.5 - ESGgrid_LAT_CTR: 55.0 + ESGgrid_LON_CTR: -112.5 # -106 in RRFS_dev1 + ESGgrid_LAT_CTR: 55.0 # 54 ESGgrid_DELX: 13000.0 ESGgrid_DELY: 13000.0 - ESGgrid_NX: 912 - ESGgrid_NY: 623 - ESGgrid_PAZI: 0.0 + ESGgrid_NX: 912 # 960 + ESGgrid_NY: 623 # 960 + ESGgrid_PAZI: 0.0 ESGgrid_WIDE_HALO_WIDTH: 6 - DT_ATMOS: 75 + DT_ATMOS: 75 # 50 LAYOUT_X: 16 LAYOUT_Y: 16 BLOCKSIZE: 30 @@ -898,14 +899,14 @@ WRTCMP_write_groups: 1 WRTCMP_write_tasks_per_group: 16 WRTCMP_output_grid: "rotated_latlon" - WRTCMP_cen_lon: -113.0 - WRTCMP_cen_lat: 55.0 - WRTCMP_lon_lwr_left: -61.0 - WRTCMP_lat_lwr_left: -37.0 - WRTCMP_lon_upr_rght: 61.0 - WRTCMP_lat_upr_rght: 37.0 - WRTCMP_dlon: 0.1169081 - WRTCMP_dlat: 0.1169081 + WRTCMP_cen_lon: -113.0 # -106 + WRTCMP_cen_lat: 55.0 # 54.0 + WRTCMP_lon_lwr_left: -61.0 # -55.82538869 + WRTCMP_lat_lwr_left: -37.0 # -48.57685654 + WRTCMP_lon_upr_rght: 61.0 # 55.82538869 + WRTCMP_lat_upr_rght: 37.0 # 48.57685654 + WRTCMP_dlon: 0.1169081 # 0.11691181 + WRTCMP_dlat: 0.1169081 # 0.11691181 # #----------------------------------------------------------------------- # From 257bcc1415a84d1a8c177a51cac61c9d640e4bbd Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 8 Mar 2023 14:26:41 +0000 Subject: [PATCH 04/52] Add workflow definitons. --- ush/config_defaults.yaml | 245 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d9e1cedfc0..d65dcaa3f9 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -51,9 +51,14 @@ user: # ACCOUNT: # The account under which to submit jobs to the queue. # + # SERVICE_ACCOUNT: + # The account under which to submit non-reservation jobs to the queue. + # Defaults to ACCOUNT if not set. + # #----------------------------------------------------------------------- MACHINE: "BIG_COMPUTER" ACCOUNT: "" + SERVICE_ACCOUNT: "" HOMEdir: '{{ user.HOMEdir }}' USHdir: '{{ user.USHdir }}' @@ -159,6 +164,47 @@ platform: # If this is not set or set to an empty string, it will be (re)set to a # machine-dependent value. # + # PARTITION_GRAPHICS + # Partition to which graphics tasks are submitted + # + # QUEUE_GRAPHICS + # Queue for graphics tasks + # + # PARTITION_ANALYSIS + # Partion to which analysis tasks are submitted + # + # QUEUE_ANALYSIS: + # The queue or QOS to which the task that runs a analysis is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # PARTITION_PRDGEN: + # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), + # the partition to which the task that remaps output grids is submitted. If + # this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. This is not used if SCHED is not set to + # "slurm". + # + # QUEUE_PRDGEN: + # The queue or QOS to which the task that prodgen is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # PARTITION_POST: + # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), + # the partition to which the task that upp is submitted. + # + # QUEUE_POST: + # The queue or QOS to which the task that upp is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # RESERVATION: + # The reservation for major tasks. + # + # RESERVATION_POST: + # The reservation for post tasks. + # #----------------------------------------------------------------------- # WORKFLOW_MANAGER: "" @@ -175,6 +221,16 @@ platform: QUEUE_HPSS: "" PARTITION_FCST: "" QUEUE_FCST: "" + PARTITION_GRAPHICS: "" + QUEUE_GRAPHICS: "" + PARTITION_ANALYSIS: "" + QUEUE_ANALYSIS: "" + PARTITION_PRDGEN: "" + QUEUE_PRDGEN: "" + PARTITION_POST: "" + QUEUE_POST: "" + RESERVATION: "" + RESERVATION_POST: "" # #----------------------------------------------------------------------- # @@ -821,6 +877,78 @@ workflow: # This is valid only when FCST_LEN_HRS = -1. # This pattern is recurred for all cycle dates. # + # STARTYEAR,STARTMONTH,STARTDAY,STARTHOUR: + # Year,month,day and hour of the first cycle in the set of forecasts to run + # ENDYEAR,ENDMONTH,ENDDAY,ENDHOUR + # Year,month,day and hour of the last cycle in the set of forecasts to run + # + # CYCL_HRS: + # An array containing the hours of the day at which to launch forecasts. + # Forecasts are launched at these hours on each day from DATE_FIRST_CYCL + # to DATE_LAST_CYCL, inclusive. Each element of this array must be a + # two-digit string representing an integer that is less than or equal to + # 23, e.g. "00", "03", "12", "23". + # + # CYCL_HRS_SPINSTART: + # An array containing the hours of the day at which the spin up cycle starts. + # + # CYCL_HRS_PRODSTART: + # An array containing the hours of the day at which the product cycle starts, + # from cold start input or from spin-up cycle forcast + # + # CYCL_HRS_PRODSTART_ENS: + # An array containing the hours of the day at which the product cycle starts, + # from cold start input or from spin-up cycle forcast, for the ensemble. + # this is only needed for locating the RRFS ensemble files for the the deterministic hybrid analysis. + # + # CYCL_HRS_RECENTER: + # An array containing the hours of the day at which the ensemble recenter is on + # + # CYCL_HRS_STOCH + # An array containing the hours of the day at which the stochastics physcis is on + # this might include: SPPT, SHUM, SKEB, SPP, LSM_SPP + # + # BOUNDARY_LEN_HRS + # The length of boundary condition for normal forecast, in integer hours. + # + # BOUNDARY_LONG_LEN_HRS + # The length of boundary condition for long forecast, in integer hours. + # + # BOUNDARY_PROC_GROUP_NUM + # The number of groups used to run make_lbcs, in integer from 1 to forecast longest hours. + # + # FCST_LEN_HRS_SPINUP: + # The length of each forecast in spin up cycles, in integer hours. + # + # FCST_LEN_HRS_CYCLES: + # The length of forecast for each cycle, in integer hours. + # When it empty, all forecast will be FCST_LEN_HRS + # + # DA_CYCLE_INTERV: + # Data assimilation cycle interval, in integer hours for now. + # + # RESTART_INTERVAL: + # frequency of the output restart files (unit:hour). + # Default=0: restart files are produced at the end of a forecast run + # For example, i) RESTART_INTERVAL: 1 -1 => restart files are produced + # every hour with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory + # ii) RESTART_INTERVAL: 1 2 5 => restart files are produced only when + # fh = 1, 2, and 5. + # + # + # RESTART_INTERVAL_LONG: + # Set up frequenency or list of the forecast hours that FV3 should + # generate the restart files. + # + # POSTPROC_LEN_HRS: + # The length of post process, in integer hours. + # + # POSTPROC_LONG_LEN_HRS: + # The length of long post process, in integer hours. + # + # CYCL_HRS_HYB_FV3LAM_ENS: + # An array containing the hours of the day at which the GSI hybrid using FV3LAM ensemeble. + # #----------------------------------------------------------------------- # DATE_FIRST_CYCL: "YYYYMMDDHH" @@ -833,6 +961,31 @@ workflow: - 12 - 6 + STARTYEAR: "YYYY" + STARTMONTH: "MM" + STARTDAY: "DD" + STARTHOUR: "HH" + ENDYEAR: "YYYY" + ENDMONTH: "MM" + ENDDAY: "DD" + ENDHOUR: "HH" + CYCL_HRS: [] + CYCL_HRS_SPINSTART: [] + CYCL_HRS_PRODSTART: [] + CYCL_HRS_PRODSTART_ENS: [] + CYCL_HRS_RECENTER: [] + CYCL_HRS_STOCH: [] + BOUNDARY_LEN_HRS: 0 + BOUNDARY_LONG_LEN_HRS: 0 + BOUNDARY_PROC_GROUP_NUM: 1 + POSTPROC_LEN_HRS: 1 + POSTPROC_LONG_LEN_HRS: 1 + FCST_LEN_HRS_SPINUP: 1 + FCST_LEN_HRS_CYCLES: [] + DA_CYCLE_INTERV: 3 + RESTART_INTERVAL: 0 # was 3,6 in rrfs_dev + RESTART_INTERVAL_LONG: 0 # was 3.6 in rrfs_dev + CYCL_HRS_HYB_FV3LAM_ENS: [ "99" ] # #----------------------------------------------------------------------- # @@ -876,10 +1029,17 @@ workflow: # messages are printed to out. Note that if DEBUG is set to TRUE, then # VERBOSE will also get reset to TRUE if it isn't already. # + # SAVE_CYCLE_LOG: + # This is a flag that determines whether or not save + # the information related to data assimilation cycling, such as background + # used in each cycle + # #----------------------------------------------------------------------- # VERBOSE: true DEBUG: false + SAVE_CYCLE_LOG: true + # #----------------------------------------------------------------------- # @@ -1350,6 +1510,91 @@ rrfs: # IS_RTMA: false +#---------------------------------- +# Start times for different setups +#---------------------------------- +start_time: + START_TIME_SPINUP: 01:10:00 + START_TIME_PROD: 02:20:00 + START_TIME_CONVENTIONAL_SPINUP: 00:40:00 + START_TIME_LATE_ANALYSIS: 01:40:00 + START_TIME_CONVENTIONAL: 00:40:00 + START_TIME_NSSLMOSIAC: 00:45:00 + START_TIME_LIGHTNINGNC: 00:45:00 + START_TIME_PROCSMOKE: 00:45:00 + +#---------------------------------- +# Cycle definiton for each group +#---------------------------------- +cycledefs: + #----------------------------------------------------------------------- + # + # Set cycle definition for each group. The cycle definition sets the cycle + # time that the group will run. It has two way to set up: + # 1) 00 HHs DDs MMs YYYYs * + # HHs can be "01-03/01" or "01,02,03" or "*" + # DDs,MMs can be "01-03" or "01,02,03" or "*" + # YYYYs can be "2020-2021" or "2020,2021" or "*" + # 2) start_time(YYYYMMDDHH00) end_time(YYYYMMDDHH00) interval(HH:MM:SS) + # for example: 202104010000 202104310000 12:00:00 + # The default cycle definition is: + # "00 01 01 01 2100 *" + # which will likely never get to run. + # + # Definitions: + # + # AT_START_CYCLEDEF: + # cycle definition for "at start" group + # This group runs: make_grid, make_orog, make_sfc_climo + # + # INITIAL_CYCLEDEF: + # cycle definition for "initial" group + # This group runs get_extrn_ics, make_ics + # + # BOUNDARY_CYCLEDEF: + # cycle definition for "boundary" group + # This group runs: get_extrn_lbcs,make_lbcs + # + # BOUNDARY_LONG_CYCLEDEF: + # cycle definition for "boundary_long" group + # This group runs: get_extrn_lbcs_long,make_lbcs + # + # SPINUP_CYCLEDEF: + # cycle definition for spin-up cycle group + # This group runs: anal_gsi_input_spinup and data process, run_fcst_spinup, run_post_spinup + # + # PROD_CYCLEDEF: + # cycle definition for product cycle group + # This group runs: anal_gsi_input and data process, run_fcst, python_skewt, run_post, run_clean + # + # SAVEINPUT_CYCLEDEF: + # cycle definition for saving INPUT files + # This group runs: saveinput + # + # RECENTER_CYCLEDEF: + # cycle definition for recenter cycle group + # This group runs: recenter + # + # PRODLONG_CYCLEDEF: + # same as PROD_CYCLEDEF, but for long forecast + # + # ARCHIVE_CYCLEDEF: + # cycle definition for "archive" group + # This group runs: run_archive + # + #----------------------------------------------------------------------- + # + AT_START_CYCLEDEF: "00 01 01 01 2100 *" + INITIAL_CYCLEDEF: "00 01 01 01 2100 *" + BOUNDARY_CYCLEDEF: "00 01 01 01 2100 *" + BOUNDARY_LONG_CYCLEDEF: "00 01 01 01 2100 *" + SPINUP_CYCLEDEF: "00 01 01 01 2100 *" + PROD_CYCLEDEF: "00 01 01 01 2100 *" + PRODLONG_CYCLEDEF: "00 01 01 01 2100 *" + RECENTER_CYCLEDEF: "00 01 01 01 2100 *" + ARCHIVE_CYCLEDEF: "00 01 01 01 2100 *" + SAVEINPUT_CYCLEDEF: "00 01 01 01 2100 *" + #---------------------------- # MAKE GRID config parameters #----------------------------- From 1e10b56120166e9d743c39a3f1c7d27d7e8b2061 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 8 Mar 2023 17:44:12 +0000 Subject: [PATCH 05/52] Modify FIXdir. --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d65dcaa3f9..d854922b97 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -740,7 +740,7 @@ workflow: # #----------------------------------------------------------------------- # - FIXdir: '{{ EXPTDIR if workflow_switches.RUN_TASK_MAKE_GRID else [user.HOMEdir, "fix"]|path_join }}' + FIXdir: '{{ [EXPTDIR, "fix"]|path_join if workflow_switches.RUN_TASK_MAKE_GRID else [user.HOMEdir, "fix"]|path_join }}' FIXam: '{{ [FIXdir, "fix_am"]|path_join }}' FIXclim: '{{ [FIXdir, "fix_clim"]|path_join }}' FIXlam: '{{ [FIXdir, "fix_lam"]|path_join }}' From 006f56295b490c26c17640401c10f832e23b284d Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 9 Mar 2023 04:24:06 +0000 Subject: [PATCH 06/52] Add 3km HRRRIC predef grid. --- ush/predef_grid_params.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index 13615606dc..afc53149f6 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -222,6 +222,41 @@ # #----------------------------------------------------------------------- # +# The RRFS CONUS domain with ~3km cells (C3357, HRRRIC). +# +#----------------------------------------------------------------------- +# +"RRFS_CONUS_3km_HRRRIC": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 1748 + ESGgrid_NY: 1038 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 30 + LAYOUT_Y: 16 + BLOCKSIZE: 32 + QUILTING: + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 16 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 1746 + WRTCMP_ny: 1014 + WRTCMP_lon_lwr_left: -122.17364391 + WRTCMP_lat_lwr_left: 21.88588562 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# # The RRFS CONUS domain with ~3km cells that can be initialized from the HRRR. # #----------------------------------------------------------------------- From 8e6787b9fcaedf3f5f2d7f265d00222290dcf415 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 9 Mar 2023 16:18:06 +0000 Subject: [PATCH 07/52] Add more stuff needed for workflow generation. --- fix/fix_am/global_hyblev_fcst_rrfsL65.txt | 68 +++++ parm/input.nml.FV3 | 7 +- ush/config_defaults.yaml | 14 + ush/create_model_configure_file.py | 1 - ush/generate_FV3LAM_wflow.py | 344 +++++++++++++++++----- ush/set_FV3nml_ens_stoch_seeds.py | 6 +- 6 files changed, 357 insertions(+), 83 deletions(-) create mode 100644 fix/fix_am/global_hyblev_fcst_rrfsL65.txt diff --git a/fix/fix_am/global_hyblev_fcst_rrfsL65.txt b/fix/fix_am/global_hyblev_fcst_rrfsL65.txt new file mode 100644 index 0000000000..b1ce286eac --- /dev/null +++ b/fix/fix_am/global_hyblev_fcst_rrfsL65.txt @@ -0,0 +1,68 @@ + 2 66 + 0.000 1.00000000 + 0.000 0.99787363 + 0.000 0.99547387 + 0.017 0.99276698 + 0.234 0.98971388 + 1.058 0.98626929 + 3.103 0.98238159 + 7.211 0.97799245 + 14.450 0.97303658 + 26.130 0.96744145 + 43.805 0.96112727 + 69.754 0.95400221 + 107.500 0.94595747 + 160.901 0.93687684 + 233.980 0.92663902 + 330.883 0.91511905 + 455.809 0.90219033 + 612.932 0.88772712 + 806.289 0.87160769 + 1039.659 0.85371818 + 1316.413 0.83395695 + 1639.350 0.81223965 + 2010.516 0.78850460 + 2431.016 0.76271850 + 2900.827 0.73488203 + 3418.626 0.70503512 + 3981.638 0.67326135 + 4585.535 0.63969119 + 5224.384 0.60450347 + 5890.665 0.56792486 + 6575.364 0.53022703 + 7268.150 0.49172137 + 7957.623 0.45275150 + 8631.631 0.41368369 + 9277.633 0.37489594 + 9883.087 0.33676625 + 10435.842 0.29966091 + 10924.498 0.26392352 + 11338.722 0.22986547 + 11669.502 0.19775844 + 11909.318 0.16782909 + 12052.236 0.14025627 + 12093.928 0.11517042 + 12031.620 0.09265514 + 11863.989 0.07275035 + 11591.011 0.05545674 + 11213.791 0.04074101 + 10734.376 0.02854147 + 10155.564 0.01877378 + 9480.727 0.01133629 + 8714.452 0.00610706 + 7877.419 0.00279793 + 7004.872 0.00097247 + 6132.608 0.00018806 + 5296.374 0.00000379 + 4530.025 0.00000000 + 3843.468 0.00000000 + 3229.725 0.00000000 + 2681.893 0.00000000 + 2193.544 0.00000000 + 1758.738 0.00000000 + 1372.014 0.00000000 + 1028.378 0.00000000 + 723.285 0.00000000 + 452.613 0.00000000 + 212.637 0.00000000 + 0.000 0.00000000 diff --git a/parm/input.nml.FV3 b/parm/input.nml.FV3 index 8cdf3258e5..4263d3f56a 100644 --- a/parm/input.nml.FV3 +++ b/parm/input.nml.FV3 @@ -36,7 +36,7 @@ checker_tr = .false. filtered_terrain = .true. gfs_dwinds = .true. - levp = 65 + levp = 66 nt_checker = 0 / @@ -95,14 +95,15 @@ make_nh = .true. mountain = .false. n_split = 5 - n_sponge = 24 + n_sponge = 9 + sg_cutoff = 10000.0 n_zs_filter = 0 na_init = 1 ncep_ic = .false. nggps_ic = .true. no_dycore = .false. nord = 3 - npz = 64 + npz = 65 nrows_blend = 10 ntiles = 1 nudge_qv = .false. diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d854922b97..352700639c 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -699,6 +699,11 @@ workflow: FIELD_TABLE_FP: '{{ [EXPTDIR, FIELD_TABLE_FN]|path_join }}' NEMS_CONFIG_FP: '{{ [EXPTDIR, NEMS_CONFIG_FN]|path_join }}' FV3_NML_FP: '{{ [EXPTDIR, FV3_NML_FN]|path_join }}' + FV3_NML_FN: '{{ FV3_NML_BASE_SUITE_FN[:-4] }}' + FV3_NML_CYCSFC_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_cycsfc"]|join ]|path_join }}' + FV3_NML_RESTART_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_restart"]|join ]|path_join }}' + FV3_NML_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_stoch"]|join ]|path_join }}' + FV3_NML_RESTART_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_restart_stoch"]|join ]|path_join }}' FCST_MODEL: "ufs-weather-model" WFLOW_XML_FN: "FV3LAM_wflow.xml" @@ -3168,6 +3173,15 @@ global: HALO_BLEND: 10 # #----------------------------------------------------------------------- + # + # PRINT_DIFF_PGR: + # Option to turn on/off pressure tendency diagnostic + # + #----------------------------------------------------------------------- + # + PRINT_DIFF_PGR: false + # + #----------------------------------------------------------------------- # #---------------------------- diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 8fa978755c..e29599d445 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -338,7 +338,6 @@ def setUp(self): set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) set_env_var("PE_MEMBER01", 24) - set_env_var("FCST_LEN_HRS", 72) set_env_var("DT_ATMOS", 1) set_env_var("OMP_NUM_THREADS_RUN_FCST", 1) set_env_var("RESTART_INTERVAL", 4) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 2355468d21..83d8d57896 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -284,8 +284,12 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # # Copy or symlink fix files # - if SYMLINK_FIX_FILES: + mkdir_vrfy("-p", FIXam) + if RUN_TASK_MAKE_GRID and RUN_ENVIR != "nco": + ln_vrfy(f"""-fsn '{HOMEdir}/fix/fix_am/global_hyblev_fcst_rrfsL65.txt'""" + f""" '{FIXam}/global_hyblev_fcst_rrfsL65.txt'""") + if SYMLINK_FIX_FILES: log_info( f""" Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): @@ -294,7 +298,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de verbose=verbose, ) - ln_vrfy(f"""-fsn '{FIXgsm}' '{FIXam}'""") + ln_vrfy(f"""-fsn '{FIXgsm}/'* '{FIXam}'""") else: log_info( @@ -305,8 +309,6 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de verbose=verbose, ) - check_for_preexist_dir_file(FIXam, "delete") - mkdir_vrfy("-p", FIXam) mkdir_vrfy("-p", os.path.join(FIXam, "fix_co2_proj")) num_files = len(FIXgsm_FILES_TO_COPY_TO_FIXam) @@ -473,6 +475,8 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de "target_lon": LON_CTR, "target_lat": LAT_CTR, "nrows_blend": HALO_BLEND, + "regional_bcs_from_gsi": False, + "write_restart_with_bcs": False, # # Question: # For a ESGgrid type grid, what should stretch_fac be set to? This depends @@ -525,15 +529,9 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de gfs_physics_nml_dict.update({ "kice": kice or None, "lsoil": lsoil or None, - "do_shum": DO_SHUM, - "do_sppt": DO_SPPT, - "do_skeb": DO_SKEB, - "do_spp": DO_SPP, - "n_var_spp": N_VAR_SPP, - "n_var_lndp": N_VAR_LNDP, - "lndp_type": LNDP_TYPE, - "fhcyc": FHCYC_LSM_SPP_OR_NOT, + "print_diff_pgr": PRINT_DIFF_PGR, }) + if CPL_AQM: gfs_physics_nml_dict.update({ "cplaqm": True, @@ -607,12 +605,211 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # if PREDEF_GRID_NAME == "RRFS_NA_3km": settings["fms2_io_nml"] = {"netcdf_default_format": "netcdf4"} + + settings_str = cfg_to_yaml_str(settings) + + log_info( + f""" + The variable 'settings' specifying values of the weather model's + namelist variables has been set as follows:\n""", + verbose=verbose, + ) + log_info("\nsettings =\n\n" + settings_str, verbose=verbose) + # + # ----------------------------------------------------------------------- + # + # Call the set_namelist.py script to create a new FV3 namelist file (full + # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as + # the base (i.e. starting) namelist file, with physics-suite-dependent + # modifications to the base file specified in the yaml configuration file + # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), + # and with additional physics-suite-independent modifications specified + # in the variable "settings" set above. + # + # ----------------------------------------------------------------------- + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_BASE_SUITE_FP, + "-c", + FV3_NML_YAML_CONFIG_FP, + CCPP_PHYS_SUITE, + "-u", + settings_str, + "-o", + FV3_NML_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' + Full path to yaml configuration file for various physics suites: + FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' + Physics suite to extract from yaml configuration file: + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + # + # If not running the TN_MAKE_GRID task (which implies the workflow will + # use pregenerated grid files), set the namelist variables specifying + # the paths to surface climatology files. These files are located in + # (or have symlinks that point to them) in the FIXlam directory. + # + # Note that if running the TN_MAKE_GRID task, this action usually cannot + # be performed here but must be performed in that task because the names + # of the surface climatology files depend on the CRES parameter (which is + # the C-resolution of the grid), and this parameter is in most workflow + # configurations is not known until the grid is created. + # + if not RUN_TASK_MAKE_GRID: + + set_FV3nml_sfc_climo_filenames() + + # + # ----------------------------------------------------------------------- + # + # Generate namelist for surface cycle + # Deleted in newer RRFS_dev version so here for testing only + # + # ----------------------------------------------------------------------- + # + if DO_SURFACE_CYCLE: + if SDF_USES_RUC_LSM: + lsoil=9 + settings = {} + settings["gfs_physics_nml"] = { + "lsoil": lsoil or None + #"fh_dfi_radar": FH_DFI_RADAR # commented out untile develop gets radar tten code + } + + settings_str = cfg_to_yaml_str(settings) + # + # populate the namelist file + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_CYCSFC_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for DA: + FV3_NML_RESTART_FP = '{FV3_NML_CYCSFC_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + + # + # ----------------------------------------------------------------------- + # + # Generate namelist for DA cycle + # + # ----------------------------------------------------------------------- + # + if DO_DACYCLE or DO_ENKFUPDATE: + + if SDF_USES_RUC_LSM: + lsoil = 9 + + lupdatebc = False + if DO_UPDATE_BC: + lupdatebc = False # not ready for setting this to true yet + + settings = {} + settings["fv_core_nml"] = { + "external_ic": False, + "make_nh": False, + "na_init": 0, + "nggps_ic": False, + "mountain": True, + "regional_bcs_from_gsi": lupdatebc, + "warm_start": True, + } + settings["gfs_physics_nml"] = { + "lsoil": lsoil or None + #"fh_dfi_radar": FH_DFI_RADAR # commented out untile develop gets radar tten code + } + + settings_str = cfg_to_yaml_str(settings) + # + # populate the namelist file + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_RESTART_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for DA: + FV3_NML_RESTART_FP = '{FV3_NML_RESTART_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + # + # ----------------------------------------------------------------------- # # Add the relevant tendency-based stochastic physics namelist variables to # "settings" when running with SPPT, SHUM, or SKEB turned on. If running # with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only # include an empty "nam_stochy" stanza. # + # ----------------------------------------------------------------------- + # + settings = {} + settings["gfs_physics_nml"] = { + "do_shum": DO_SHUM, + "do_sppt": DO_SPPT, + "do_skeb": DO_SKEB, + "do_spp": DO_SPP, + "n_var_spp": N_VAR_SPP, + "n_var_lndp": N_VAR_LNDP, + "lndp_type": LNDP_TYPE, + "fhcyc": FHCYC_LSM_SPP_OR_NOT, + } nam_stochy_dict = {} if DO_SPPT: nam_stochy_dict.update( @@ -696,76 +893,71 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de settings["nam_sfcperts"] = nam_sfcperts_dict settings_str = cfg_to_yaml_str(settings) - - log_info( - f""" - The variable 'settings' specifying values of the weather model's - namelist variables has been set as follows:\n""", - verbose=verbose, - ) - log_info("\nsettings =\n\n" + settings_str, verbose=verbose) # - # ----------------------------------------------------------------------- + #----------------------------------------------------------------------- # - # Call the set_namelist.py script to create a new FV3 namelist file (full - # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as - # the base (i.e. starting) namelist file, with physics-suite-dependent - # modifications to the base file specified in the yaml configuration file - # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), - # and with additional physics-suite-independent modifications specified - # in the variable "settings" set above. + # Generate namelist files with stochastic physics if needed # - # ----------------------------------------------------------------------- + #----------------------------------------------------------------------- # - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_BASE_SUITE_FP, - "-c", - FV3_NML_YAML_CONFIG_FP, - CCPP_PHYS_SUITE, - "-u", - settings_str, - "-o", - FV3_NML_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" + if DO_ENSEMBLE and ( DO_SPP or DO_SPPT or DO_SHUM or DO_SKEB or DO_LSM_SPP): + + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_STOCH_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for stochastics: + FV3_NML_STOCH_FP = '{FV3_NML_STOCH_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str ) - + settings_str - ) - # - # If not running the TN_MAKE_GRID task (which implies the workflow will - # use pregenerated grid files), set the namelist variables specifying - # the paths to surface climatology files. These files are located in - # (or have symlinks that point to them) in the FIXlam directory. - # - # Note that if running the TN_MAKE_GRID task, this action usually cannot - # be performed here but must be performed in that task because the names - # of the surface climatology files depend on the CRES parameter (which is - # the C-resolution of the grid), and this parameter is in most workflow - # configurations is not known until the grid is created. - # - if not RUN_TASK_MAKE_GRID: - set_FV3nml_sfc_climo_filenames() + if DO_DACYCLE or DO_ENKFUPDATE: + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_RESTART_FP, + "-u", + settings_str, + "-o", + FV3_NML_RESTART_STOCH_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for stochastics: + FV3_NML_RESTART_STOCH_FP = '{FV3_NML_RESTART_STOCH_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) # # ----------------------------------------------------------------------- diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 08ed944f46..04aa628d43 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -58,7 +58,7 @@ def set_FV3nml_ens_stoch_seeds(cdate): # # ----------------------------------------------------------------------- # - fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" + fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}_base" ensmem_num = ENSMEM_INDX @@ -174,7 +174,7 @@ def setUp(self): mkdir_vrfy("-p", EXPTDIR) cp_vrfy( os.path.join(PARMdir, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml"), + os.path.join(EXPTDIR, "input.nml_base"), ) for i in range(2): mkdir_vrfy( @@ -192,7 +192,7 @@ def setUp(self): set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml_base")) set_env_var("DO_SHUM", True) set_env_var("DO_SKEB", True) set_env_var("DO_SPPT", True) From b233633554a5a3c91ec6963039cf6892a81bc098 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 9 Mar 2023 16:51:36 +0000 Subject: [PATCH 08/52] Add machne file modifications. --- ush/config_defaults.yaml | 54 ++++++++++++++++++++++++++++++++++++ ush/machine/cheyenne.yaml | 4 +++ ush/machine/gaea.yaml | 4 +++ ush/machine/hera.yaml | 25 +++++++++++++++-- ush/machine/jet.yaml | 8 ++++++ ush/machine/odin.yaml | 8 ++++++ ush/machine/orion.yaml | 8 ++++++ ush/machine/singularity.yaml | 8 ++++++ ush/machine/stampede.yaml | 8 ++++++ ush/machine/wcoss2.yaml | 4 +++ 10 files changed, 129 insertions(+), 2 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 352700639c..c3fe7d6686 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -475,6 +475,60 @@ platform: COMINgfs: "" COMINgefs: "" COMINairnow: "/path/to/real/time/airnow/data" + # + #----------------------------------------------------------------------- + # + # Setup default observation locations for data assimilation: + # + # OBSPATH: observation BUFR file path + # OBSPATH_NSSLMOSIAC: location of NSSL radar reflectivity + # LIGHTNING_ROOT: location of lightning observations + # ENKF_FCSTL: location of global ensemble forecast + # FFG_DIR: location of flash flood guidance for QPF comparison + # + # Setup default locations for global SST and update time: + # SST_ROOT: locations of global SST + # SST_update_hour: cycle time for updating SST + # + # Setup default locations for GVF and update time: + # GVF_ROOT: locations of GVF observations + # GVF_update_hour: cycle time for updating GVF + # + # Setup default locations for IMS snow/ice and update time: + # IMSSNOW_ROOT: locations of IMS snow/ice observations + # SNOWICE_update_hour: cycle time for updating snow/ice + # + # Setup default resource data locations for soil surgery and time: + # RAPHRR_SOIL_ROOT: locations of RAP/HRRR forecast netcdf files + # SOIL_SURGERY_time: cycle time for soil surgery + # + # Setup default locations for FIRE_RRFS files and update time + # FIRE_RAVE_DIR + # FIRE_RRFS_update_hour + # + #----------------------------------------------------------------------- + # + ARCHIVEDIR: "" + NCARG_ROOT: "" + NCL_HOME: "" + OBSPATH: "" + OBSPATH_NSSLMOSIAC: "" + LIGHTNING_ROOT: "" + ENKF_FCST: "" + FFG_DIR: "" + SST_ROOT: "" + GVF_ROOT: "" + IMSSNOW_ROOT: "" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "" + SFCOBS_USELIST: "" + + NCL_REGION: "conus" + SST_update_hour: 99 + GVF_update_hour: 99 + SNOWICE_update_hour: 99 + SOIL_SURGERY_time: 9999999999 #----------------------------- # WORKFLOW config parameters diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index 672a8043e8..11f7c601ad 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: regular QUEUE_FCST: regular QUEUE_HPSS: regular + QUEUE_GRAPHICS: regular + QUEUE_ANALYSIS: regular + QUEUE_PRDGEN: regular + QUEUE_POST: regular RUN_CMD_FCST: mpirun -np ${PE_MEMBER01} RUN_CMD_POST: mpirun -np $nprocs RUN_CMD_SERIAL: time diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 8f16435cf9..c12e117959 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: normal QUEUE_FCST: normal QUEUE_HPSS: normal + QUEUE_GRAPHICS: normal + QUEUE_ANALYSIS: normal + QUEUE_PRDGEN: normal + QUEUE_POST: normal RUN_CMD_FCST: srun --export=ALL --mpi=pmi2 -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL --mpi=pmi2 -n $nprocs RUN_CMD_SERIAL: time diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 08bc7a49fd..ece7253990 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: hera + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: hera + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: hera + QUEUE_PRDGEN: batch + PARTITION_POST: hera + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_SERIAL: time @@ -35,7 +43,21 @@ platform: FIXsfc: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_sfc_climo FIXshp: /scratch2/BMC/det/UFS_SRW_App/develop/NaturalEarth EXTRN_MDL_DATA_STORES: hpss aws nomads - + ARCHIVEDIR: "" + NCARG_ROOT: "" + NCL_HOME: "" + OBSPATH: "/scratch2/BMC/zrtrr/rli/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/scratch2/BMC/zrtrr/rli/data/reflectivity" + LIGHTNING_ROOT: "/scratch2/BMC/zrtrr/rli/data/lightning" + ENKF_FCST: "/scratch2/BMC/zrtrr/rli/data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" + GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" + IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" + SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" data: obs: RAP_obs: /scratch2/BMC/public/data/grids/rap/obs @@ -50,4 +72,3 @@ cpl_aqm_parm: NEXUS_INPUT_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/nexus NEXUS_FIX_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/nexus/fix PT_SRC_BASEDIR: /scratch1/RDARCH/rda-arl-gpu/YouHua.Tang/nei2016v1-pt - diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index ae6f8a8b6e..14f9154b70 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: sjet,vjet,kjet,xjet + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: sjet,vjet,kjet,xjet + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: sjet,vjet,kjet,xjet + QUEUE_PRDGEN: batch + PARTITION_POST: sjet,vjet,kjet,xjet + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_SERIAL: time diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index b92c8245b6..48dfd6ea0a 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: workq PARTITION_HPSS: workq QUEUE_HPSS: workq + PARTITION_GRAPHICS: workq + QUEUE_GRAPHICS: workq + PARTITION_ANALYSIS: workq + QUEUE_ANALYSIS: workq + PARTITION_PRDGEN: workq + QUEUE_PRDGEN: workq + PARTITION_POST: workq + QUEUE_POST: workq RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL -n 1 RUN_CMD_SERIAL: srun --export=ALL -n 1 diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 8f03a376e8..76739f5053 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: orion + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: orion + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: orion + QUEUE_PRDGEN: batch + PARTITION_POST: orion + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL RUN_CMD_SERIAL: time diff --git a/ush/machine/singularity.yaml b/ush/machine/singularity.yaml index efa8d9f997..2669fa72bb 100644 --- a/ush/machine/singularity.yaml +++ b/ush/machine/singularity.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: null + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: null + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: null + QUEUE_PRDGEN: batch + PARTITION_POST: null + QUEUE_POST: batch RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} --oversubscribe RUN_CMD_POST: mpirun -np $nprocs --oversubscribe RUN_CMD_SERIAL: time diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml index 6200df86bc..aa24d9d315 100644 --- a/ush/machine/stampede.yaml +++ b/ush/machine/stampede.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: normal PARTITION_HPSS: normal QUEUE_HPSS: normal + PARTITION_GRAPHICS: normal + QUEUE_GRAPHICS: normal + PARTITION_ANALYSIS: normal + QUEUE_ANALYSIS: normal + PARTITION_PRDGEN: normal + QUEUE_PRDGEN: normal + PARTITION_POST: normal + QUEUE_POST: normal RUN_CMD_FCST: ibrun -np $nprocs RUN_CMD_POST: ibrun -np $nprocs RUN_CMD_SERIAL: time diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index be3afa4f22..6b9f3891db 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: dev QUEUE_FCST: dev QUEUE_HPSS: dev_transfer + QUEUE_GRAPHICS: dev + QUEUE_ANALYSIS: dev + QUEUE_PRDGEN: dev + QUEUE_POST: dev RUN_CMD_FCST: mpiexec -n ${PE_MEMBER01} -ppn ${PPN_RUN_FCST} --cpu-bind core -depth ${OMP_NUM_THREADS_RUN_FCST} RUN_CMD_POST: mpiexec -n ${nprocs} RUN_CMD_SERIAL: mpiexec From 83963394ec3d28b429b59e6d5b91f727156b9894 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 9 Mar 2023 17:39:20 +0000 Subject: [PATCH 09/52] Add FIX directories. --- ush/config_defaults.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index c3fe7d6686..b1f797c8a3 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -437,6 +437,21 @@ platform: # FIXsfc: # System directory where surface climatology data is located # + # FIXgsi: + # System directory where GSI fixed files are loacated + # + # FIXsmoke: + # System directory where smoke and dust fixed files are loacated + # + # FIXbufr: + # System directory where bufrsnd fixed files are loacated + # + # FIXcrtm: + # System directory where CRTM fixed files are loacated + # + # FIXcrtmupp: + # System directory where CRTM fixed files specifically for UPP are loacated + # #----------------------------------------------------------------------- # FIXgsm: "" @@ -445,6 +460,11 @@ platform: FIXorg: "" FIXsfc: "" FIXshp: "" + FIXgsi: "" + FIXsmoke: "" + FIXbufr: "" + FIXcrtm: "" + FIXcrtmupp: "" # #----------------------------------------------------------------------- # From 3bffba4f0ba961f9fe5d2a9f19cb2a6f0b8f5b7e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 9 Mar 2023 17:59:47 +0000 Subject: [PATCH 10/52] Add workflow and test case. --- parm/FV3LAM_wflow.xml | 4035 +++++++++++++++++++++++++++++++++++++---- ush/config.rrfs.yaml | 137 ++ 2 files changed, 3863 insertions(+), 309 deletions(-) create mode 100644 ush/config.rrfs.yaml diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index a9297029a9..745a66b6b1 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -13,11 +13,18 @@ See README.xml_templating.md for information on using the Templating mechanisms. - - - - - + + + + + + + + + + + + {%- if partition_default is not none %} -&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}"> +&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}&RRFS_RESERVE;"> {%- else %} &ACCOUNT;&QUEUE_DEFAULT;"> {%- endif %} {%- if partition_hpss is not none %} -&ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}"> +&SERVICE_ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}&RRFS_POST_RESERVE;"> {%- else %} -&ACCOUNT;&QUEUE_HPSS;"> +&SERVICE_ACCOUNT;&QUEUE_HPSS;"> {%- endif %} {%- if partition_fcst is not none %} &ACCOUNT;&QUEUE_FCST;{{ partition_fcst }}"> {%- else %} &ACCOUNT;&QUEUE_FCST;"> {%- endif %} +{%- if partition_analysis is not none %} +&ACCOUNT;&QUEUE_ANALYSIS;{{ partition_analysis }}"> +&ACCOUNT;&QUEUE_ANALYSIS;{{ partition_analysis }}&RRFS_RESERVE;"> +{%- else %} +&ACCOUNT;&QUEUE_ANALYSIS;"> +&ACCOUNT;&QUEUE_ANALYSIS;"> +{%- endif %} +{%- if partition_post is not none %} +&SERVICE_ACCOUNT;&QUEUE_POST;{{ partition_post }}&RRFS_POST_RESERVE;"> +{%- else %} +&SERVICE_ACCOUNT;&QUEUE_POST;"> +{%- endif %} +{%- if partition_prdgen is not none %} +&SERVICE_ACCOUNT;&QUEUE_PRDGEN;{{ partition_prdgen }}&RRFS_POST_RESERVE;"> +{%- else %} +&SERVICE_ACCOUNT;&QUEUE_PRDGEN;"> +{%- endif %} +{%- if partition_graphics is not none %} +{%- if do_ens_graphics %} +&SERVICE_ACCOUNT;&QUEUE_GRAPHICS;{{ partition_graphics }}&RRFS_POST_RESERVE;"> +{%- else %} +&SERVICE_ACCOUNT;&QUEUE_GRAPHICS;{{ partition_graphics }}&RRFS_POST_RESERVE;"> +{%- endif %} +{%- else %} +&SERVICE_ACCOUNT;&QUEUE_GRAPHICS;--exclusive"> +{%- endif %} + + +{% if do_retro %} + + + + + + + +{% else %} + + + + + + + +{% endif %} + + + + + + + + + + +{%- if do_retro %} + + + + + + + + +{% else %} +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +{% endif %} + + + + + + + + + + ]> + + +{%- if do_retro %} +{% else %} + +{%- endif %} {# Double quotes are required inside the strftime! Expect an error from reading the template if using single quotes. #} + + +{%- if not do_rrfs_dev %} {{ cdate_first_cycl.strftime("%M %H %d %m %Y *") }} - - {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} - + {{ cdate_first_cycl.strftime("%M %H %d %m %Y *") }} + {%- if cycl_next != date_first_cycl %} {{- cycl_next ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} {%- endif %} + {%- if fcst_len_hrs == -1 %} {%- if num_fcst_len_cycl >= 1 %} {{- date_first_cycl ~ " " ~ date_1st_last_cycl ~ " " ~ "24:00:00" -}} @@ -230,6 +389,46 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. {{- date_4th_cycl ~ " " ~ date_4th_last_cycl ~ " " ~ "24:00:00" -}} {%- endif %} {%- endif %} + + {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} + {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} + + {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} + {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} + {{- date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}} + +{%- if do_save_input %} + {{ date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq }} +{%- endif %} + {{ date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq }} + {{ date_first_cycl ~ " " ~ date_last_cycl ~ " " ~ cycl_freq }} + +{%- else -%} + + {{ at_start_cycledef }} + {{ initial_cycledef }} + {{ boundary_cycledef }} + {{ boundary_long_cycledef }} + + {{ spinup_cycledef }} + {{ prod_cycledef }} + {{ prodlong_cycledef }} + +{%- if do_save_input %} + {{ saveinput_cycledef }} +{%- endif %} + {{ recenter_cycledef }} + {{ archive_cycledef }} + +{%- endif -%} + + {%- if run_envir == "nco" %} &LOGDIR;/FV3LAM_wflow.{{ workflow_id }}.log @@ -265,11 +464,11 @@ J-job then uses exec to run the J-job (while also terminating the LOAD_- MODULES_RUN_TASK_FP script. --> -{%- if run_task_make_grid %} +{%- if run_task_make_grid %} &RSRV_DEFAULT; @@ -278,7 +477,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_grid }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_MAKE_GRID; + &TAG;&TN_MAKE_GRID; &LOGDIR;/&TN_MAKE_GRID;&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -290,11 +489,11 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_make_orog %} +{%- if run_task_make_orog %} &RSRV_DEFAULT; @@ -303,7 +502,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_orog }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_MAKE_OROG; + &TAG;&TN_MAKE_OROG; &LOGDIR;/&TN_MAKE_OROG;&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -323,11 +522,11 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_make_sfc_climo %} +{%- if run_task_make_sfc_climo %} &RSRV_DEFAULT; @@ -336,7 +535,7 @@ MODULES_RUN_TASK_FP script. {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_MAKE_SFC_CLIMO; + &TAG;&TN_MAKE_SFC_CLIMO; &LOGDIR;/&TN_MAKE_SFC_CLIMO;&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -363,12 +562,12 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_nexus_gfs_sfc %} - +{%- if run_task_nexus_gfs_sfc %} + {%- if do_real_time %} &RSRV_DEFAULT; @@ -378,16 +577,16 @@ MODULES_RUN_TASK_FP script. &LOAD_MODULES_RUN_TASK_FP; "&TN_NEXUS_GFS_SFC;" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC" {{ nnodes_nexus_gfs_sfc }}:ppn={{ ppn_nexus_gfs_sfc }} {%- if machine not in ["GAEA", "NOAACLOUD"] %} - {%- if not do_real_time %} + {%- if not do_real_time %} {{ mem_nexus_gfs_sfc }} - {%- endif %} + {%- endif %} {%- endif %} {{ wtime_nexus_gfs_sfc }} &NCORES_PER_NODE; {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; {%- endif %} - &TN_NEXUS_GFS_SFC; + &TAG;&TN_NEXUS_GFS_SFC; &LOGDIR;/&TN_NEXUS_GFS_SFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -398,14 +597,18 @@ MODULES_RUN_TASK_FP script. LOGDIR&LOGDIR; SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} -{%- if do_real_time %} + {%- if do_real_time %} &COMINgfs;/gfs.@Y@m@d/@H/atmos -{%- endif %} + {%- endif %} - + {%- endif %} + {%- if run_task_nexus_emission %} {% for h in range(0, num_split_nexus) %}{{ " %02d" % h }}{% endfor %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_NEXUS_EMISSION;" "&JOBSdir;/JREGIONAL_NEXUS_EMISSION" {%- if machine in ["HERA"] %} @@ -427,32 +630,35 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_nexus_emission }} &NCORES_PER_NODE; - &TN_NEXUS_EMISSION;_#nspt# + &TAG;&TN_NEXUS_EMISSION;_#nspt# &LOGDIR;/&TN_NEXUS_EMISSION;_@Y@m@d@H_s#nspt#&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; - PDY@Y@m@d + PDY@Y@m@d cyc@H subcyc@M nspt#nspt# - LOGDIR&LOGDIR; - SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} - + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} {%- if run_task_nexus_gfs_sfc %} - + {%- endif %} - + + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_NEXUS_POST_SPLIT;" "&JOBSdir;/JREGIONAL_NEXUS_POST_SPLIT" {{ nnodes_nexus_post_split }}:ppn={{ ppn_nexus_post_split }} {{ wtime_nexus_post_split }} &NCORES_PER_NODE; - &TN_NEXUS_POST_SPLIT; + &TAG;&TN_NEXUS_POST_SPLIT; &LOGDIR;/&TN_NEXUS_POST_SPLIT;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -469,12 +675,12 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_fire_emission %} - +{%- if run_task_fire_emission %} + {%- if do_aqm_save_fire %} &RSRV_HPSS; @@ -493,7 +699,7 @@ MODULES_RUN_TASK_FP script. {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; {%- endif %} - &TN_FIRE_EMISSION; + &TAG;&TN_FIRE_EMISSION; &LOGDIR;/&TN_FIRE_EMISSION;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -505,19 +711,19 @@ MODULES_RUN_TASK_FP script. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} {%- endif %} -{%- if run_task_point_source %} - +{%- if run_task_point_source %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_POINT_SOURCE;" "&JOBSdir;/JREGIONAL_POINT_SOURCE" {{ nnodes_point_source }}:ppn={{ ppn_point_source }} {{ wtime_point_source }} &NCORES_PER_NODE; - &TN_POINT_SOURCE; + &TAG;&TN_POINT_SOURCE; &LOGDIR;/&TN_POINT_SOURCE;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -538,26 +744,46 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_get_extrn_ics %} - +{%- if not do_ensfcst %} +{%- if do_rrfs_dev %} + + + + +{%- if do_spinup %} + spinupcyc prodcyc,prodcyc_long + spinup prod +{% else %} + prodcyc,prodcyc_long + prod +{%- endif %} - &RSRV_HPSS; - &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_ICS;" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES" - {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} - {%- if machine not in ["GAEA", "NOAACLOUD"] %} - {{ mem_get_extrn_ics }} - {%- endif %} - {{ wtime_get_extrn_ics }} + +{%- if do_smoke_dust %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_SMOKE;" "&JOBSdir;/JREGIONAL_PROCESS_SMOKE" + + {{ nnodes_process_smoke }}:ppn={{ ppn_process_smoke }} + {{ wtime_process_smoke }} &NCORES_PER_NODE; - {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; + {%- if machine not in ["GAEA", "NOAACLOUD"] %} + {{ mem_process_smoke }} {%- endif %} - &TN_GET_EXTRN_ICS; - &LOGDIR;/&TN_GET_EXTRN_ICS;_@Y@m@d@H&LOGEXT; + + &TAG;&TN_PROCESS_SMOKE;_#type# + &LOGDIR;/&TN_PROCESS_SMOKE;_#type#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; @@ -565,73 +791,79 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - ICS_OR_LBCSICS -{%- if do_real_time %} + + CYCLE_TYPE#type# + CYCLE_SUBTYPE + NWGES_BASEDIR&NWGES_BASEDIR; + NWGES_DIR&NWGES_BASEDIR; + - &COMINgfs;/gfs.@Y@m@d/@H/atmos + + @Y@m@d@H@M00 + -{%- endif %} + {%- endif %} -{%- if run_task_get_extrn_lbcs %} + +{%- if do_nonvar_cldanal or do_refl2tten or do_enkf_radar_ref or do_envar_radar_ref %} - +--> + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; - &RSRV_HPSS; - &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_LBCS;" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES" - {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} - {%- if machine not in ["GAEA", "NOAACLOUD"] %} - {{ mem_get_extrn_lbcs }} - {%- endif %} - {{ wtime_get_extrn_lbcs }} + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_PROCESS_RADARREF" + + {{ nnodes_process_radarref }}:ppn={{ ppn_process_radarref }} + {{ wtime_process_radarref }} &NCORES_PER_NODE; - {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; - {%- endif %} - &TN_GET_EXTRN_LBCS; - &LOGDIR;/&TN_GET_EXTRN_LBCS;_@Y@m@d@H&LOGEXT; + &TAG;&TN_PROCESS_RADAR_REF;_#type# + &LOGDIR;/&TN_PROCESS_RADAR_REF;_#type#_@Y@m@d@H&LOGEXT; + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; PDY@Y@m@d cyc@H subcyc@M LOGDIR&LOGDIR; - ICS_OR_LBCSLBCS -{%- if do_real_time %} + CYCLE_TYPE#type# + CYCLE_SUBTYPE + RADAR_REF_THINNING{{ radar_ref_thinning }} + NWGES_BASEDIR&NWGES_BASEDIR; + - &COMINgfs;/gfs.@Y@m@d/@H/atmos + + @Y@m@d@H@M00 + -{%- endif %} -{%- endif %} -{%- if do_ensemble %} - - - {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} +{%- if do_nldn_lght %} + -{%- endif %} + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; -{%- if run_task_make_ics %} - + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_PROCESS_LIGHTNING" - &RSRV_DEFAULT; - &LOAD_MODULES_RUN_TASK_FP; "&TN_MAKE_ICS;" "&JOBSdir;/JREGIONAL_MAKE_ICS" - {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} - {{ wtime_make_ics }} + {{ nnodes_process_lightning }}:ppn={{ ppn_process_lightning }} + {{ wtime_process_lightning }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_MAKE_ICS;{{ uscore_ensmem_name }} - &LOGDIR;/&TN_MAKE_ICS;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + &TAG;&TN_PROCESS_LIGHTNING;_#type# + &LOGDIR;/&TN_PROCESS_LIGHTNING;_#type#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; @@ -639,47 +871,38 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} - ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPE#type# + CYCLE_SUBTYPE - - - - &EXPTDIR;/grid/&TN_MAKE_GRID;&CMPEXT; - &RUN_TASK_MAKE_GRID;FALSE - - - - &EXPTDIR;/orog/&TN_MAKE_OROG;&CMPEXT; - &RUN_TASK_MAKE_OROG;FALSE - - - - &EXPTDIR;/sfc_climo/&TN_MAKE_SFC_CLIMO;&CMPEXT; - &RUN_TASK_MAKE_SFC_CLIMO;FALSE - + @Y@m@d@H@M00 {%- endif %} -{%- if run_task_make_lbcs %} - + &RSRV_DEFAULT; - &LOAD_MODULES_RUN_TASK_FP; "&TN_MAKE_LBCS;" "&JOBSdir;/JREGIONAL_MAKE_LBCS" - {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} - {{ wtime_make_lbcs }} + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_PROCESS_BUFR" + + {{ nnodes_process_bufr }}:ppn={{ ppn_process_bufr }} + {{ wtime_process_bufr }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_MAKE_LBCS;{{ uscore_ensmem_name }} - &LOGDIR;/&TN_MAKE_LBCS;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + {%- if machine not in ["GAEA", "NOAACLOUD"] %} + {{ mem_process_bufr }} + {%- endif %} + &TAG;&TN_PROCESS_BUFR;_#type# + &LOGDIR;/&TN_PROCESS_BUFR;_#type#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; @@ -687,38 +910,58 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} - ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPE#type# + CYCLE_SUBTYPE - - - - &EXPTDIR;/grid/&TN_MAKE_GRID;&CMPEXT; - &RUN_TASK_MAKE_GRID;FALSE - - - - &EXPTDIR;/orog/&TN_MAKE_OROG;&CMPEXT; - &RUN_TASK_MAKE_OROG;FALSE - - - - &EXPTDIR;/sfc_climo/&TN_MAKE_SFC_CLIMO;&CMPEXT; - &RUN_TASK_MAKE_SFC_CLIMO;FALSE - + @Y@m@d@H@M00 + +{%- endif %} + + + +{%- endif %} +{%- endif %} + +{% if not is_rtma %} + +{%- if do_ensemble %} + + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} + {%- if do_enscontrol %} + /gec00{% for m in range(2, num_ens_members+1) %}{{ " /gep%02d"%m }}{% endfor %} + _mem0000{% for m in range(2, num_ens_members+1) %}{{ " _mem%04d"%m }}{% endfor %} + {%- else %} + {% for m in range(1, num_ens_members+1) %}{{ " /gep%02d"%m }}{% endfor %} + {% for m in range(1, num_ens_members+1) %}{{ " _mem%04d"%m }}{% endfor %} + {% for m in range(1, num_ens_members+1) %}{{ " mem%03d"%m }}{% endfor %} + {%- endif %} + {% for m in range(1, num_ens_members+1) %}{{ " /mem%03d"%m }}{% endfor %} {%- endif %} + + {%- if run_task_aqm_ics %} - {%- if not coldstart %} + {%- if not coldstart %} &RSRV_DEFAULT; @@ -726,7 +969,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_aqm_ics }}:ppn={{ ppn_aqm_ics }} {{ wtime_aqm_ics }} &NCORES_PER_NODE; - &TN_AQM_EXTRN_ICS; + &TAG;&TN_AQM_EXTRN_ICS; &LOGDIR;/&TN_AQM_EXTRN_ICS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -761,7 +1004,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_aqm_ics }}:ppn={{ ppn_aqm_ics }} {{ wtime_aqm_ics }} &NCORES_PER_NODE; - &TN_AQM_ICS; + &TAG;&TN_AQM_ICS; &LOGDIR;/&TN_AQM_ICS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -784,20 +1027,24 @@ MODULES_RUN_TASK_FP script. + {%- endif %} -{%- if run_task_aqm_lbcs %} - +{%- if run_task_aqm_lbcs %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_AQM_LBCS;" "&JOBSdir;/JREGIONAL_AQM_LBCS" {{ nnodes_aqm_lbcs }}:ppn={{ ppn_aqm_lbcs }} {{ wtime_aqm_lbcs }} &NCORES_PER_NODE; - &TN_AQM_LBCS; + &TAG;&TN_AQM_LBCS; &LOGDIR;/&TN_AQM_LBCS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -814,30 +1061,30 @@ MODULES_RUN_TASK_FP script. {%- endif %} -{%- if run_task_run_fcst %} - +{%- if run_task_get_extrn_ics %} - &RSRV_FCST; - &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_FCST;" "&JOBSdir;/JREGIONAL_RUN_FCST" - {%- if machine in ["JET", "HERA", "LINUX"] %} - {{ ncores_run_fcst }} - {{ native_run_fcst }} - {%- elif machine in ["WCOSS2"] %} - {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }} - &SCHED_NATIVE_CMD; - &NCORES_PER_NODE; - {%- else %} - {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} - &NCORES_PER_NODE; + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_ICS;" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES" + + {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} + {%- if machine not in ["GAEA", "NOAACLOUD"] %} + {{ mem_get_extrn_ics }} {%- endif %} + {{ wtime_get_extrn_ics }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; - {{ wtime_run_fcst }} - &TN_RUN_FCST;{{ uscore_ensmem_name }} - &LOGDIR;/&TN_RUN_FCST;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + {%- endif %} + + &TAG;&TN_GET_EXTRN_ICS;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_GET_EXTRN_ICS;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; @@ -848,45 +1095,2684 @@ MODULES_RUN_TASK_FP script. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# + ICS_OR_LBCSICS + GEFS_INPUT_SUBDIR#subdirGE# + WRF_MEM_NAME#memNameWRF# + GDASENKF_INPUT_SUBDIR#subdirGDAS# + GDAS_MEM_NAME#memNameGDAS# + + {%- if not do_retro %} + {%- if machine in ["WCOSS2"] %} + {%- if fv3gfs_file_fmt_ics in ["netcdf"] %} + {{ extrn_mdl_sysbasedir_ics }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.atmf{{ "%03d" % extrn_mdl_ics_offset_hrs }}.nc + {% else %} + {%- if extrn_mdl_name_ics in ["GEFS"] %} + {{ extrn_mdl_sysbasedir_ics }}/gefs.@Y@m@d/@H/atmos/pgrb2bp5/#subdirGE#.t@Hz.pgrb2b.0p50.f{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {% else %} + {{ extrn_mdl_sysbasedir_ics }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {%- endif %} + {%- endif %} + {%- elif machine in ["ORION"] %} + {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {% else %} + {%- if extrn_mdl_name_ics in ["GEFS"] %} + {{ extrn_mdl_sysbasedir_ics }}/#subdirGE#/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {%- elif extrn_mdl_name_ics in ["GDASENKF"] %} - - -{%- if run_task_nexus_emission %} - + {%- if machine in ["HERA"] %} + {{ extrn_mdl_sysbasedir_ics }}/enkfgdas.@Y@m@d/@H/atmos/#subdirGDAS#/gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.nc + {{ extrn_mdl_sysbasedir_ics }}/enkfgdas.@Y@m@d/@H/atmos/#subdirGDAS#/gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.nc + {%- elif machine in ["JET"] %} + {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc + {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc + {%- endif %} + + {%- elif extrn_mdl_name_ics in ["HRRRDAS"] %} + {{ extrn_mdl_sysbasedir_ics }}/@Y@m@d@H/postprd#memNameWRF#/wrfnat#memNameWRF#_00.grib2 + {%- else %} + {%- if machine in ["JET"] %} + {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {%- elif machine in ["HERA"] %} + {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} + {%- endif %} + {%- endif %} + {%- endif %} + + {%- endif %} + +{%- if do_real_time %} + + &COMINgfs;/gfs.@Y@m@d/@H/atmos + {%- endif %} -{%- if run_task_fire_emission %} - + + {%- endif %} -{%- if run_task_point_source %} - + +{%- if run_task_get_extrn_lbcs %} + + + + &WALL_LIMIT_PRE; + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_LBCS;" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES" + + {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} + {%- if machine not in ["GAEA", "NOAACLOUD"] %} + {{ mem_get_extrn_lbcs }} + {%- endif %} + {{ wtime_get_extrn_lbcs }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_GET_EXTRN_LBCS;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_GET_EXTRN_LBCS;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + EXTRN_MDL_NAME{{ extrn_mdl_name_lbcs }} + ICS_OR_LBCSLBCS + BOUNDARY_LEN{{ boundary_len_hrs }} + GEFS_INPUT_SUBDIR#subdirGE# + GDASENKF_INPUT_SUBDIR#subdirGDAS# + GDAS_MEM_NAME#memNameGDAS# + + {%- if not do_retro %} + + + {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_len_hrs+extrn_mdl_lbcs_offset_hrs+1, lbc_spec_intvl_hrs) %} + {%- if machine in ["WCOSS2"] %} + {%- if fv3gfs_file_fmt_lbcs in ["netcdf"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.atmf{{ "%03d" % h }}.nc + {%- else %} + {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} + {%- endif %} + {%- elif machine in ["ORION"] %} + {%- if fv3gfs_file_fmt_lbcs in ["netcdf"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.atmf{{ "%03d" % h }}.nc + {%- else %} + {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} + {%- endif %} + {%- elif machine in ["ORION"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} + {%- else %} + {%- if extrn_mdl_name_lbcs in ["GEFS"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/#subdirGE#/@y@j@H000{{ "%03d" % h }} + {%- elif extrn_mdl_name_lbcs in ["GDASENKF"] %} + {%- if machine in ["HERA"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/enkfgdas.@Y@m@d/@H/atmos/#subdirGDAS#/gdas.t@Hz.atmf{{ "%03d" % h }}.nc + {{ extrn_mdl_sysbasedir_lbcs }}/enkfgdas.@Y@m@d/@H/atmos/#subdirGDAS#/gdas.t@Hz.sfcf{{ "%03d" % h }}.nc + {%- elif machine in ["JET"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc + {%- endif %} + {%- else %} + {%- if machine in ["JET"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} + {%- elif machine in ["HERA"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} + {%- endif %} + {%- endif %} + {%- endif %} + {%- endfor %} + + + {%- endif %} + +{%- if do_real_time %} + + &COMINgfs;/gfs.@Y@m@d/@H/atmos + +{%- endif %} + + + +{%- endif %} + +{%- if do_rrfs_dev %} +{%- if run_task_get_extrn_lbcs %} + + + + &WALL_LIMIT_PRE; + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_LBCS;" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES" + + {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} + {%- if machine not in ["GAEA", "NOAACLOUD"] %} + {{ mem_get_extrn_lbcs }} + {%- endif %} + {{ wtime_get_extrn_lbcs }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_GET_EXTRN_LBCS_LONG;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_GET_EXTRN_LBCS_LONG;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + EXTRN_MDL_NAME{{ extrn_mdl_name_lbcs }} + ICS_OR_LBCSLBCS + BOUNDARY_LEN{{ boundary_len_hrs }} + GEFS_INPUT_SUBDIR#subdirGE# + GDASENKF_INPUT_SUBDIR#subdirGDAS# + GDAS_MEM_NAME#memNameGDAS# + + {%- if not do_retro %} + + + {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, lbc_spec_intvl_hrs) %} + {%- if machine in ["WCOSS2"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} + {%- elif machine in ["ORION"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf{{ "%03d" % h }}.nc + {%- else %} + {%- if extrn_mdl_name_lbcs in ["GEFS"] %} + {{ extrn_mdl_sysbasedir_lbcs }}/#subdirGE#/@y@j@H000{{ "%03d" % h }} + {%- else %} + {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} + {%- endif %} + {%- endif %} + {%- endfor %} + + + {%- endif %} + + + +{%- endif %} +{%- endif %} + +{%- if run_task_make_ics %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + &LOAD_MODULES_RUN_TASK_FP; "&TN_MAKE_ICS;" "&JOBSdir;/JREGIONAL_MAKE_ICS" + + {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} + {{ wtime_make_ics }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + &TAG;&TN_MAKE_ICS;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_MAKE_ICS;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + + + + + + + &EXPTDIR;/grid/&TN_MAKE_GRID;&CMPEXT; + &RUN_TASK_MAKE_GRID;FALSE + + + + &EXPTDIR;/orog/&TN_MAKE_OROG;&CMPEXT; + &RUN_TASK_MAKE_OROG;FALSE + + + + &EXPTDIR;/sfc_climo/&TN_MAKE_SFC_CLIMO;&CMPEXT; + &RUN_TASK_MAKE_SFC_CLIMO;FALSE + + + + + +{%- endif %} + +{%- if run_task_make_lbcs %} + + {% for h in range(0, boundary_proc_group_num) %}{{ " %02d" % h }}{% endfor %} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + &LOAD_MODULES_RUN_TASK_FP; "&TN_MAKE_LBCS;" "&JOBSdir;/JREGIONAL_MAKE_LBCS" + + {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} + {{ wtime_make_lbcs }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_MAKE_LBCS;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_MAKE_LBCS;{{ uscore_ensmem_name }}_@Y@m@d@H_#bcgrp#&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + bcgrp#bcgrp# + bcgrpnum{{ boundary_proc_group_num }} + + + + + + + + + + &EXPTDIR;/grid/&TN_MAKE_GRID;&CMPEXT; + &RUN_TASK_MAKE_GRID;FALSE + + + + &EXPTDIR;/orog/&TN_MAKE_OROG;&CMPEXT; + &RUN_TASK_MAKE_OROG;FALSE + + + + &EXPTDIR;/sfc_climo/&TN_MAKE_SFC_CLIMO;&CMPEXT; + &RUN_TASK_MAKE_SFC_CLIMO;FALSE + + + + + + +{%- endif %} + + + +{% if do_ensemble %} + +{% endif %} + +{%- endif %} + +{%- if not do_ensfcst %} + + +{%- if do_spinup %} + +{%- if do_ensemble %} + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} +{%- endif %} + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PREPSTART;" "&JOBSdir;/JREGIONAL_RUN_PREPSTART" + + {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} + {{ wtime_run_prepstart }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_PREP_CYC_SPINUP;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_PREP_CYC_SPINUP;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + FG_ROOT&FG_ROOT; + LBCS_ROOT&FG_ROOT; + CYCLE_TYPEspinup + {%- if do_ensinit %} + CYCLE_SUBTYPEensinit + {%- else %} + CYCLE_SUBTYPE + {%- endif %} + NWGES_BASEDIR&NWGES_BASEDIR; + + + + + + + {%- for h in cycl_hrs_spinstart %} + {{ h }}@H + {%- endfor %} + + + + {%- for h in range(0,lbcs_search_hrs) %} + + {%- endfor %} + +{%- if do_retro %} + {%- if not do_ensemble %} + &FG_ROOT;/@Y@m@d@H/{{ slash_ensmem_subdir }}/fcst_fv3lam/INPUT/gfs_ctrl.nc + {%- endif %} +{%- else %} + @Y@m@d@H@M00 +{%- endif %} + + + + {%- for h in cycl_hrs_spinstart %} + {{ h }}@H + {%- endfor %} +{%- if do_retro %} + &FG_ROOT;/@Y@m@d@H/{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res +{%- else %} + @Y@m@d@H@M00 + &FG_ROOT;/@Y@m@d@H/{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res +{%- endif %} + + + + + + +{% if do_ensinit -%} + + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_FCST;" "&JOBSdir;/JREGIONAL_RUN_FCST" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_fcst }} + + &TAG;&TN_RUN_FCST;_ensinit{{ uscore_ensmem_name }} + &LOGDIR;/&TN_RUN_FCST;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEspinup + CYCLE_SUBTYPEensinit + NWGES_BASEDIR&NWGES_BASEDIR; + RESTART_HRS0 + + + + + {%- for h in cycl_hrs_spinstart %} + {{ h }}@H + {%- endfor %} + + + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + &LOAD_MODULES_RUN_TASK_FP; "&TN_SAVE_RESTART;" "&JOBSdir;/JREGIONAL_SAVE_RESTART" + + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_SAVE_RESTART;_ensinit{{ uscore_ensmem_name }} + &LOGDIR;/&TN_SAVE_RESTART;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + fhr0 + CYCLE_TYPEspinup + CYCLE_SUBTYPEensinit + + + + @Y@m@d@H@M00 + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_ensinit/RESTART/coupler.res + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PREPSTART;" "&JOBSdir;/JREGIONAL_RUN_PREPSTART" + + {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} + {{ wtime_run_prepstart }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_PREP_CYC_SPINUP;_ensinit{{ uscore_ensmem_name }} + &LOGDIR;/&TN_PREP_CYC_SPINUP;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + FG_ROOT&FG_ROOT; + LBCS_ROOT&FG_ROOT; + CYCLE_TYPEspinup + CYCLE_SUBTYPEspinup + NWGES_BASEDIR&NWGES_BASEDIR; + + + + + + + + + &RSRV_ENKF; + &WALL_LIMIT_RECENTER; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_RECENTER;" &JOBSdir;/JREGIONAL_RUN_RECENTER + + {{ nnodes_run_recenter }}:ppn={{ ppn_run_recenter }} + {{ wtime_run_recenter }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_RECENTER;_spinup + &LOGDIR;/&TN_RUN_RECENTER;_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + nens{{ num_ens_members }} + ENSCTRL_COMIN_DIR&ENSCTRL_COMIN_BASEDIR;/@Y@m@d@H + ENSCTRL_COMIN_ROOT&ENSCTRL_COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + CYCLE_TYPEspinup + CYCLE_SUBTYPE + + + + {%- for m in range(1, num_ens_members+1) %} + + {%- endfor %} + &ENSCTRL_COMIN_BASEDIR;/@Y@m@d@H/nonvar_cldanl_spinup/nonvarcldanl_complete.txt + + + + +{%- endif %} + +{%- if not do_ensemble %} +{%- if do_dacycle %} + + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_ANAL_GSI;_spinup{{ uscore_ensmem_name }} + &LOGDIR;/&TN_ANAL_GSI;{{ uscore_ensmem_name }}_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEspinup + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens{{ num_ens_members }} + + + + @Y@m@d@H@M00 + + {%- if use_rrfse_ens %} + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + + {%- for h in cycl_hrs_prodstart_ens %} + {{ h }}@H + {%- endfor %} + + {% for h in range(1, num_ens_members+1) %} + &RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + + + {%- for h in cycl_hrs_prodstart_ens %} + {{ h }}@H + {%- endfor %} + + {% for h in range(1, num_ens_members+1) %} + &RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + {%- endif %} + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POSTANAL" + + {{ nnodes_run_postanal }}:ppn={{ ppn_run_postanal }} + {{ wtime_run_postanal }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_POSTANAL;_spinup{{ uscore_ensmem_name }} + &LOGDIR;/&TN_POSTANAL;{{ uscore_ensmem_name }}_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEspinup + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens{{ num_ens_members }} + + + + + + + + +{%- if do_envar_radar_ref %} + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_HYBRID_RADAR_REF;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_HYBRID_RADAR_REF;_spinup + &LOGDIR;/&TN_HYBRID_RADAR_REF;_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEspinup + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens30 + OB_TYPEradardbz + + + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + + +{% endif -%} + + +{% endif -%} + + +{%- if do_refl2tten %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_REF2TTEN" + + {{ nnodes_run_ref2tten }}:ppn={{ ppn_run_ref2tten }} + {{ wtime_run_ref2tten }} + {{ mem_run_ref2tten }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RADAR_REFL2TTEN;_spinup + &LOGDIR;/&TN_RADAR_REFL2TTEN;_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEspinup + CYCLE_SUBTYPE + MEM_TYPEMEMBER + + + + + + + + + + + + +{%- endif %} + +{%- if do_jedi_envar_ioda %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "jedienvar_ioda" "&JOBSdir;/JREGIONAL_JEDIENVAR_IODA" + + {{ nnodes_run_jedi_envar_ioda }}:ppn={{ ppn_run_jedi_envar_ioda }} + {{ wtime_run_jedi_envar_ioda }} + {{ mem_run_jedi_envar_ioda }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_JEDI_ENVAR_IODA;_spinup + &LOGDIR;/&TN_JEDI_ENVAR_IODA;_spinup_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEspinup + CYCLE_SUBTYPE + MEM_TYPEMEMBER + + + + + + + + +{%- endif %} + + +{%- if do_nonvar_cldanal %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_NONVARCLDANL" + + {{ nnodes_run_nonvarcldanl }}:ppn={{ ppn_run_nonvarcldanl }} + {{ wtime_run_nonvarcldanl }} + {{ mem_run_nonvarcldanl }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_CLDANL_NONVAR;_spinup{{ uscore_ensmem_name }} + &LOGDIR;/&TN_CLDANL_NONVAR;_spinup{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEspinup + CYCLE_SUBTYPE + MEM_TYPEMEMBER + + + + + {%- if do_enkfupdate %} + + {%- if do_enkf_radar_ref %} + + {%- else %} + + {%- endif %} + + {%- elif do_envar_radar_ref %} + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + {%- else %} + + {%- endif %} + + + + +{%- endif %} + +{%- endif %} + + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_FCST;" "&JOBSdir;/JREGIONAL_RUN_FCST" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_fcst }} + + &TAG;&TN_RUN_FCST;_spinup{{ uscore_ensmem_name }} + &LOGDIR;/&TN_RUN_FCST;_spinup{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEspinup + CYCLE_SUBTYPE + NWGES_BASEDIR&NWGES_BASEDIR; + RESTART_HRS1 + + + {%- if do_dacycle and do_refl2tten and do_nonvar_cldanal%} + + + + + {%- elif do_dacycle and do_nonvar_cldanal%} + + {%- elif do_dacycle and do_refl2tten%} + + {%- elif do_dacycle %} + {%- if do_envar_radar_ref %} + + {%- else %} + + {%- endif %} + {%- elif do_enkfupdate or do_enkf_radar_ref %} + + + {%- for h in cycl_hrs_spinstart %} + {{ h }}@H + {%- endfor %} + + + {%- if do_ensinit %} + + {%- else %} + + {%- endif %} + + + {%- else %} + + {%- endif %} + + + + + + + {% for h in range(da_cycle_interv, fcst_len_hrs_spinup+da_cycle_interv, da_cycle_interv) %}{{ " %03d" % h }}{% endfor %} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_SAVE_RESTART;" "&JOBSdir;/JREGIONAL_SAVE_RESTART" + + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_SAVE_RESTART;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_SAVE_RESTART;_spinup{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + fhr#fhr# + CYCLE_TYPEspinup + CYCLE_SUBTYPE + + + + @Y@m@d@H@M00 + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/coupler.res + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res + + + + + + + + + +{% if do_post_spinup %} + + + {% for h in range(0, fcst_len_hrs_spinup+1) %}{{ " %03d" % h }}{% endfor %} + + + + &RSRV_POST; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" + + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_POST;_spinup{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_POST;_spinup{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + fhr#fhr# + CYCLE_TYPEspinup + CYCLE_SUBTYPE + TMMARKtm00 + + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/logf#fhr# + + + + + + + &RSRV_PRDGEN; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PRDGEN;" "&JOBSdir;/JREGIONAL_RUN_PRDGEN" + + {{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }} + {{ wtime_run_prdgen }} + {{ mem_run_prdgen }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_PRDGEN;_spinup{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_PRDGEN;_spinup{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + fhr#fhr# + CYCLE_TYPEspinup + CYCLE_SUBTYPE + TMMARKtm00 + + + + + + + + +{%- endif %} + + +{%- if do_ensemble %} + +{%- endif %} + + +{%- endif %} + + +{%- endif %} + + + + +{%- if do_rrfs_dev %} + +{%- if do_ensemble %} + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} +{%- endif %} + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PREPSTART;" "&JOBSdir;/JREGIONAL_RUN_PREPSTART" + + {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} + {{ wtime_run_prepstart }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_PREP_CYC_PROD;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_PREP_CYC_PROD;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + FG_ROOT&FG_ROOT; + LBCS_ROOT&FG_ROOT; + CYCLE_TYPEprod + CYCLE_SUBTYPE + NWGES_BASEDIR&NWGES_BASEDIR; + + + {%- if do_ensfcst %} + + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/DA_OUTPUT/coupler.res + + + {%- else %} + + + + + {%- for h in cycl_hrs_prodstart %} + {{ h }}@H + {%- endfor %} + +{%- if do_spinup %} +{%- if do_retro %} + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res +{% else %} + + + @Y@m@d@H@M00 + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res + + + @Y@m@d@H@M00 + + {%- for h in range(da_cycle_interv, 6+1, da_cycle_interv) %} + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + + +{%- endif %} +{% else %} +{%- if do_retro %} + + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/ics/gfs_data.tile7.halo0.nc + + {%- for h in range(0, extrn_mdl_ics_offset_hrs+1) %} + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/lbcs/gfs_bndy.tile7.{{ "%03d" % boundary_len_hrs }}.nc + {%- endfor %} + + +{% else %} + + @Y@m@d@H@M00 + + + + + + +{%- endif %} +{%- endif %} + + + + {%- for h in cycl_hrs_prodstart %} + {{ h }}@H + {%- endfor %} +{%- if do_retro %} + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res +{% else %} + + + @Y@m@d@H@M00 + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + + + @Y@m@d@H@M00 + + {%- for h in range(da_cycle_interv+da_cycle_interv, 6+1, da_cycle_interv) %} + &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + + +{%- endif %} + + +{%- endif %} + + + + + +{%- if do_ensemble %} + +{%- endif %} + +{%- endif %} + +{%- if not do_ensfcst %} +{%- if do_gsiobserver %} + + + + &RSRV_ENKF; + &WALL_LIMIT_RECENTER; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PREPSTART;" "&JOBSdir;/JREGIONAL_CALC_ENSMEAN" + + {{ nnodes_run_recenter }}:ppn={{ ppn_run_recenter }} + {{ wtime_run_recenter }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_CALC_ENSMEAN; + &LOGDIR;/&TN_CALC_ENSMEAN;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + nens{{ num_ens_members }} + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + + + + {%- for m in range(1, num_ens_members+1) %} + + {%- endfor %} + + + + + + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_OBSERVER_GSI_ENSMEAN; + &LOGDIR;/&TN_OBSERVER_GSI_ENSMEAN;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + GSI_TYPEOBSERVER + MEM_TYPEMEAN + SATBIAS_DIR&NWGES_BASEDIR;/satbias + + + + + {%- if do_enkf_radar_ref %} + + {%- endif %} + {% if machine in ["JET", "HERA"] -%} + &OBSPATH;/@Y@m@d@H.rap.t@Hz.prepbufr.tm00 + {% else -%} + &OBSPATH;/rap.@Y@m@d/rap.t@Hz.prepbufr.tm00 + {% endif -%} + + + + + + {%- if do_ensemble %} + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} + {%- endif %} + + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_OBSERVER_GSI;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_OBSERVER_GSI;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + GSI_TYPEOBSERVER + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + + + + + + + + + + +{%- if do_ensemble %} + +{%- endif %} + +{%- endif %} + + +{%- if do_enkfupdate %} + + &RSRV_ENKF; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ENKFUPDT;" &JOBSdir;/JREGIONAL_RUN_ENKF + + {{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }} + {{ wtime_run_enkf }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_ENKFUPDT; + &LOGDIR;/&TN_RUN_ENKFUPDT;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + nens{{ num_ens_members }} + CYCLE_ROOT&COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + OB_TYPEconv + + + + + + {%- endif %} -{%- if run_task_aqm_ics %} + + +{%- if do_enkf_radar_ref %} + + &RSRV_ENKF; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_ENKF_RADAR_REF;" &JOBSdir;/JREGIONAL_RUN_ENKF + + {{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }} + {{ wtime_run_enkf }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_ENKF_RADAR_REF; + &LOGDIR;/&TN_ENKF_RADAR_REF;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + nens{{ num_ens_members }} + CYCLE_ROOT&COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + OB_TYPEradardbz + + + + + {%- if do_retro %} + + {%- for m in range(1, num_ens_members+1) %} + &NWGES_BASEDIR;/@Y@m@d@H/mem{{ "%04d" % m }}/observer_gsi/diag_conv_rw_ges.@Y@m@d@H.nc4 + {%- endfor %} + + {%- endif %} + + + + +{%- endif %} + +{%- if do_recenter %} + + &RSRV_ENKF; + &WALL_LIMIT_RECENTER; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_RECENTER;" &JOBSdir;/JREGIONAL_RUN_RECENTER + + {{ nnodes_run_recenter }}:ppn={{ ppn_run_recenter }} + {{ wtime_run_recenter }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_RECENTER; + &LOGDIR;/&TN_RUN_RECENTER;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + nens{{ num_ens_members }} + CYCLE_ROOT&COMIN_BASEDIR; + ENSCTRL_CYCLE_DIR&ENSCTRL_COMIN_BASEDIR;/@Y@m@d@H + ENSCTRL_CYCLE_ROOT&ENSCTRL_COMIN_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + + + + {%- if do_nonvar_cldanal %} + + {%- for m in range(1, num_ens_members+1) %} + + {%- endfor %} + + {%- elif do_enkf_radar_ref %} + + {%- elif do_enkfupdate %} + + {%- endif %} + + &ENSCTRL_COMIN_BASEDIR;/@Y@m@d@H/nonvar_cldanl/nonvarcldanl_complete.txt + &ENSCTRL_COMIN_BASEDIR;/@Y@m@d@H/nonvar_cldanl_spinup/nonvarcldanl_complete.txt + + + + + +{%- endif %} + +{% if do_rrfs_dev %} + +{% if do_ensemble %} + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} +{%- endif %} + +{%- if do_dacycle %} + + + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_ANAL_GSI;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&TN_ANAL_GSI;{{ uscore_ensmem_name }}_prod_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEprod + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens{{ num_ens_members }} + + + + @Y@m@d@H@M00 + + {%- if use_rrfse_ens %} + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + {%- if do_spinup %} + + + {%- for h in cycl_hrs_prodstart_ens %} + {{ h }}@H + {%- endfor %} + + {% for h in range(1, num_ens_members+1) %} + &RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + + + {%- for h in cycl_hrs_prodstart_ens %} + {{ h }}@H + {%- endfor %} + + {% for h in range(1, num_ens_members+1) %} + &RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + {%- else %} + + {% for h in range(1, num_ens_members+1) %} + &RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + {%- endfor %} + + {%- endif %} + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + {%- endif %} + + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POSTANAL" + + {{ nnodes_run_postanal }}:ppn={{ ppn_run_postanal }} + {{ wtime_run_postanal }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_POSTANAL;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&TN_POSTANAL;{{ uscore_ensmem_name }}_prod_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEprod + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens{{ num_ens_members }} + + + + + + + + +{%- if do_envar_radar_ref %} + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_HYBRID_RADAR_REF;" "&JOBSdir;/JREGIONAL_RUN_ANAL" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_anal }} + &SCHED_NATIVE_CMD; &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}:tpp={{ omp_num_threads_run_anal }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_anal }} + + &TAG;&TN_HYBRID_RADAR_REF;_prod + &LOGDIR;/&TN_HYBRID_RADAR_REF;_prod_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_ROOT&COMIN_BASEDIR; + RRFSE_FG_ROOT&RRFSE_FG_ROOT; + CYCLE_TYPEprod + CYCLE_SUBTYPE + GSI_TYPEANALYSIS + MEM_TYPEMEMBER + SATBIAS_DIR&NWGES_BASEDIR;/satbias + nens30 + OB_TYPEradardbz + + + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + + +{% endif -%} + + + +{% endif -%} + + +{%- if do_refl2tten %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_REFL2TTEN" + + {{ nnodes_run_ref2tten }}:ppn={{ ppn_run_ref2tten }} + {{ wtime_run_ref2tten }} + {{ mem_run_ref2tten }} + &NCORES_PER_NODE; + + &TAG;&TN_RADAR_REFL2TTEN;_prod + &LOGDIR;/&TN_RADAR_REFL2TTEN;_prod_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + MEM_TYPEMEMBER + + + + + + + + + + + + + +{%- endif %} + + +{%- if do_jedi_envar_ioda %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "jedienvar_ioda" "&JOBSdir;/JREGIONAL_JEDIENVAR_IODA" + + {{ nnodes_run_jedi_envar_ioda }}:ppn={{ ppn_run_jedi_envar_ioda }} + {{ wtime_run_jedi_envar_ioda }} + {{ mem_run_jedi_envar_ioda }} + &NCORES_PER_NODE; + + &TAG;&TN_JEDI_ENVAR_IODA;_prod + &LOGDIR;/&TN_JEDI_ENVAR_IODA;_prod_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + MEM_TYPEMEMBER + cyc@H + + + + + + + + + +{%- endif %} + + +{%- if do_nonvar_cldanal %} + + + &RSRV_DEFAULT; + &WALL_LIMIT_ANAL; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_ANAL;" "&JOBSdir;/JREGIONAL_RUN_NONVARCLDANL" + + {{ nnodes_run_nonvarcldanl }}:ppn={{ ppn_run_nonvarcldanl }} + {{ wtime_run_nonvarcldanl }} + {{ mem_run_nonvarcldanl }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_CLDANL_NONVAR;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&TN_CLDANL_NONVAR;_prod{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + MEM_TYPEMEMBER + + + + + {%- if do_enkfupdate %} + + {%- if do_enkf_radar_ref %} + + {%- else %} + + {%- endif %} + + {%- elif do_envar_radar_ref %} - -{%- if not coldstart %} - - + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + + + {%- for h in cycl_hrs_hyb_fv3lam_ens %} + {{ h }}@H + {%- endfor %} + + + {%- else %} + + {%- endif %} + + + + +{%- endif %} + + +{%- if do_ensemble %} + +{%- endif %} + +{%- endif %} + +{%- endif %} + + +{%- if do_save_input and do_ensemble %} + + + {%- for m in range(1, num_ens_members_fcst+1) -%}{%- set fmtstr=" %03d" -%}{{- fmtstr%m -}}{%- endfor %} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_SAVE_RESTART;" "&JOBSdir;/JREGIONAL_SAVE_INPUT" + + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + + &TAG;&TN_SAVE_INPUT;{{ uscore_ensmem_name }} + &LOGDIR;/&TN_SAVE_INPUT;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + CYCLE_TYPEprod + CYCLE_SUBTYPE + + + {%- if do_recenter %} + + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + &COMIN_BASEDIR;/@Y@m@d@H/recenter/fcst_fv3lam/INPUT/recenter_complete.txt + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + + + {%- elif do_nonvar_cldanal%} + + {%- elif do_enkf_radar_ref %} + + {%- elif do_enkfupdate %} + + {%- endif %} + + + + +{%- endif %} + +{% if do_ensemble %} + + {% for m in range(1, num_ens_members+1) %}{{ "%03d " % m }}{% endfor %} +{%- endif %} + + +{%- if run_task_run_fcst %} + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_FCST;" "&JOBSdir;/JREGIONAL_RUN_FCST" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_fcst }} + + &TAG;&TN_RUN_FCST;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&TN_RUN_FCST;_prod{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + NWGES_BASEDIR&NWGES_BASEDIR; + RESTART_HRS{{ restart_interval }} + + + {%- if do_ensfcst %} + + {%- elif do_dacycle and do_refl2tten and do_nonvar_cldanal%} + + + + + {%- elif do_dacycle and do_nonvar_cldanal%} + + {%- elif do_dacycle and do_refl2tten%} + + {%- elif do_dacycle %} + {%- if do_envar_radar_ref %} + + {%- else %} + + {%- endif %} + {%- elif do_enkfupdate or do_enkf_radar_ref %} + {%- if do_recenter %} + + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + &COMIN_BASEDIR;/@Y@m@d@H/recenter/fcst_fv3lam/INPUT/recenter_complete.txt + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + + + {%- elif do_nonvar_cldanal%} + + {%- elif do_enkf_radar_ref %} + + {%- else %} + + {%- endif %} + {%- else %} + {%- if do_rrfs_dev %} + + {%- else %} + + + + + + {%- if run_task_nexus_emission %} + + {%- endif %} + {%- if run_task_fire_emission %} + + {%- endif %} + {%- if run_task_point_source %} + + {%- endif %} + {%- if run_task_aqm_ics %} + + + {%- if not coldstart %} + + + + + {%- else %} + + {%- endif %} + + {%- endif %} + {%- if run_task_aqm_lbcs %} + + {%- endif %} + + + + {%- endif %} + + {%- endif %} + + + +{%- endif %} + +{%- if do_rrfs_dev %} + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_FCST;" "&JOBSdir;/JREGIONAL_RUN_FCST" + + {%- if machine in ["JET", "HERA", "LINUX"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {%- elif machine in ["WCOSS2"] %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }} + &SCHED_NATIVE_CMD; + &NCORES_PER_NODE; + {%- else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {%- endif %} + &SCHED_NATIVE_CMD; + {{ wtime_run_fcst_long }} + + &TAG;&TN_RUN_FCST;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&TN_RUN_FCST;_prod{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + NWGES_BASEDIR&NWGES_BASEDIR; + RESTART_HRS{{ restart_interval_long }} + + + {%- if do_dacycle and do_refl2tten and do_nonvar_cldanal%} + + + + + {%- elif do_dacycle and do_nonvar_cldanal%} + + {%- elif do_dacycle and do_refl2tten%} + + {%- elif do_dacycle %} + {%- if do_envar_radar_ref %} + + {%- else %} + + {%- endif %} + {%- elif do_enkfupdate or do_enkf_radar_ref %} + {%- if do_recenter %} + + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + &COMIN_BASEDIR;/@Y@m@d@H/recenter/fcst_fv3lam/INPUT/recenter_complete.txt + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + + + {%- elif do_nonvar_cldanal%} + + {%- elif do_enkf_radar_ref %} + + {%- else %} + + {%- endif %} + {%- else %} + + {%- endif %} + + +{%- endif %} + +{%- if not do_ensfcst %} +{%- if not is_rtma %} +{%- if do_rrfs_dev %} + + + {{ restart_interval }} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_SAVE_RESTART;" "&JOBSdir;/JREGIONAL_SAVE_RESTART" + + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + + &TAG;&TN_SAVE_RESTART;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_SAVE_RESTART;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + fhr#fhr# + CYCLE_TYPEprod + CYCLE_SUBTYPE + + + + @Y@m@d@H@M00 + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + + + + + + + + + + + {{ restart_interval_long }} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_SAVE_RESTART;" "&JOBSdir;/JREGIONAL_SAVE_RESTART" + + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + + &TAG;&TN_SAVE_RESTART;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_SAVE_RESTART;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + fhr#fhr# + CYCLE_TYPEprod + CYCLE_SUBTYPE + + + + @Y@m@d@H@M00 + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + + + + + + + +{%- endif %} +{%- endif %} +{%- endif %} + + +{%- if do_post_prod %} + + + + {% for h in range(0, postproc_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + +{% if nsout_min > 0 %} + {% for h in range(0, 60, nsout_min) %}{{ " #fullfhr#-%02d-00 " % h }}{% endfor %} +{%- else %} + #fullfhr# +{%- endif %} + + + + &RSRV_POST; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" + + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + CYCLE_TYPEprod + CYCLE_SUBTYPE + fhr#fhr# + TMMARKtm00 + + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/logf#fhr# + + + + + + + + + {% for h in range(0, postproc_long_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + +{% if nsout_min > 0 %} + {% for h in range(0, 60, nsout_min) %}{{ " #fullfhr#-%02d-00 " % h }}{% endfor %} {%- else %} - -{%- endif %} - -{%- endif %} -{%- if run_task_aqm_lbcs %} - + #fullfhr# {%- endif %} - - - -{%- endif %} + + + &RSRV_POST; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" + + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + CYCLE_TYPEprod + CYCLE_SUBTYPE + fhr#fhr# + TMMARKtm00 + + + &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/logf#fhr# + + + + + + + + + {% for h in range(0, postproc_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + + + + &RSRV_PRDGEN; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PRDGEN;" "&JOBSdir;/JREGIONAL_RUN_PRDGEN" + + {{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }} + {{ wtime_run_prdgen }} + {{ mem_run_prdgen }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_PRDGEN;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_PRDGEN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + fhr#fhr# + TMMARKtm00 + + + + + + + + + + {% for h in range(0, postproc_long_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + + + &RSRV_PRDGEN; + &WALL_LIMIT_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_PRDGEN;" "&JOBSdir;/JREGIONAL_RUN_PRDGEN" + + {{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }} + {{ wtime_run_prdgen }} + {{ mem_run_prdgen }} + &NCORES_PER_NODE; + &SCHED_NATIVE_CMD; + + &TAG;&TN_RUN_PRDGEN;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_PRDGEN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + CYCLE_TYPEprod + CYCLE_SUBTYPE + fhr#fhr# + TMMARKtm00 + + + + + + + +{%- else %} + {%- if run_task_run_post %} + {%- if fcst_len_hrs == -1 %} {%- for icyc in range(num_fcst_len_cycl) %} @@ -915,7 +3801,7 @@ MODULES_RUN_TASK_FP script. {%- elif icyc == 3 %} &TN_RUN_POST_CYC4;{{ uscore_ensmem_name }}_f#fhr# {%- endif %} - &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; PDY@Y@m@d @@ -955,20 +3841,30 @@ Note that we wrap this task in a metatask in order to be able to declare and use the variables fhr and fmn. This allows the block of code inside the tag (except for the dependencies) to be identical to the ones later below for other output times. +--> + - 000 00 - + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# + + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -982,6 +3878,9 @@ later below for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEprod + CYCLE_SUBTYPE + TMMARKtm00 + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; + + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; USHdir&USHdir; @@ -1039,6 +3952,10 @@ for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEprod + CYCLE_SUBTYPE + TMMARKtm00 + @@ -1050,12 +3967,20 @@ for other output times. + {%- endif %} + {%- if sub_hourly_post %} + {%- else %} {% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} - + {%- endif %} &RSRV_DEFAULT; @@ -1088,11 +4017,11 @@ always zero). &NCORES_PER_NODE; &SCHED_NATIVE_CMD; {%- if sub_hourly_post %} - &TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; {%- else %} - &TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr# - &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; {%- endif %} GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1110,6 +4039,10 @@ always zero). SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEprod + CYCLE_SUBTYPE + TMMARKtm00 + @@ -1126,6 +4059,10 @@ always zero). + {%- if sub_hourly_post %} @@ -1133,7 +4070,10 @@ always zero). {%- endif %} - {%- if sub_hourly_post %} + + {%- if sub_hourly_post %} {{ "%03d" % fcst_len_hrs }} 00 - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST" @@ -1158,7 +4099,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_run_post }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# + &TAG;&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1172,6 +4113,10 @@ the tag to be identical to the ones above for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEprod + CYCLE_SUBTYPE + TMMARKtm00 + @@ -1186,14 +4131,156 @@ the tag to be identical to the ones above for other output times. {%- endif %} + +{%- endif %} +{%- endif %} + +{%- endif %} + +{% if do_bufrsnd %} + + + &RSRV_POST; + &WALL_LIMIT_BUFRSND; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_BUFRSND;" "&JOBSdir;/JREGIONAL_BUFRSND" + + {{ nnodes_run_bufrsnd }}:ppn={{ ppn_run_bufrsnd }} + {{ wtime_run_bufrsnd }} + &NCORES_PER_NODE; + + &TAG;&TN_RUN_BUFRSND; + &LOGDIR;/&TN_RUN_BUFRSND;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + TMMARKtm00 + + + &COMIN_BASEDIR;/@Y@m@d@H/fcst_fv3lam/logf{{ "%03d" % postproc_long_len_hrs }} + + + +{%- endif %} + +{%- if do_rrfs_dev %} +{%- if machine in ["JET", "HERA"] %} +{%- if not do_ensemble %} + + + + {{ tile_labels }} + {{ tile_sets }} + + + &RSRV_GRAPHICS; + &WALL_LIMIT_GRAPHICS; + + {{ wtime_run_fcst_long }} + {{ nnodes_run_graphics }}:ppn={{ ppn_run_graphics }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;python{{ uscore_ensmem_name }}_@H_#tilelabel# + &LOGDIR;/python{{ uscore_ensmem_name }}_#tilelabel#&LOGEXT; + &LOAD_MODULES_RUN_TASK_FP; "run_graphics" "&JOBSdir;/JREGIONAL_RUN_GRAPHICS" + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + GRAPHICS_TYPEmaps + TILES#tileset# + TILELABEL#tilelabel# + ENSCTRL_COMOUT_BASEDIR&ENSCTRL_COMOUT_BASEDIR; + + + + + + + + + + + + + &RSRV_GRAPHICS; + &WALL_LIMIT_GRAPHICS; + + {{ wtime_run_fcst_long }} + {{ nnodes_run_graphics }}:ppn={{ ppn_run_graphics }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;python{{ uscore_ensmem_name }}_@H_skewts + &LOGDIR;/python_skewts{{ uscore_ensmem_name }}_@H&LOGEXT; + &LOAD_MODULES_RUN_TASK_FP; "run_graphics" "&JOBSdir;/JREGIONAL_RUN_GRAPHICS" + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + GRAPHICS_TYPEskewts + ENSCTRL_COMOUT_BASEDIR&ENSCTRL_COMOUT_BASEDIR; + + + + + + + + + +{%- endif %} {%- endif %} {%- endif %} -{%- if run_task_plot_allvars %} - +{%- if run_task_plot_allvars %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_PLOT_ALLVARS;" "&JOBSdir;/JREGIONAL_PLOT_ALLVARS" @@ -1203,7 +4290,8 @@ the tag to be identical to the ones above for other output times. {%- if machine not in ["WCOSS2"] %} &SCHED_NATIVE_CMD; {%- endif %} - &TN_PLOT_ALLVARS; + + &TAG;&TN_PLOT_ALLVARS; &LOGDIR;/&TN_PLOT_ALLVARS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1233,18 +4321,18 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_pre_post_stat %} - +{%- if run_task_pre_post_stat %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_PRE_POST_STAT;" "&JOBSdir;/JREGIONAL_PRE_POST_STAT" {{ nnodes_pre_post_stat }}:ppn={{ ppn_pre_post_stat }} {{ wtime_pre_post_stat }} &NCORES_PER_NODE; - &TN_PRE_POST_STAT; + &TAG;&TN_PRE_POST_STAT; &LOGDIR;/&TN_PRE_POST_STAT;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1268,12 +4356,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_post_stat_o3 %} - +{%- if run_task_post_stat_o3 %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_POST_STAT_O3;" "&JOBSdir;/JREGIONAL_POST_STAT_O3" {{ nnodes_post_stat_o3 }}:ppn={{ ppn_post_stat_o3 }} @@ -1282,7 +4370,7 @@ it, we'd have to include more jinja if-statements here. {%- endif %} {{ wtime_post_stat_o3 }} &NCORES_PER_NODE; - &TN_POST_STAT_O3; + &TAG;&TN_POST_STAT_O3; &LOGDIR;/&TN_POST_STAT_O3;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1299,12 +4387,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_post_stat_pm25 %} - +{%- if run_task_post_stat_pm25 %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_POST_STAT_PM25;" "&JOBSdir;/JREGIONAL_POST_STAT_PM25" {{ nnodes_post_stat_pm25 }}:ppn={{ ppn_post_stat_pm25 }} @@ -1313,7 +4401,7 @@ it, we'd have to include more jinja if-statements here. {%- endif %} {{ wtime_post_stat_pm25 }} &NCORES_PER_NODE; - &TN_POST_STAT_PM25; + &TAG;&TN_POST_STAT_PM25; &LOGDIR;/&TN_POST_STAT_PM25;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1330,12 +4418,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_bias_correction_o3 %} - +{%- if run_task_bias_correction_o3 %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_BIAS_CORRECTION_O3;" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_O3" {{ nnodes_bias_correction_o3 }}:ppn={{ ppn_bias_correction_o3 }} @@ -1344,7 +4432,7 @@ it, we'd have to include more jinja if-statements here. {%- endif %} {{ wtime_bias_correction_o3 }} &NCORES_PER_NODE; - &TN_BIAS_CORRECTION_O3; + &TAG;&TN_BIAS_CORRECTION_O3; &LOGDIR;/&TN_BIAS_CORRECTION_O3;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1361,12 +4449,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_bias_correction_pm25 %} - +{%- if run_task_bias_correction_pm25 %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_BIAS_CORRECTION_PM25;" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_PM25" {{ nnodes_bias_correction_pm25 }}:ppn={{ ppn_bias_correction_pm25 }} @@ -1375,7 +4463,7 @@ it, we'd have to include more jinja if-statements here. {%- endif %} {{ wtime_bias_correction_pm25 }} &NCORES_PER_NODE; - &TN_BIAS_CORRECTION_PM25; + &TAG;&TN_BIAS_CORRECTION_PM25; &LOGDIR;/&TN_BIAS_CORRECTION_PM25;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1392,12 +4480,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} -{%- if run_task_get_obs_ccpa %} - +{%- if run_task_get_obs_ccpa %} + &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA" @@ -1408,7 +4496,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_GET_OBS_CCPA; + &TAG;&TN_GET_OBS_CCPA; &LOGDIR;/&TN_GET_OBS_CCPA;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1423,13 +4511,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} - -{%- if run_task_get_obs_mrms %} - +{%- if run_task_get_obs_mrms %} + &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS" @@ -1440,7 +4527,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_GET_OBS_MRMS; + &TAG;&TN_GET_OBS_MRMS; &LOGDIR;/&TN_GET_OBS_MRMS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1456,13 +4543,12 @@ it, we'd have to include more jinja if-statements here. {%- endif %} - -{%- if run_task_get_obs_ndas %} - +{%- if run_task_get_obs_ndas %} + &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS" @@ -1473,7 +4559,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_GET_OBS_NDAS; + &TAG;&TN_GET_OBS_NDAS; &LOGDIR;/&TN_GET_OBS_NDAS;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1488,12 +4574,16 @@ it, we'd have to include more jinja if-statements here. {%- endif %} + {%- if run_task_vx_gridstat %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1504,7 +4594,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_apcp01h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_APCP01H;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_APCP01H;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_APCP01H;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1545,7 +4635,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1556,7 +4646,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_apcp03h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_APCP03H;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_APCP03H;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_APCP03H;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1587,7 +4677,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1598,7 +4688,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_apcp06h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_APCP06H;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_APCP06H;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_APCP06H;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1626,10 +4716,10 @@ it, we'd have to include more jinja if-statements here. {%- if fcst_len_hrs >= 24 %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1640,7 +4730,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_apcp24h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_APCP24H;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_APCP24H;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_APCP24H;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1670,7 +4760,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1681,7 +4771,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_refc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_REFC;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_REFC;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_REFC;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1720,7 +4810,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX" @@ -1731,7 +4821,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_retop }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_RETOP;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_GRIDSTAT_VX_RETOP;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_RETOP;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1772,7 +4862,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX" @@ -1783,7 +4873,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_sfc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_SFC;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_POINTSTAT_VX_SFC;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_SFC;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1821,7 +4911,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX" @@ -1832,7 +4922,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_upa }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_UPA;{{ uscore_ensmem_name }} + &TAG;&TN_RUN_MET_POINTSTAT_VX_UPA;{{ uscore_ensmem_name }} &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_UPA;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1877,7 +4967,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -1888,7 +4978,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_apcp01h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_APCP01H; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_APCP01H; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_APCP01H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1913,7 +5003,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -1924,7 +5014,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_apcp03h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_APCP03H; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_APCP03H; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_APCP03H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1950,7 +5040,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -1961,7 +5051,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_apcp06h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_APCP06H; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_APCP06H; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_APCP06H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -1987,7 +5077,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -1998,7 +5088,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_apcp24h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_APCP24H; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_APCP24H; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_APCP24H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2023,7 +5113,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -2034,7 +5124,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_refc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_REFC; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_REFC; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_REFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2057,7 +5147,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_GRID" @@ -2068,7 +5158,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_retop }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_RETOP; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_RETOP; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_RETOP;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2093,7 +5183,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_POINT" @@ -2104,7 +5194,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_sfc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_SFC; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_SFC; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_SFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2126,7 +5216,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_ENSEMBLESTAT_VX_POINT" @@ -2137,7 +5227,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_ensemblestat_vx_upa }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_ENSEMBLESTAT_VX_UPA; + &TAG;&TN_RUN_MET_ENSEMBLESTAT_VX_UPA; &LOGDIR;/&TN_RUN_MET_ENSEMBLESTAT_VX_UPA;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2162,7 +5252,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSMEAN" @@ -2173,7 +5263,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensmean_apcp01h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP01H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP01H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP01H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2198,7 +5288,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSMEAN" @@ -2209,7 +5299,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensmean_apcp03h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP03H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP03H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP03H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2235,7 +5325,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSMEAN" @@ -2246,7 +5336,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensmean_apcp06h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP06H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP06H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP06H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2272,7 +5362,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSMEAN" @@ -2283,7 +5373,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensmean_apcp24h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP24H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP24H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSMEAN_APCP24H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2310,7 +5400,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX_ENSMEAN" @@ -2321,7 +5411,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_ensmean_sfc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_ENSMEAN_SFC; + &TAG;&TN_RUN_MET_POINTSTAT_VX_ENSMEAN_SFC; &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_ENSMEAN_SFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2343,7 +5433,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX_ENSMEAN" @@ -2354,7 +5444,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_ensmean_upa }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_ENSMEAN_UPA; + &TAG;&TN_RUN_MET_POINTSTAT_VX_ENSMEAN_UPA; &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_ENSMEAN_UPA;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2379,7 +5469,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2390,7 +5480,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_apcp01h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP01H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP01H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP01H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2415,7 +5505,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2426,7 +5516,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_apcp03h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP03H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP03H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP03H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2452,7 +5542,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2463,7 +5553,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_apcp06h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP06H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP06H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP06H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2489,7 +5579,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2500,7 +5590,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_apcp24h }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP24H; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP24H; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_APCP24H;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2525,7 +5615,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2536,7 +5626,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_refc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_REFC; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_REFC; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_REFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2559,7 +5649,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_VX_ENSPROB" @@ -2570,7 +5660,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_gridstat_vx_ensprob_retop }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_GRIDSTAT_VX_ENSPROB_RETOP; + &TAG;&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_RETOP; &LOGDIR;/&TN_RUN_MET_GRIDSTAT_VX_ENSPROB_RETOP;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2595,7 +5685,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX_ENSPROB" @@ -2606,7 +5696,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_ensprob_sfc }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_ENSPROB_SFC; + &TAG;&TN_RUN_MET_POINTSTAT_VX_ENSPROB_SFC; &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_ENSPROB_SFC;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2628,7 +5718,7 @@ it, we'd have to include more jinja if-statements here. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&TN_VX;" "&JOBSdir;/JREGIONAL_RUN_MET_POINTSTAT_VX_ENSPROB" @@ -2639,7 +5729,7 @@ it, we'd have to include more jinja if-statements here. {{ wtime_run_met_pointstat_vx_ensprob_upa }} &NCORES_PER_NODE; &SCHED_NATIVE_CMD; - &TN_RUN_MET_POINTSTAT_VX_ENSPROB_UPA; + &TAG;&TN_RUN_MET_POINTSTAT_VX_ENSPROB_UPA; &LOGDIR;/&TN_RUN_MET_POINTSTAT_VX_ENSPROB_UPA;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -2658,5 +5748,332 @@ it, we'd have to include more jinja if-statements here. {%- endif %} + + +{%- if do_rrfs_dev -%} + + + + &RSRV_POST; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_CLEAN" + + 1:ppn=1 + 00:15:00 + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_RUN_CLEAN; + &LOGDIR;/&TN_RUN_CLEAN;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + NWGES_BASEDIR&NWGES_BASEDIR; +{%- if do_ensemble %} + nens{{ num_ens_members }} +{%- else %} + nens0 +{%- endif %} + + + + +{%- if machine in ["JET", "HERA"] %} + + +{%- if do_ensemble %} + + + &RSRV_HPSS; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_ICS;" "&JOBSdir;/JREGIONAL_RUN_ARCHIVE" + + 1:ppn=1 + 23:00:00 + 24G + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_RUN_ARCHIVE; + &LOGDIR;/&TN_RUN_ARCHIVE;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + nens{{ num_ens_members }} + + + +{%- else %} + + + + &RSRV_HPSS; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_ICS;" "&JOBSdir;/JREGIONAL_RUN_ARCHIVE" + + 1:ppn=1 + 08:00:00 + 24G + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_RUN_ARCHIVE; + &LOGDIR;/&TN_RUN_ARCHIVE;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + +{%- endif %} + +{%- endif %} + + +{%- if machine in ["WCOSS2"] %} + + + &RSRV_HPSS; + + &LOAD_MODULES_RUN_TASK_FP; "&TN_GET_EXTRN_ICS;" "&JOBSdir;/JREGIONAL_RUN_ARCHIVE" + + 1:ppn=1 + 08:00:00 + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;&TN_RUN_ARCHIVE; + &LOGDIR;/&TN_RUN_ARCHIVE;_@Y@m@d@H&LOGEXT; + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + + + + + 00@H + 12@H + + + + + 00@H + 12@H + + + + + + +{%- endif %} + +{% if do_ensemble and do_ens_graphics %} +{%- if machine in ["JET", "HERA"] %} + + + {{ tile_labels }} + {{ tile_sets }} + + + &RSRV_GRAPHICS; + &WALL_LIMIT_GRAPHICS; + + {{ wtime_run_fcst }} + {{ nnodes_run_graphics }}:ppn={{ ppn_run_graphics }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;python_@H_#tilelabel# + &LOGDIR;/python_#tilelabel#&LOGEXT; + &LOAD_MODULES_RUN_TASK_FP; "run_graphics" "&JOBSdir;/JREGIONAL_RUN_GRAPHICS" + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + GRAPHICS_TYPEenspanel + TILES#tileset# + TILELABEL#tilelabel# + ALL_LEADSFALSE + ENSCTRL_COMOUT_BASEDIR&ENSCTRL_COMOUT_BASEDIR; + + + + &ENSCTRL_COMOUT_DIR;/RRFS_CONUS.t@Hz.bgdawpf000.tm00.grib2 + {%- for m in range(1, 10) %} + + {%- endfor %} + + + + +{%- endif %} +{%- endif %} + + +{%- if do_ensemble and do_ensfcst and do_enspost %} + + + {% for h in range(0, postproc_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_POST; + + {{ wtime_run_enspost }} + {{ nnodes_run_enspost }}:ppn={{ ppn_run_enspost }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;enspost_@H_#fhr# + &LOGDIR;/enspost_f#fhr#_@Y@m@d@H&LOGEXT; + &LOAD_MODULES_RUN_TASK_FP; "run_enspost" "&JOBSdir;/JREGIONAL_RUN_ENSPOST" + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + fhr#fhr# + ENSCTRL_COMOUT_BASEDIR&ENSCTRL_COMOUT_BASEDIR; + + + + &ENSCTRL_COMOUT_DIR;/RRFS_CONUS.t@Hz.bgsfcf#fhr#.tm00.grib2 + {%- for m in range(1, num_ens_members_fcst+1) %} + + {%- endfor %} + + + + + + +{% if machine in ["JET", "HERA"] %} + + + &RSRV_GRAPHICS; + &WALL_LIMIT_GRAPHICS; + + {{ wtime_run_enspost }} + {{ nnodes_run_graphics }}:ppn={{ ppn_run_graphics }} + &NCORES_PER_NODE; + {%- if machine not in ["WCOSS2"] %} + &SCHED_NATIVE_CMD; + {%- endif %} + + &TAG;python_@H_enspost_maps + &LOGDIR;/python_enspost_maps&LOGEXT; + &LOAD_MODULES_RUN_TASK_FP; "run_graphics" "&JOBSdir;/JREGIONAL_RUN_GRAPHICS" + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + USHdir&USHdir; + PDY@Y@m@d + cyc@H + subcyc@M + LOGDIR&LOGDIR; + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + + GRAPHICS_TYPEmaps + TILESfull + TILELABELfull + ALL_LEADSFALSE + ENSCTRL_COMOUT_BASEDIR&ENSCTRL_COMOUT_BASEDIR; + ENSPRODensprod + IMAGES_FNrrfs_ensprod.yml + + + + + +{%- endif %} + +{%- endif %} +{%- endif %} diff --git a/ush/config.rrfs.yaml b/ush/config.rrfs.yaml new file mode 100644 index 0000000000..6b12ce2910 --- /dev/null +++ b/ush/config.rrfs.yaml @@ -0,0 +1,137 @@ +metadata: + description: >- + Sample config file for the work of importing RRFS_dev1 tasks + to SRW +user: + RUN_ENVIR: nco + MACHINE: hera + ACCOUNT: zrtrr + SERVICE_ACCOUNT: zrtrr +platform: + RESERVATION: "" + SST_update_hour: 1 + GVF_update_hour: 4 + SNOWICE_update_hour: 0 + ARCHIVEDIR: "/1year/BMC/wrfruc/rrfs_dev1" + NCARG_ROOT: "/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" + NCL_HOME: "/home/rtrr/RRFS/graphics" + NCL_REGION: "conus" + DOMAIN_PREGEN_BASEDIR: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/lam" + FIXgsi: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/gsi" + FIXcrtm: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/crtm/CRTM_v2.3.0" + FIXcrtmupp: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/crtm/CRTM_v2.4.0" +workflow: + TAG: RRFS_CONUS_3km_ + USE_CRON_TO_RELAUNCH: false + EXPT_SUBDIR: RRFS_CONUS_3km + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: RRFS_CONUS_3km + DATE_FIRST_CYCL: '2022072000' + DATE_LAST_CYCL: '2022072100' + FCST_LEN_HRS: 3 + FCST_LEN_HRS_SPINUP: 1 + POSTPROC_LEN_HRS: 3 + POSTPROC_LONG_LEN_HRS: 12 + PREEXISTING_DIR_METHOD: rename + DEBUG: true + VERBOSE: true + COMPILER: intel + CYCL_HRS: [0, 12] + CYCL_HRS_SPINSTART: [3, 15] + CYCL_HRS_PRODSTART: [9, 21] + FCST_LEN_HRS_CYCLES: [12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3] + DA_CYCLE_INTERV: 1 + RESTART_INTERVAL: 1 + BOUNDARY_LEN_HRS: 18 + BOUNDARY_PROC_GROUP_NUM: 10 +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false + RUN_TASK_PLOT_ALLVARS: false +rrfs: + DO_DACYCLE: true + DO_SURFACE_CYCLE: true + DO_SPINUP: true + DO_RETRO: true + DO_NONVAR_CLDANAL: true + DO_REFL2TTEN: true # turn off later + DO_NLDN_LGHT: true # turn off later + DO_SOIL_ADJUST: true + DO_RADDA: false + DO_BUFRSND: false + DO_RRFS_DEV: false + DO_POST_SPINUP: true + DO_POST_PROD: true +cycledefs: + INITIAL_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0300 ", workflow.DATE_LAST_CYCL[0:8], "2300 12:00:00"]|join }}' + BOUNDARY_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0000 ", workflow.DATE_LAST_CYCL[0:8], "2300 06:00:00"]|join }}' + PROD_CYCLEDEF: "00 10,11,13,14,16,17,19,20,22,23 20-21 7 2022 *" + PRODLONG_CYCLEDEF: "00 9-23/3 20-21 7 2022 *" + #ARCHIVE_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0700 ", workflow.DATE_LAST_CYCL[0:8], "2300 24:00:00"]|join }}' + SPINUP_CYCLEDEF: "00 03-08,15-20 20-21 7 2022 *" +#gsi: +# HYBENSMEM_NMIN: 66 +nco: + NET: rrfs + envir: para + model_ver: v1.0 + RUN: RRFS_conus_3km +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: FV3GFS + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{yy}{jjj}{hh}0000{fcst_hr:02d}' + EXTRN_MDL_ICS_OFFSET_HRS: 3 + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_ICS: "/scratch2/BMC/zrtrr/rli/data/gfs/0p25deg/grib2" +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: FV3GFS + FV3GFS_FILE_FMT_LBCS: grib2 + EXTRN_MDL_FILES_LBCS: + - '{yy}{jjj}{hh}0000{fcst_hr:02d}' + LBC_SPEC_INTVL_HRS: 1 + EXTRN_MDL_LBCS_OFFSET_HRS: 6 + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/scratch2/BMC/zrtrr/rli/data/gfs/0p25deg/grib2" +task_make_ics: + NNODES_MAKE_ICS: 3 + PPN_MAKE_ICS: 20 +task_make_lbcs: + NNODES_MAKE_LBCS: 3 + PPN_MAKE_LBCS: 20 + WTIME_MAKE_LBCS: 02:00:00 +task_run_fcst: + DT_ATMOS: 60 + QUILTING: true +# IO_LAYOUT_Y: 1 + WTIME_RUN_FCST: 02:45:00 +# FH_DFI_RADAR: "0.0,0.25,0.5" +task_run_post: + NNODES_RUN_POST: 1 + PPN_RUN_POST: 40 + USE_CUSTOM_POST_CONFIG_FILE: true + POST_OUTPUT_DOMAIN_NAME: RRFS_CONUS_3km + CUSTOM_POST_CONFIG_FP: '{{ [user.PARMdir, "upp/postxconfig-NT-fv3lam_rrfs.txt"]|path_join }}' +# CUSTOM_POST_PARAMS_FP: '{{ [user.PARMdir, "upp/params_grib2_tbl_new"]|path_join }}' +# TESTBED_FIELDS_FN: "testbed_fields_bgdawp.txt" +# TESTBED_FIELDS_FN2: "testbed_fields_bgrd3d.txt" +#task_process_radarref: +# RADARREFL_TIMELEVEL: [0] +#task_run_prdgen: +# ADDNL_OUTPUT_GRIDS: [] +#task_run_anal: +# NCORES_RUN_ANAL: 240 +# NCORES_RUN_OBSERVER: 80 +#task_run_ref2tten: +# PPN_RUN_REF2TTEN: 1 +#task_run_nonvarcldanl: +# PPN_RUN_NONVARCLDANL: '{{ task_run_fcst.IO_LAYOUT_Y }}' +#task_run_graphics: +# TILE_LABELS: "full" +# TILE_SETS: "full" +global: + DO_ENSEMBLE: false + NUM_ENS_MEMBERS: 9 + HALO_BLEND: 20 + PRINT_DIFF_PGR: true From 7bcb169520c6908bdd0747c076a28deee844d54a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 10 Mar 2023 00:41:59 +0000 Subject: [PATCH 11/52] Turn off all rrfs options in test case, turn them on later as PRs get merged. --- ush/config.rrfs.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ush/config.rrfs.yaml b/ush/config.rrfs.yaml index 6b12ce2910..ef25385a2a 100644 --- a/ush/config.rrfs.yaml +++ b/ush/config.rrfs.yaml @@ -50,19 +50,19 @@ workflow_switches: RUN_TASK_MAKE_SFC_CLIMO: false RUN_TASK_PLOT_ALLVARS: false rrfs: - DO_DACYCLE: true - DO_SURFACE_CYCLE: true - DO_SPINUP: true + DO_DACYCLE: false # turn on later + DO_SURFACE_CYCLE: false # turn on later + DO_SPINUP: false # turn on later DO_RETRO: true - DO_NONVAR_CLDANAL: true - DO_REFL2TTEN: true # turn off later - DO_NLDN_LGHT: true # turn off later - DO_SOIL_ADJUST: true + DO_NONVAR_CLDANAL: false # turn on later + DO_REFL2TTEN: false + DO_NLDN_LGHT: false + DO_SOIL_ADJUST: false # turn on later DO_RADDA: false DO_BUFRSND: false DO_RRFS_DEV: false - DO_POST_SPINUP: true - DO_POST_PROD: true + DO_POST_SPINUP: false # true + DO_POST_PROD: false # true cycledefs: INITIAL_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0300 ", workflow.DATE_LAST_CYCL[0:8], "2300 12:00:00"]|join }}' BOUNDARY_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0000 ", workflow.DATE_LAST_CYCL[0:8], "2300 06:00:00"]|join }}' From 913e41aa2a5b43b69ed597f828694df4a9a6c3cc Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 10 Mar 2023 00:44:43 +0000 Subject: [PATCH 12/52] Temporarily turn off jinja template variable existence test for RRFS dev. --- parm/FV3LAM_wflow.xml | 14 +++++++++++++- ush/config.rrfs.yaml | 6 +++--- ush/fill_jinja_template.py | 8 +++++++- ush/generate_FV3LAM_wflow.py | 5 ++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 745a66b6b1..8e2bce095f 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -752,6 +752,7 @@ MODULES_RUN_TASK_FP script. {%- if do_rrfs_dev %} +{%- if do_smoke_dust or do_nonvar_cldanal or do_refl2tten or do_enkf_radar_ref or do_envar_radar_ref %} {%- if do_spinup %} @@ -926,7 +927,12 @@ MODULES_RUN_TASK_FP script. ************************************************************************ --> {%- endif %} + +{%- endif %} {%- endif %} @@ -2337,6 +2343,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> +{%- if tn_prep_cyc_prod %} &RSRV_DEFAULT; @@ -2452,6 +2459,7 @@ MODULES_RUN_TASK_FP script. +{%- endif %} -{%- if do_rrfs_dev %} +{%- if do_rrfs_dev and wtime_run_fcst_long %} &RSRV_FCST; @@ -3480,6 +3488,7 @@ MODULES_RUN_TASK_FP script. {%- if not do_ensfcst %} {%- if not is_rtma %} {%- if do_rrfs_dev %} +{%- if tn_save_restart %} {{ restart_interval }} @@ -3574,6 +3583,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {%- endif %} {%- endif %} +{%- endif %} + +{%- if do_rrfs_dev %} + +{%- else %} + +{%- endif %} + {%- if run_envir == "nco" %} {%- if do_ensemble %} -@Y@m@d@H/fcst_fv3lam/dyn"> -@Y@m@d@H/fcst_fv3lam/phy"> +@Y@m@d@H/&FCST_DIR;dyn"> +@Y@m@d@H/&FCST_DIR;phy"> {%- else %} -@Y@m@d@H/fcst_fv3lam/dyn"> -@Y@m@d@H/fcst_fv3lam/phy"> +@Y@m@d@H/&FCST_DIR;dyn"> +@Y@m@d@H/&FCST_DIR;phy"> {%- endif %} {%- else %} -@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/dyn"> -@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/phy"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;dyn"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;phy"> {%- endif %} From fb41d4f8f97a5676b385a2f33e1df959feff4e5e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 14 Mar 2023 00:24:28 +0000 Subject: [PATCH 18/52] Make stochastic physics forecast work. --- scripts/exregional_run_fcst.sh | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6ae2447e87..001be2038b 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -450,6 +450,24 @@ cat > itag < Date: Tue, 14 Mar 2023 23:21:01 +0000 Subject: [PATCH 19/52] Temporarily fix run_fcst name to prod. --- scripts/exregional_run_post.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 673ac861d7..555241e796 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -167,7 +167,7 @@ fi # Set the names of the forecast model's write-component output files. # if [ "${RUN_ENVIR}" = "nco" ]; then - DATAFCST=$DATAROOT/run_fcst${dot_ensmem/./_}.${share_pid} + DATAFCST=$DATAROOT/run_fcst_prod${dot_ensmem/./_}.${share_pid} else DATAFCST=$DATA fi From 60b6717a9d604f13399f440952b7e23498f03a0a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 17 Mar 2023 01:04:07 +0000 Subject: [PATCH 20/52] Remove custom rrfs fix file. --- fix/fix_am/global_hyblev_fcst_rrfsL65.txt | 68 ----------------------- ush/generate_FV3LAM_wflow.py | 3 - 2 files changed, 71 deletions(-) delete mode 100644 fix/fix_am/global_hyblev_fcst_rrfsL65.txt diff --git a/fix/fix_am/global_hyblev_fcst_rrfsL65.txt b/fix/fix_am/global_hyblev_fcst_rrfsL65.txt deleted file mode 100644 index b1ce286eac..0000000000 --- a/fix/fix_am/global_hyblev_fcst_rrfsL65.txt +++ /dev/null @@ -1,68 +0,0 @@ - 2 66 - 0.000 1.00000000 - 0.000 0.99787363 - 0.000 0.99547387 - 0.017 0.99276698 - 0.234 0.98971388 - 1.058 0.98626929 - 3.103 0.98238159 - 7.211 0.97799245 - 14.450 0.97303658 - 26.130 0.96744145 - 43.805 0.96112727 - 69.754 0.95400221 - 107.500 0.94595747 - 160.901 0.93687684 - 233.980 0.92663902 - 330.883 0.91511905 - 455.809 0.90219033 - 612.932 0.88772712 - 806.289 0.87160769 - 1039.659 0.85371818 - 1316.413 0.83395695 - 1639.350 0.81223965 - 2010.516 0.78850460 - 2431.016 0.76271850 - 2900.827 0.73488203 - 3418.626 0.70503512 - 3981.638 0.67326135 - 4585.535 0.63969119 - 5224.384 0.60450347 - 5890.665 0.56792486 - 6575.364 0.53022703 - 7268.150 0.49172137 - 7957.623 0.45275150 - 8631.631 0.41368369 - 9277.633 0.37489594 - 9883.087 0.33676625 - 10435.842 0.29966091 - 10924.498 0.26392352 - 11338.722 0.22986547 - 11669.502 0.19775844 - 11909.318 0.16782909 - 12052.236 0.14025627 - 12093.928 0.11517042 - 12031.620 0.09265514 - 11863.989 0.07275035 - 11591.011 0.05545674 - 11213.791 0.04074101 - 10734.376 0.02854147 - 10155.564 0.01877378 - 9480.727 0.01133629 - 8714.452 0.00610706 - 7877.419 0.00279793 - 7004.872 0.00097247 - 6132.608 0.00018806 - 5296.374 0.00000379 - 4530.025 0.00000000 - 3843.468 0.00000000 - 3229.725 0.00000000 - 2681.893 0.00000000 - 2193.544 0.00000000 - 1758.738 0.00000000 - 1372.014 0.00000000 - 1028.378 0.00000000 - 723.285 0.00000000 - 452.613 0.00000000 - 212.637 0.00000000 - 0.000 0.00000000 diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 5c4c328df2..82dfecd91f 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -288,9 +288,6 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # Copy or symlink fix files # mkdir_vrfy("-p", FIXam) - if RUN_TASK_MAKE_GRID and RUN_ENVIR != "nco": - ln_vrfy(f"""-fsn '{HOMEdir}/fix/fix_am/global_hyblev_fcst_rrfsL65.txt'""" - f""" '{FIXam}/global_hyblev_fcst_rrfsL65.txt'""") if SYMLINK_FIX_FILES: log_info( From 2873640736cf7cdd60feb2898826b4caa2c3b9a0 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 17 Mar 2023 01:16:06 +0000 Subject: [PATCH 21/52] Update nwges comment. --- ush/config_defaults.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index c7a8c47cef..a38f705b6c 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1195,9 +1195,7 @@ nco: # The beginning portion of the directory that will contain the output # files from the forecast for a given cycle. For a cycle # that starts on the date specified by yyyymmdd and hour specified by hh - # (where yyyymmdd and hh are as described above), the directory in which - # the forecast output files will be placed will be: - # $NWGES/$NET/$envir/$RUN.$yyyymmdd/$hh + # ${NWGES}/yyyymmddhh # # For more information on NCO standards # From 0a4ea2d86488b0d7377d1c8afe44afcce424bdba Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 20 Mar 2023 14:37:00 +0000 Subject: [PATCH 22/52] Add back accidentally removed FCST_DIR --- parm/FV3LAM_wflow.xml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index ba915dd0bb..c9e8a673c6 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -212,18 +212,26 @@ Directories and files. {%- endif %} + + +{%- if do_rrfs_dev %} + +{%- else %} + +{%- endif %} + {%- if run_envir == "nco" %} {%- if do_ensemble %} -@Y@m@d@H/fcst_fv3lam/dyn"> -@Y@m@d@H/fcst_fv3lam/phy"> +@Y@m@d@H/&FCST_DIR;dyn"> +@Y@m@d@H/&FCST_DIR;phy"> {%- else %} -@Y@m@d@H/fcst_fv3lam/dyn"> -@Y@m@d@H/fcst_fv3lam/phy"> +@Y@m@d@H/&FCST_DIR;dyn"> +@Y@m@d@H/&FCST_DIR;phy"> {%- endif %} {%- else %} -@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/dyn"> -@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/phy"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;dyn"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;phy"> {%- endif %} From db8524fb80d421000f8723eb791c90396909d3c8 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 23 Mar 2023 02:25:06 +0000 Subject: [PATCH 23/52] Set RRFS data patsh for realtime/retrospective runs. --- ush/machine/hera.yaml | 42 ++++++++++++++++++++++++++--------------- ush/machine/jet.yaml | 27 ++++++++++++++++++++++++++ ush/machine/wcoss2.yaml | 27 ++++++++++++++++++++++++++ ush/setup.py | 24 +++++++++++++++++++---- 4 files changed, 101 insertions(+), 19 deletions(-) diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index e832108713..ee712e5e87 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -44,24 +44,36 @@ platform: FIXsfc: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_sfc_climo FIXshp: /scratch2/BMC/det/UFS_SRW_App/develop/NaturalEarth EXTRN_MDL_DATA_STORES: hpss aws nomads - ARCHIVEDIR: "" - NCARG_ROOT: "" - NCL_HOME: "" - OBSPATH: "/scratch2/BMC/zrtrr/rli/data/obs_rap" - OBSPATH_NSSLMOSIAC: "/scratch2/BMC/zrtrr/rli/data/reflectivity" - LIGHTNING_ROOT: "/scratch2/BMC/zrtrr/rli/data/lightning" - ENKF_FCST: "/scratch2/BMC/zrtrr/rli/data/enkf/atm" - FFG_DIR: "" - SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" - GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" - IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" - RAPHRR_SOIL_ROOT: "" - FIRE_RAVE_DIR: "" - AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" - SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" data: obs: RAP_obs: /scratch2/BMC/public/data/grids/rap/obs + rrfs: + retro: + OBSPATH: "/scratch2/BMC/zrtrr/rli/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/scratch2/BMC/zrtrr/rli/data/reflectivity" + LIGHTNING_ROOT: "/scratch2/BMC/zrtrr/rli/data/lightning" + ENKF_FCST: "/scratch2/BMC/zrtrr/rli/data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" + GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" + IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" + SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" + real: + OBSPATH: "/scratch2/BMC/public/data/grids/rap/obs" + OBSPATH_NSSLMOSIAC: "/scratch2/BMC/public/data/radar/nssl/mrms/conus" + LIGHTNING_ROOT: "/scratch2/BMC/public/data/lightning" + ENKF_FCST: "/scratch1/NCEPDEV/rstprod/com/gfs/prod" + FFG_DIR: "" + SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" + GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" + IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" + SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" cpl_aqm_parm: AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data AQM_BIO_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 6f490bebf2..45b0c92bd0 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -54,3 +54,30 @@ data: GFS_obs: prepbufr: /public/data/grids/gfs/prepbufr tcvitals: /public/data/grids/gfs/bufr + rrfs: + retro: + OBSPATH: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/reflectivity" + LIGHTNING_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/lightning" + ENKF_FCST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/highres_sst" + GVF_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/gvf/grib2" + IMSSNOW_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/rap_hrrr_soil" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/amdar_reject_lists" + SFCOBS_USELIST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/mesonet_uselists" + real: + OBSPATH: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/public/data/radar/nssl/mrms/conus" + LIGHTNING_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/lightning" + ENKF_FCST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/enkf/atm" + FFG_DIR: "/public/data/grids/ncep/ffg/grib2" + SST_ROOT: "/lfs4/BMC/public/data/grids/ncep/sst/0p083deg/grib2" + GVF_ROOT: "/public/data/sat/ncep/viirs/gvf/grib2" + IMSSNOW_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/rap_hrrr_soil" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/home/amb-verif/acars_RR/amdar_reject_lists" + SFCOBS_USELIST: "/lfs4/BMC/amb-verif/rap_ops_mesonet_uselists" diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index f260cdf629..2abb45a61c 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -44,6 +44,33 @@ data: RAP: compath.py ${envir}/rap/${rap_ver}/rap.${PDYext} NAM: compath.py ${envir}/nam/${nam_ver}/nam.${PDYext} HRRR: compath.py ${envir}/hrrr/${hrrr_ver}/hrrr.${PDYext}/conus + rrfs: + retro: + OBSPATH: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/obs_rap" + OBSPATH_NSSLMOSIAC: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/reflectivity" + LIGHTNING_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/lightning" + ENKF_FCST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/highres_sst" + GVF_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/gvf/grib2" + IMSSNOW_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/amdar_reject_list" + SFCOBS_USELIST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/mesonet_uselists" + real: + OBSPATH: "/lfs/h1/ops/prod/com/obsproc/v1.1" + OBSPATH_NSSLMOSIAC: "/lfs/h1/ops/prod/dcom/ldmdata/obs/upperair/mrms/conus/MergedReflectivityQC" + LIGHTNING_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/lightning" + ENKF_FCST: "/lfs/h1/ops/prod/com/gfs/v16.3" + FFG_DIR: "" + SST_ROOT: "/lfs/h1/ops/prod/com/gfs/v16.3" + GVF_ROOT: "/lfs/h1/ops/prod/dcom/viirs" + IMSSNOW_ROOT: "/lfs/h1/ops/prod/com/obsproc/v1.1" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "/lfs/h2/emc/physics/noscrub/partha.bhattacharjee/RAVE_rawdata/RAVE_NA" + AIRCRAFT_REJECT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/amdar_reject_list" + SFCOBS_USELIST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/mesonet_uselists" cpl_aqm_parm: AQM_CONFIG_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/epa/data AQM_BIO_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio diff --git a/ush/setup.py b/ush/setup.py index c6ea815d9d..e0426ab905 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -484,10 +484,6 @@ def get_location(xcs, fmt, expt_cfg): ) get_extrn_lbcs["EXTRN_MDL_SYSBASEDIR_LBCS"] = extrn_mdl_sysbasedir_lbcs - # remove the data key -- it's not needed beyond this point - if "data" in expt_config: - expt_config.pop("data") - # Check for the user-specified directories for external model files if # USE_USER_STAGED_EXTRN_FILES is set to TRUE task_keys = zip( @@ -513,6 +509,26 @@ def get_location(xcs, fmt, expt_cfg): {data_key} = \"{basedir}\"''' ) + # + # ----------------------------------------------------------------------- + # + # Set RRFS data paths for retrospective/realtime runs + # + # ----------------------------------------------------------------------- + # + if expt_config["rrfs"]["DO_RRFS_DEV"]: + mode = "retro" if expt_config["rrfs"]["DO_RETRO"] else "real" + rrfs = expt_config.get("data", {}).get("rrfs") + if rrfs is not None: + for k,v in rrfs[mode].items(): + expt_config["platform"][k] = v + + # ----------------------------------------------------------------------- + # remove the data key -- it's not needed beyond this point + # ----------------------------------------------------------------------- + if "data" in expt_config: + expt_config.pop("data") + # # ----------------------------------------------------------------------- # From dba1c0f33b5702de10fb5e74b47a664c086bf9e4 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 24 Mar 2023 23:41:55 +0000 Subject: [PATCH 24/52] Fixes due to merge. --- ush/config_defaults.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 50097076bc..acbac3e741 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1445,9 +1445,6 @@ rrfs: # DO_POST_PROD: # Flag turn on the UPP for prod cycle. # - # DO_PARALLEL_PRDGEN: - # Flag turn on parallel wgrib2 runs in prdgen . - # #----------------------------------------------------------------------- # DO_RRFS_DEV: false # Turn this on to test RRFS dev workflow @@ -1455,7 +1452,6 @@ rrfs: DO_POST_PROD: false # Turn this off temporarily DO_SPINUP: false DO_POST_SPINUP: false - DO_PARALLEL_PRDGEN: false # #----------------------------------------------------------------------- # From 742846be101713092fe2f9fb8a08252cde6d40bd Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 24 Mar 2023 23:51:36 +0000 Subject: [PATCH 25/52] Remove DO_RETRO flag. --- parm/FV3LAM_wflow.xml | 24 ++++++++++++------------ ush/config.rrfs.yaml | 1 - ush/config_defaults.yaml | 4 ---- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index c9e8a673c6..062b07b6c2 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -308,7 +308,7 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. {%- endif %} -{% if do_retro %} +{% if not do_real_time %} @@ -335,7 +335,7 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. -{%- if do_retro %} +{%- if not do_real_time %} @@ -373,7 +373,7 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. ************************************************************************ --> -{%- if do_retro %} +{%- if not do_real_time %} {% else %} @@ -1120,7 +1120,7 @@ MODULES_RUN_TASK_FP script. GDASENKF_INPUT_SUBDIR#subdirGDAS# GDAS_MEM_NAME#memNameGDAS# - {%- if not do_retro %} + {%- if do_real_time %} {%- if machine in ["WCOSS2"] %} {%- if fv3gfs_file_fmt_ics in ["netcdf"] %} @@ -1209,7 +1209,7 @@ MODULES_RUN_TASK_FP script. GDASENKF_INPUT_SUBDIR#subdirGDAS# GDAS_MEM_NAME#memNameGDAS# - {%- if not do_retro %} + {%- if do_real_time %} {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_len_hrs+extrn_mdl_lbcs_offset_hrs+1, lbc_spec_intvl_hrs) %} @@ -1302,7 +1302,7 @@ MODULES_RUN_TASK_FP script. GDASENKF_INPUT_SUBDIR#subdirGDAS# GDAS_MEM_NAME#memNameGDAS# - {%- if not do_retro %} + {%- if do_real_time %} {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, lbc_spec_intvl_hrs) %} @@ -1521,7 +1521,7 @@ MODULES_RUN_TASK_FP script. {%- endfor %} -{%- if do_retro %} +{%- if not do_real_time %} {%- if not do_ensemble %} &FG_ROOT;/@Y@m@d@H/{{ slash_ensmem_subdir }}/fcst_fv3lam/INPUT/gfs_ctrl.nc {%- endif %} @@ -1534,7 +1534,7 @@ MODULES_RUN_TASK_FP script. {%- for h in cycl_hrs_spinstart %} {{ h }}@H {%- endfor %} -{%- if do_retro %} +{%- if not do_real_time %} &FG_ROOT;/@Y@m@d@H/{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res {%- else %} @Y@m@d@H@M00 @@ -2402,7 +2402,7 @@ MODULES_RUN_TASK_FP script. {%- endfor %} {%- if do_spinup %} -{%- if do_retro %} +{%- if not do_real_time %} &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res {% else %} @@ -2421,7 +2421,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {% else %} -{%- if do_retro %} +{%- if not do_real_time %} &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/ics/gfs_data.tile7.halo0.nc @@ -2447,7 +2447,7 @@ MODULES_RUN_TASK_FP script. {%- for h in cycl_hrs_prodstart %} {{ h }}@H {%- endfor %} -{%- if do_retro %} +{%- if not do_real_time %} &FG_ROOT;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res {% else %} @@ -2729,7 +2729,7 @@ MODULES_RUN_TASK_FP script. - {%- if do_retro %} + {%- if not do_real_time %} {%- for m in range(1, num_ens_members+1) %} &NWGES_BASEDIR;/@Y@m@d@H/mem{{ "%04d" % m }}/observer_gsi/diag_conv_rw_ges.@Y@m@d@H.nc4 diff --git a/ush/config.rrfs.yaml b/ush/config.rrfs.yaml index f6764135c6..dc406f62f2 100644 --- a/ush/config.rrfs.yaml +++ b/ush/config.rrfs.yaml @@ -53,7 +53,6 @@ rrfs: DO_DACYCLE: false # turn on later DO_SURFACE_CYCLE: false # turn on later DO_SPINUP: false # turn on later - DO_RETRO: true DO_NONVAR_CLDANAL: false # turn on later DO_REFL2TTEN: false DO_NLDN_LGHT: false diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index acbac3e741..2d47bf1ac3 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1433,9 +1433,6 @@ rrfs: # Flag to turn on RRFS dev workflow. Should be removed # in the future once SRW and RRFS dev workflows are completely integrated # - # DO_RETRO: - # Flag turn on the retrospective experiments. - # # DO_SPINUP: # Flag turn on the spin-up cycle. # @@ -1448,7 +1445,6 @@ rrfs: #----------------------------------------------------------------------- # DO_RRFS_DEV: false # Turn this on to test RRFS dev workflow - DO_RETRO: true # Turn this on temporarily DO_POST_PROD: false # Turn this off temporarily DO_SPINUP: false DO_POST_SPINUP: false From 3320ee5fccf2e9168e1289df86fa9c91ed85df7c Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 25 Mar 2023 17:38:37 +0000 Subject: [PATCH 26/52] Fix RESTART_INTERVAL locations. --- ush/config.rrfs.yaml | 2 +- ush/config_defaults.yaml | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/ush/config.rrfs.yaml b/ush/config.rrfs.yaml index dc406f62f2..3ed31186f2 100644 --- a/ush/config.rrfs.yaml +++ b/ush/config.rrfs.yaml @@ -41,7 +41,6 @@ workflow: CYCL_HRS_PRODSTART: [9, 21] FCST_LEN_HRS_CYCLES: [12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3] DA_CYCLE_INTERV: 1 - RESTART_INTERVAL: 1 BOUNDARY_LEN_HRS: 18 BOUNDARY_PROC_GROUP_NUM: 10 workflow_switches: @@ -102,6 +101,7 @@ task_make_lbcs: WTIME_MAKE_LBCS: 02:00:00 task_run_fcst: DT_ATMOS: 60 + RESTART_INTERVAL: 1 QUILTING: true # IO_LAYOUT_Y: 1 WTIME_RUN_FCST: 02:45:00 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 2d47bf1ac3..03b0e39329 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1009,19 +1009,6 @@ workflow: # DA_CYCLE_INTERV: # Data assimilation cycle interval, in integer hours for now. # - # RESTART_INTERVAL: - # frequency of the output restart files (unit:hour). - # Default=0: restart files are produced at the end of a forecast run - # For example, i) RESTART_INTERVAL: 1 -1 => restart files are produced - # every hour with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory - # ii) RESTART_INTERVAL: 1 2 5 => restart files are produced only when - # fh = 1, 2, and 5. - # - # - # RESTART_INTERVAL_LONG: - # Set up frequenency or list of the forecast hours that FV3 should - # generate the restart files. - # # POSTPROC_LEN_HRS: # The length of post process, in integer hours. # @@ -1065,8 +1052,6 @@ workflow: FCST_LEN_HRS_SPINUP: 1 FCST_LEN_HRS_CYCLES: [] DA_CYCLE_INTERV: 3 - RESTART_INTERVAL: 0 # was 3,6 in rrfs_dev - RESTART_INTERVAL_LONG: 0 # was 3.6 in rrfs_dev CYCL_HRS_HYB_FV3LAM_ENS: [ "99" ] # #----------------------------------------------------------------------- @@ -2281,6 +2266,10 @@ task_run_fcst: # ii) RESTART_INTERVAL: 1 2 5 => restart files are produced only when # fh = 1, 2, and 5. # + # RESTART_INTERVAL_LONG: + # Set up frequenency or list of the forecast hours that FV3 should + # generate the restart files. + # # WRITE_DOPOST: # Flag that determines whether or not to use the inline post feature # [i.e. calling the Unified Post Processor (UPP) from within the weather @@ -2291,7 +2280,8 @@ task_run_fcst: #----------------------------------------------------------------------- # DT_ATMOS: "" - RESTART_INTERVAL: 0 + RESTART_INTERVAL: 0 # was 3,6 in rrfs_dev + RESTART_INTERVAL_LONG: 0 # was 3.6 in rrfs_dev WRITE_DOPOST: false # From 4523e8cdc18c83a3c491d4e106824e2031440711 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 25 Mar 2023 19:00:03 +0000 Subject: [PATCH 27/52] Revert back to old way of creating FIXam --- ush/generate_FV3LAM_wflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 82dfecd91f..07150090ea 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -287,8 +287,6 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # # Copy or symlink fix files # - mkdir_vrfy("-p", FIXam) - if SYMLINK_FIX_FILES: log_info( f""" @@ -298,7 +296,7 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de verbose=verbose, ) - ln_vrfy(f"""-fsn '{FIXgsm}/'* '{FIXam}'""") + ln_vrfy(f"""-fsn '{FIXgsm}' '{FIXam}'""") else: log_info( @@ -309,6 +307,8 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de verbose=verbose, ) + check_for_preexist_dir_file(FIXam, "delete") + mkdir_vrfy("-p", FIXam) mkdir_vrfy("-p", os.path.join(FIXam, "fix_co2_proj")) num_files = len(FIXgsm_FILES_TO_COPY_TO_FIXam) From 289a318ed3bf278ea8ea485973ed28511093261f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 25 Mar 2023 19:06:07 +0000 Subject: [PATCH 28/52] Other fixes. --- ush/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index e0426ab905..5b5c2b6f9b 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -517,7 +517,7 @@ def get_location(xcs, fmt, expt_cfg): # ----------------------------------------------------------------------- # if expt_config["rrfs"]["DO_RRFS_DEV"]: - mode = "retro" if expt_config["rrfs"]["DO_RETRO"] else "real" + mode = "real" if expt_config["workflow"]["DO_REAL_TIME"] else "retro" rrfs = expt_config.get("data", {}).get("rrfs") if rrfs is not None: for k,v in rrfs[mode].items(): From f32b6afbcea301e0c47e6bbe7fa25e497d324621 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 27 Mar 2023 00:35:54 +0000 Subject: [PATCH 29/52] Remove unused variables. --- parm/FV3LAM_wflow.xml | 62 +++++++++++----------------------------- ush/config_defaults.yaml | 14 --------- 2 files changed, 17 insertions(+), 59 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 062b07b6c2..e62ad80d04 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -215,23 +215,23 @@ Directories and files. {%- if do_rrfs_dev %} - + {%- else %} - + {%- endif %} {%- if run_envir == "nco" %} {%- if do_ensemble %} -@Y@m@d@H/&FCST_DIR;dyn"> -@Y@m@d@H/&FCST_DIR;phy"> +@Y@m@d@H/&FCST_SUBDIR;dyn"> +@Y@m@d@H/&FCST_SUBDIR;phy"> {%- else %} -@Y@m@d@H/&FCST_DIR;dyn"> -@Y@m@d@H/&FCST_DIR;phy"> +@Y@m@d@H/&FCST_SUBDIR;dyn"> +@Y@m@d@H/&FCST_SUBDIR;phy"> {%- endif %} {%- else %} -@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;dyn"> -@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_DIR;phy"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_SUBDIR;dyn"> +@Y@m@d@H{{ slash_ensmem_subdir }}/&FCST_SUBDIR;phy"> {%- endif %} @@ -308,24 +308,6 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. {%- endif %} -{% if not do_real_time %} - - - - - - - -{% else %} - - - - - - - -{% endif %} - @@ -345,27 +327,17 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task. {% else %} -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> -@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> +@Y@m@d@H@M'> {% endif %} - - - - - - - - - - ]> +{%- if do_smoke_dust or do_nonvar_cldanal or do_refl2tten or do_enkf_radar_ref or do_envar_radar_ref %} {%- if do_spinup %} @@ -917,7 +918,12 @@ MODULES_RUN_TASK_FP script. ************************************************************************ --> {%- endif %} + +{%- endif %} {%- endif %} @@ -2328,6 +2334,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> +{%- if tn_prep_cyc_prod %} &RSRV_DEFAULT; @@ -2443,6 +2450,7 @@ MODULES_RUN_TASK_FP script. +{%- endif %} -{%- if do_rrfs_dev %} +{%- if do_rrfs_dev and wtime_run_fcst_long %} &RSRV_FCST; @@ -3471,6 +3479,7 @@ MODULES_RUN_TASK_FP script. {%- if not do_ensfcst %} {%- if not is_rtma %} {%- if do_rrfs_dev %} +{%- if tn_save_restart %} {{ restart_interval }} @@ -3565,6 +3574,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {%- endif %} {%- endif %} +{%- endif %}