ufs-community · MichaelLueken · Mar 29, 2023 · Mar 9, 2023 · Mar 9, 2023 · Mar 9, 2023
@@ -1,43 +1,35 @@
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR
-grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR
+grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16
 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta
 grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
-grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
-grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR
-grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
+grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0
+grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR
+grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0
+grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16
+grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta
 custom_ESGgrid
 deactivate_tasks
-inline_post
-specify_DOT_OR_USCORE
-specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
-specify_RESTART_INTERVAL
 specify_template_filenames
-community_ensemble_2mems
-community_ensemble_2mems_stoch
-community_ensemble_008mems
 custom_GFDLgrid
 custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE
-custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE
 grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
-grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
+grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
-grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
+grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp
-grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp
 grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2
-grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16
 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km
 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
@@ -46,12 +38,9 @@ grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta
 grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
-grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
-grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2
 nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km
@@ -3,7 +3,5 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
 nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
@@ -1,9 +1,7 @@
 grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR
-grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR
-specify_RESTART_INTERVAL
+grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16
 specify_template_filenames
-grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR
+grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
 pregen_grid_orog_sfc_climo
 custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE
-custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE
@@ -4,6 +4,5 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta
 nco_ensemble
-grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
+grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta
@@ -1,5 +1,3 @@
-MET_verification
 MET_verification_only_vx
-community_ensemble_2mems
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional_plot
@@ -2,9 +2,6 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2
-grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional_plot
-MET_ensemble_verification
-community_ensemble_2mems_stoch
 pregen_grid_orog_sfc_climo
@@ -5,6 +5,3 @@ grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2
 nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
 custom_GFDLgrid
 custom_ESGgrid
-specify_DOT_OR_USCORE
-specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
-specify_RESTART_INTERVAL
@@ -1,5 +1,3 @@
 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta
-grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta
-community_ensemble_008mems
+grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta
 deactivate_tasks
-inline_post
@@ -29,10 +29,11 @@ def monitor_jobs(expts_dict: dict, monitor_file: str = '', procs: int = 1, debug
              contains results/summary)
     """
 
-    starttime = datetime.now()
+    monitor_start = datetime.now()
     # Write monitor_file, which will contain information on each monitored experiment
+    monitor_start_string = monitor_start.strftime("%Y%m%d%H%M%S")
     if not monitor_file:
-        monitor_file = f'WE2E_tests_{starttime.strftime("%Y%m%d%H%M%S")}.yaml'
+        monitor_file = f'WE2E_tests_{monitor_start_string}.yaml'
     logging.info(f"Writing information for all experiments to {monitor_file}")
 
     write_monitor_file(monitor_file,expts_dict)
@@ -68,24 +69,30 @@ def monitor_jobs(expts_dict: dict, monitor_file: str = '', procs: int = 1, debug
         for expt in running_expts.copy():
             running_expts[expt] = expts_dict[expt]
             if running_expts[expt]["status"] in ['DEAD','ERROR','COMPLETE']:
-                logging.info(f'Experiment {expt} is {running_expts[expt]["status"]};'\
-                              'will no longer monitor.')
+                # If start_time is in dictionary, compute total walltime
+                walltimestr = ''
+                if running_expts[expt].get("start_time",{}):
+                    end = datetime.now()
+                    start = datetime.strptime(running_expts[expt]["start_time"],'%Y%m%d%H%M%S')
+                    walltime = end - start
+                    walltimestr = f'Took {str(walltime)}; '
+                    running_expts[expt]["walltime"] = str(walltime)
+
+                logging.info(f'Experiment {expt} is {running_expts[expt]["status"]}')
+                logging.info(f'{walltimestr}will no longer monitor.')
                 running_expts.pop(expt)
                 continue
             logging.debug(f'Experiment {expt} status is {expts_dict[expt]["status"]}')
 
         write_monitor_file(monitor_file,expts_dict)
         endtime = datetime.now()
-        total_walltime = endtime - starttime
+        total_walltime = endtime - monitor_start
 
         logging.debug(f"Finished loop {i}\nWalltime so far is {str(total_walltime)}")
         #Slow things down just a tad between loops so experiments behave better
         time.sleep(5)
 
-    endtime = datetime.now()
-    total_walltime = endtime - starttime
-
-    logging.info(f'All {len(expts_dict)} experiments finished in {str(total_walltime)}')
+    logging.info(f'All {len(expts_dict)} experiments finished')
     logging.info('Calculating core-hour usage and printing final summary')
 
     # Calculate core hours and update yaml

@@ -138,6 +138,8 @@ def run_we2e_tests(homedir, args) -> None:
     for test in tests_to_run:
         #Starting with test yaml template, fill in user-specified and machine- and
         # test-specific options, then write resulting complete config.yaml
+        starttime = datetime.now()
+        starttime_string = starttime.strftime("%Y%m%d%H%M%S")
         test_name = os.path.basename(test).split('.')[1]
         logging.debug(f"For test {test_name}, constructing config.yaml")
         test_cfg = load_config_file(test)
@@ -213,10 +215,11 @@ def run_we2e_tests(homedir, args) -> None:
             monitor_yaml[test_name] = dict()
             monitor_yaml[test_name].update({"expt_dir": expt_dir})
             monitor_yaml[test_name].update({"status": "CREATED"})
+            monitor_yaml[test_name].update({"start_time": starttime_string})
 
     if not args.use_cron_to_relaunch:
         logging.info("calling function that monitors jobs, prints summary")
-        monitor_file = f'WE2E_tests_{datetime.now().strftime("%Y%m%d%H%M%S")}.yaml'
+        monitor_file = f'WE2E_tests_{starttime_string}.yaml'
         write_monitor_file(monitor_file,monitor_yaml)
         try:
             monitor_file = monitor_jobs(monitor_yaml, monitor_file=monitor_file, procs=args.procs,

@@ -2,16 +2,16 @@ metadata:
   description: |-
     This test is to ensure that the workflow running in community mode
     completes successfully on the CONUS_3km_GFDLgrid grid (which is a
-    GFDLgrid type of grid) using the GFS_v16 suite with ICs and LBCs
+    GFDLgrid type of grid) using the RRFS_v1beta suite with ICs and LBCs
     derived from the FV3GFS.
 user:
   RUN_ENVIR: community
 workflow:
-  CCPP_PHYS_SUITE: FV3_GFS_v16
+  CCPP_PHYS_SUITE: FV3_RRFS_v1beta
   PREDEF_GRID_NAME: CONUS_3km_GFDLgrid
   DATE_FIRST_CYCL: '2019070100'
   DATE_LAST_CYCL: '2019070100'
-  FCST_LEN_HRS: 6
+  FCST_LEN_HRS: 3
   PREEXISTING_DIR_METHOD: rename
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS

@@ -1,21 +1,21 @@
 metadata:
   description: |-
     This test is to ensure that the workflow running in community mode
-    completes successfully on the RRFS_AK_3km grid using the GFS_v16
+    completes successfully on the RRFS_AK_3km grid using the HRRR
     physics suite with ICs and LBCs derived from the FV3GFS.
 user:
   RUN_ENVIR: community
 workflow:
-  CCPP_PHYS_SUITE: FV3_GFS_v16
+  CCPP_PHYS_SUITE: FV3_HRRR
   PREDEF_GRID_NAME: RRFS_AK_3km
   DATE_FIRST_CYCL: '2019070100'
   DATE_LAST_CYCL: '2019070100'
-  FCST_LEN_HRS: 6
+  FCST_LEN_HRS: 3
   PREEXISTING_DIR_METHOD: rename
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   USE_USER_STAGED_EXTRN_FILES: true
 task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
-  LBC_SPEC_INTVL_HRS: 6
+  LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
@@ -3,13 +3,16 @@ metadata:
     This test is to ensure that the workflow running in community mode
     completes successfully on the RRFS_CONUS_25km grid using the GFS_2017_gfdlmp
     physics suite with ICs and LBCs derived from the FV3GFS.
+    This test additionally includes the cycling ensemble capability, with
+    2 members run over 2 cycles.
 user:
   RUN_ENVIR: community
 workflow:
   CCPP_PHYS_SUITE: FV3_GFS_2017_gfdlmp
   PREDEF_GRID_NAME: RRFS_CONUS_25km
-  DATE_FIRST_CYCL: '2019070100'
-  DATE_LAST_CYCL: '2019070100'
+  DATE_FIRST_CYCL: '2019070112'
+  DATE_LAST_CYCL: '2019070200'
+  INCR_CYCL_FREQ: 12
   FCST_LEN_HRS: 6
   PREEXISTING_DIR_METHOD: rename
 task_get_extrn_ics:
@@ -19,3 +22,6 @@ task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+global:
+  DO_ENSEMBLE: true
+  NUM_ENS_MEMBERS: 2
@@ -3,6 +3,7 @@ metadata:
     This test is to ensure that the workflow running in community mode
     completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2
     physics suite with ICs and LBCs derived from the FV3GFS.
+    In addition, this tests the "inline post" option (WRITE_DOPOST: true)
 user:
   RUN_ENVIR: community
 workflow:
@@ -19,3 +20,5 @@ task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+task_run_fcst:
+  WRITE_DOPOST: true
@@ -3,6 +3,7 @@ metadata:
     This test is to ensure that the workflow running in community mode
     completes successfully on the RRFS_CONUS_25km grid using the GFS_v16
     physics suite with ICs and LBCs derived from the FV3GFS.
+    Also tests the capability to write restart files.
 user:
   RUN_ENVIR: community
 workflow:
@@ -19,3 +20,5 @@ task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+task_run_fcst:
+  RESTART_INTERVAL: 1 2 5
@@ -4,25 +4,29 @@ metadata:
     completes successfully on the RRFS_CONUS_25km grid using the HRRR
     physics suite with ICs derived from the FV3GFS and LBCs derived from
     the RAP.
-    Note that this test specifies the file format of the FV3GFS external
-    model data (from which to generate ICs) to be "grib2" as opposed to
-    the default value of "nemsio".
+    This test also ensures that MET deterministic verification runs
+    successfully.
 user:
   RUN_ENVIR: community
 workflow:
   CCPP_PHYS_SUITE: FV3_HRRR
   PREDEF_GRID_NAME: RRFS_CONUS_25km
-  DATE_FIRST_CYCL: '2020081000'
-  DATE_LAST_CYCL: '2020081000'
+  DATE_FIRST_CYCL: '2019061518'
+  DATE_LAST_CYCL: '2019061518'
   FCST_LEN_HRS: 6
   PREEXISTING_DIR_METHOD: rename
+workflow_switches:
+  RUN_TASK_VX_GRIDSTAT: true
+  RUN_TASK_VX_POINTSTAT: true
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   FV3GFS_FILE_FMT_ICS: grib2
   USE_USER_STAGED_EXTRN_FILES: true
 task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: RAP
   LBC_SPEC_INTVL_HRS: 1
+  EXTRN_MDL_LBCS_OFFSET_HRS: 0
   USE_USER_STAGED_EXTRN_FILES: true
-  EXTRN_MDL_FILES_LBCS:
-    - '{yy}{jjj}{hh}00{fcst_hr:02d}00'
+verification:
+  RUN_TASKS_METVX_DET: true
+  VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km
@@ -1,21 +1,36 @@
 metadata:
   description: |-
     This test is to ensure that the workflow running in community mode
-    completes successfully on the RRFS_CONUS_25km grid using the HRRR
+    completes successfully on the RRFS_CONUS_25km grid using the GFS_v16
     physics suite with ICs and LBCs derived from the NAM.
+    This test also runs with two ensemble members, and ensures the MET
+    ensemble-specific tasks run successfully.
 user:
   RUN_ENVIR: community
 workflow:
-  CCPP_PHYS_SUITE: FV3_HRRR
+  CCPP_PHYS_SUITE: FV3_GFS_v16
   PREDEF_GRID_NAME: RRFS_CONUS_25km
-  DATE_FIRST_CYCL: '2021061500'
-  DATE_LAST_CYCL: '2021061500'
+  DATE_FIRST_CYCL: '2021051212'
+  DATE_LAST_CYCL: '2021051212'
   FCST_LEN_HRS: 6
   PREEXISTING_DIR_METHOD: rename
+workflow_switches:
+  RUN_TASK_VX_GRIDSTAT: true
+  RUN_TASK_VX_POINTSTAT: true
+  RUN_TASK_VX_ENSGRID: true
+  RUN_TASK_VX_ENSPOINT: true
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: NAM
   USE_USER_STAGED_EXTRN_FILES: true
 task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: NAM
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+global:
+  DO_ENSEMBLE: true
+  NUM_ENS_MEMBERS: 2
+verification:
+  RUN_TASKS_METVX_DET: true
+  RUN_TASKS_METVX_ENS: true
+  VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km
+