[develop] Updates for fire behavior model: OpenMP parallelization; new wind option

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = a796a4f
+hash = 36d5e2a
 local_path = sorc/ufs-weather-model
 required = True
 

doc/UsersGuide/BuildingRunningTesting/FIRE.rst

@@ -98,6 +98,14 @@ The fire module has the ability to print out additional messages to the log file
    fire:
      FIRE_PRINT_MSG: 1
 
+The fire module now supports OpenMP parallelization. By default, parallelization is off, but it can be enabled by setting ``OMP_NUM_THREADS_FIRE`` to a value greater than 1. In testing with the default case (200x200 domain) we have observed a 10-25% speedup with ``OMP_NUM_THREADS_FIRE: 2`` vs 1, with only marginal additional speedup for ``OMP_NUM_THREADS_FIRE>2``. However, larger domains may benefit from additional parallel threads.
+
+.. code-block:: console
+
+   fire:
+     OMP_NUM_THREADS_FIRE: 2
+
+
 Additional boundary conditions file
 -----------------------------------
 
@@ -148,6 +156,8 @@ In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (``
 
 The CFBM creates output files in :term:`netCDF` format, with the naming scheme ``fire_output_YYYY-MM-DD_hh:mm:ss.nc``. In this case the output files are written every 30 minutes (``OUTPUT_DT_FIRE: 1800``).
 
+Users can also modify the wind profile used in the CFBM with the setting ``WIND_VINTERP_OPT``. By default, the full prognostic wind field from the atmospheric model is used to calculate a wind profile for spreading the fire. Setting ``WIND_VINTERP_OPT: 1`` changes this to use only the diagnostic 10m wind, which is less compute-intensive and therefore runs faster.
+
 .. note::
 
   Any of the settings under :fire-ug:`the &fire section of the namelist <Configuration.html#fire>` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail.

parm/ufs.configure

@@ -33,7 +33,8 @@ AQM_attributes::
 {% if ufs_fire %}
 # FIRE #
 FIRE_model:                     fire_behavior
-FIRE_petlist_bounds:            {{ FIRE_pb }}
+FIRE_petlist_bounds:            {{ FIRE_pb }}{{
+FIRE_omp_num_threads_line }}
 FIRE_attributes::
   Verbosity = high
   Diagnostic = 0

scripts/exregional_integration_test.py

@@ -126,7 +126,7 @@ def test_output_files_created(self) -> None:
 
     def test_namelist_creation(self) -> None:
         base_params = {
-            "time": {"dt", "interval_output"},
+            "time": {"dt", "interval_output", "num_tiles"},
             "atm": {"interval_atm", "kde"},
             "fire": {
                 "fire_num_ignitions",
@@ -137,6 +137,7 @@ def test_namelist_creation(self) -> None:
                 "fire_upwinding",
                 "fire_lsm_zcoupling",
                 "fire_lsm_zcoupling_ref",
+                "wind_vinterp_opt",
             },
         }
         multifire_params = (

tests/WE2E/machine_suites/comprehensive.derecho

@@ -25,7 +25,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.hera.gnu

@@ -33,7 +33,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.hera.intel

@@ -34,7 +34,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.noaacloud

@@ -14,7 +14,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.orion

@@ -25,7 +25,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.ursa.gnu

@@ -33,7 +33,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/comprehensive.ursa.intel

@@ -34,7 +34,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP

tests/WE2E/machine_suites/coverage.hercules

@@ -1,6 +1,6 @@
 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot
 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2
-#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
+grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR
 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP
 grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16
 specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS

tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_two-way-coupled.yaml

@@ -1,6 +1,8 @@
 metadata:
   description: >-
-    Tests the UFS_FIRE capability for multiple fire ignitions with one-way (ATM-->FIRE) coupling
+    Tests the UFS_FIRE capability for multiple fire ignitions with two-way (ATM<-->FIRE) coupling.
+    Also tests OpenMP parallelism (OMP_NUM_THREADS_FIRE=2) and namelist option WIND_VINTERP_OPT=1
+
 user:
   RUN_ENVIR: community
 workflow:
@@ -73,11 +75,13 @@ fire:
   FIRE_UPWINDING: 9
   FIRE_LSM_ZCOUPLING: False
   FIRE_LSM_ZCOUPLING_REF: 60.0
+  OMP_NUM_THREADS_FIRE: 2
+  WIND_VINTERP_OPT: 1
 task_run_fcst:
   fv3:
     execution:
       batchargs:
         walltime: 01:00:00
 task_integration_test:
   envvars:
-    TEST_UFS_FIRE: true
+    TEST_UFS_FIRE: true

ush/config_defaults.yaml

@@ -1842,7 +1842,7 @@ task_run_fcst:
   #
   QUILTING: true
 
-  PE_MEMBER01: !int '{{ task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group) + fire.FIRE_NUM_TASKS if task_run_fcst.QUILTING else task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X) + fire.FIRE_NUM_TASKS }}'
+  PE_MEMBER01: !int '{{ task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group) + (fire.FIRE_NUM_TASKS * fire.OMP_NUM_THREADS_FIRE) if task_run_fcst.QUILTING else task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X) + (fire.FIRE_NUM_TASKS * fire.OMP_NUM_THREADS_FIRE) }}'
 
   WRTCMP_write_groups: ""
   WRTCMP_write_tasks_per_group: ""
@@ -3109,14 +3109,17 @@ fire:
   # The fire behavior component’s output timestep
   # 
   # FIRE_NUM_TASKS
-  # Number of MPI tasks assigned to the FIRE_BEHAVIOR component.
+  # Number of MPI tasks assigned to the fire behavior component.
   #
+  # OMP_NUM_THREADS_FIRE
+  # Number of OpenMP tasks assigned to the fire behavior component.
   envvars:
     UFS_FIRE: False
     FIRE_INPUT_DIR: ""
   DT_FIRE: 0.5
   OUTPUT_DT_FIRE: 300
   FIRE_NUM_TASKS: 0
+  OMP_NUM_THREADS_FIRE: 1
 
 
   # The following options control namelist values in the Community Fire
@@ -3207,3 +3210,9 @@ fire:
   FIRE_IGNITION_START_TIME: 6480
   FIRE_IGNITION_END_TIME: 7000
 
+  # WIND_VINTERP_OPT
+  # Option for which wind variable should be used for driving fire behavior
+  #   0: Use prognostic 3-d wind
+  #   1: Use diagnosed 10-meter wind
+  WIND_VINTERP_OPT: 0
+

ush/create_ufs_configure_file.py

@@ -31,9 +31,11 @@ def create_ufs_configure_file(run_dir,cfg):
 
     # Set necessary variables for each coupled configuration
 
-    atm_end = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"]) -1)
+    atm_end = str(int(cfg["PE_MEMBER01"]) -
+                  int(cfg["FIRE_NUM_TASKS"] * cfg['OMP_NUM_THREADS_FIRE']) - 1)
     aqm_end = str(int(cfg["LAYOUT_X"]) * int(cfg["LAYOUT_Y"]) - 1)
-    fire_start = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"]))
+    fire_start = str(int(cfg["PE_MEMBER01"]) -
+                     int(cfg["FIRE_NUM_TASKS"] * cfg['OMP_NUM_THREADS_FIRE']))
     fire_end = str(int(cfg["PE_MEMBER01"]) - 1)
 
     atm_petlist_bounds = f'0 {atm_end}'
@@ -55,6 +57,8 @@ def create_ufs_configure_file(run_dir,cfg):
             f"\nATM_omp_num_threads:            {cfg['OMP_NUM_THREADS_RUN_FCST']}"
         atm_diag_line = ''
         fire_petlist_bounds = f'{fire_start} {fire_end}'
+        fire_omp_num_threads_line = \
+            f"\nFIRE_omp_num_threads:            {cfg['OMP_NUM_THREADS_FIRE']}"
         runseq = [ f"  @{cfg['DT_ATMOS']}\n",
                    "    ATM -> FIRE\n",
                    "    FIRE -> ATM :remapmethod=conserve\n",
@@ -106,6 +110,7 @@ def create_ufs_configure_file(run_dir,cfg):
       "runseq": runseq,
       "AQM_pb": "",
       "FIRE_pb": "",
+      "FIRE_omp_num_threads_line": "",
       "dt_atmos": cfg["DT_ATMOS"],
       "print_esmf": cfg["PRINT_ESMF"],
       "cpl_aqm": cfg["CPL_AQM"]
@@ -114,6 +119,7 @@ def create_ufs_configure_file(run_dir,cfg):
         settings["AQM_pb"] = aqm_petlist_bounds
     if cfg["UFS_FIRE"]:
         settings["FIRE_pb"] = fire_petlist_bounds
+        settings["FIRE_omp_num_threads_line"] = fire_omp_num_threads_line
 
     print_info_msg(
         dedent(

ush/generate_FV3LAM_wflow.py

@@ -471,6 +471,8 @@ def generate_FV3LAM_wflow(
                 fire_nml_dict["time"]["dt"] = expt_config["fire"][setting]
             elif setting == "OUTPUT_DT_FIRE":
                 fire_nml_dict["time"]["interval_output"] = expt_config["fire"][setting]
+            elif setting == "OMP_NUM_THREADS_FIRE":
+                fire_nml_dict["time"]["num_tiles"] = expt_config["fire"][setting]
             else:
                 # For all other settings in config.yaml, convert to lowercase
                 # and enter into namelist.fire's &fire section

ush/setup.py

@@ -1735,6 +1735,8 @@ def _get_location(xcs, fmt, expt_cfg):
                       FV3_HRRR, FV3_HRRR_gf, and RRFS_sas""" ))
             if fire_conf["FIRE_NUM_TASKS"] < 1:
                 raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True")
+            if fire_conf["OMP_NUM_THREADS_FIRE"] < 1:
+                raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True")
             if fire_conf["FIRE_NUM_TASKS"] > 1:
                 raise ValueError("FIRE_NUM_TASKS > 1 not yet supported")