diff --git a/Externals.cfg b/Externals.cfg index 7e69c3a21b..8230973deb 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = a796a4f +hash = 36d5e2a local_path = sorc/ufs-weather-model required = True diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst index 69f5a52cf1..e8a96083a6 100644 --- a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -98,6 +98,14 @@ The fire module has the ability to print out additional messages to the log file fire: FIRE_PRINT_MSG: 1 +The fire module now supports OpenMP parallelization. By default, parallelization is off, but it can be enabled by setting ``OMP_NUM_THREADS_FIRE`` to a value greater than 1. In testing with the default case (200x200 domain) we have observed a 10-25% speedup with ``OMP_NUM_THREADS_FIRE: 2`` vs 1, with only marginal additional speedup for ``OMP_NUM_THREADS_FIRE>2``. However, larger domains may benefit from additional parallel threads. + +.. code-block:: console + + fire: + OMP_NUM_THREADS_FIRE: 2 + + Additional boundary conditions file ----------------------------------- @@ -148,6 +156,8 @@ In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (`` The CFBM creates output files in :term:`netCDF` format, with the naming scheme ``fire_output_YYYY-MM-DD_hh:mm:ss.nc``. In this case the output files are written every 30 minutes (``OUTPUT_DT_FIRE: 1800``). +Users can also modify the wind profile used in the CFBM with the setting ``WIND_VINTERP_OPT``. By default, the full prognostic wind field from the atmospheric model is used to calculate a wind profile for spreading the fire. Setting ``WIND_VINTERP_OPT: 1`` changes this to use only the diagnostic 10m wind, which is less compute-intensive and therefore runs faster. + .. note:: Any of the settings under :fire-ug:`the &fire section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. diff --git a/parm/ufs.configure b/parm/ufs.configure index 74bd602231..c775793c5b 100644 --- a/parm/ufs.configure +++ b/parm/ufs.configure @@ -33,7 +33,8 @@ AQM_attributes:: {% if ufs_fire %} # FIRE # FIRE_model: fire_behavior -FIRE_petlist_bounds: {{ FIRE_pb }} +FIRE_petlist_bounds: {{ FIRE_pb }}{{ +FIRE_omp_num_threads_line }} FIRE_attributes:: Verbosity = high Diagnostic = 0 diff --git a/scripts/exregional_integration_test.py b/scripts/exregional_integration_test.py index 299bbcee7d..ee95b3a121 100755 --- a/scripts/exregional_integration_test.py +++ b/scripts/exregional_integration_test.py @@ -126,7 +126,7 @@ def test_output_files_created(self) -> None: def test_namelist_creation(self) -> None: base_params = { - "time": {"dt", "interval_output"}, + "time": {"dt", "interval_output", "num_tiles"}, "atm": {"interval_atm", "kde"}, "fire": { "fire_num_ignitions", @@ -137,6 +137,7 @@ def test_namelist_creation(self) -> None: "fire_upwinding", "fire_lsm_zcoupling", "fire_lsm_zcoupling_ref", + "wind_vinterp_opt", }, } multifire_params = ( diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho index 3350ad8278..a900fd06df 100644 --- a/tests/WE2E/machine_suites/comprehensive.derecho +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -25,7 +25,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.hera.gnu b/tests/WE2E/machine_suites/comprehensive.hera.gnu index d293388d83..71f8096986 100644 --- a/tests/WE2E/machine_suites/comprehensive.hera.gnu +++ b/tests/WE2E/machine_suites/comprehensive.hera.gnu @@ -33,7 +33,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.hera.intel b/tests/WE2E/machine_suites/comprehensive.hera.intel index f4391bcf1d..9fdffcb57f 100644 --- a/tests/WE2E/machine_suites/comprehensive.hera.intel +++ b/tests/WE2E/machine_suites/comprehensive.hera.intel @@ -34,7 +34,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.noaacloud b/tests/WE2E/machine_suites/comprehensive.noaacloud index bda895182e..d972e08b71 100644 --- a/tests/WE2E/machine_suites/comprehensive.noaacloud +++ b/tests/WE2E/machine_suites/comprehensive.noaacloud @@ -14,7 +14,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.orion b/tests/WE2E/machine_suites/comprehensive.orion index 4c465febe6..e504650af1 100644 --- a/tests/WE2E/machine_suites/comprehensive.orion +++ b/tests/WE2E/machine_suites/comprehensive.orion @@ -25,7 +25,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.ursa.gnu b/tests/WE2E/machine_suites/comprehensive.ursa.gnu index d293388d83..71f8096986 100644 --- a/tests/WE2E/machine_suites/comprehensive.ursa.gnu +++ b/tests/WE2E/machine_suites/comprehensive.ursa.gnu @@ -33,7 +33,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/comprehensive.ursa.intel b/tests/WE2E/machine_suites/comprehensive.ursa.intel index f4391bcf1d..9fdffcb57f 100644 --- a/tests/WE2E/machine_suites/comprehensive.ursa.intel +++ b/tests/WE2E/machine_suites/comprehensive.ursa.intel @@ -34,7 +34,7 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP diff --git a/tests/WE2E/machine_suites/coverage.hercules b/tests/WE2E/machine_suites/coverage.hercules index af7332e750..007bf9b718 100644 --- a/tests/WE2E/machine_suites/coverage.hercules +++ b/tests/WE2E/machine_suites/coverage.hercules @@ -1,6 +1,6 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -#grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_two-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_two-way-coupled.yaml index 394dc03de0..16db96f024 100644 --- a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_two-way-coupled.yaml +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_two-way-coupled.yaml @@ -1,6 +1,8 @@ metadata: description: >- - Tests the UFS_FIRE capability for multiple fire ignitions with one-way (ATM-->FIRE) coupling + Tests the UFS_FIRE capability for multiple fire ignitions with two-way (ATM<-->FIRE) coupling. + Also tests OpenMP parallelism (OMP_NUM_THREADS_FIRE=2) and namelist option WIND_VINTERP_OPT=1 + user: RUN_ENVIR: community workflow: @@ -73,6 +75,8 @@ fire: FIRE_UPWINDING: 9 FIRE_LSM_ZCOUPLING: False FIRE_LSM_ZCOUPLING_REF: 60.0 + OMP_NUM_THREADS_FIRE: 2 + WIND_VINTERP_OPT: 1 task_run_fcst: fv3: execution: @@ -80,4 +84,4 @@ task_run_fcst: walltime: 01:00:00 task_integration_test: envvars: - TEST_UFS_FIRE: true \ No newline at end of file + TEST_UFS_FIRE: true diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 1edaf11429..b8fb4f0db6 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1842,7 +1842,7 @@ task_run_fcst: # QUILTING: true - PE_MEMBER01: !int '{{ task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group) + fire.FIRE_NUM_TASKS if task_run_fcst.QUILTING else task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X) + fire.FIRE_NUM_TASKS }}' + PE_MEMBER01: !int '{{ task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X + task_run_fcst.WRTCMP_write_groups * task_run_fcst.WRTCMP_write_tasks_per_group) + (fire.FIRE_NUM_TASKS * fire.OMP_NUM_THREADS_FIRE) if task_run_fcst.QUILTING else task_run_fcst.fv3.execution.threads * (task_run_fcst.LAYOUT_Y * task_run_fcst.LAYOUT_X) + (fire.FIRE_NUM_TASKS * fire.OMP_NUM_THREADS_FIRE) }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" @@ -3109,14 +3109,17 @@ fire: # The fire behavior component’s output timestep # # FIRE_NUM_TASKS - # Number of MPI tasks assigned to the FIRE_BEHAVIOR component. + # Number of MPI tasks assigned to the fire behavior component. # + # OMP_NUM_THREADS_FIRE + # Number of OpenMP tasks assigned to the fire behavior component. envvars: UFS_FIRE: False FIRE_INPUT_DIR: "" DT_FIRE: 0.5 OUTPUT_DT_FIRE: 300 FIRE_NUM_TASKS: 0 + OMP_NUM_THREADS_FIRE: 1 # The following options control namelist values in the Community Fire @@ -3207,3 +3210,9 @@ fire: FIRE_IGNITION_START_TIME: 6480 FIRE_IGNITION_END_TIME: 7000 + # WIND_VINTERP_OPT + # Option for which wind variable should be used for driving fire behavior + # 0: Use prognostic 3-d wind + # 1: Use diagnosed 10-meter wind + WIND_VINTERP_OPT: 0 + diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index f13d34b990..a4c5a1be77 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -31,9 +31,11 @@ def create_ufs_configure_file(run_dir,cfg): # Set necessary variables for each coupled configuration - atm_end = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"]) -1) + atm_end = str(int(cfg["PE_MEMBER01"]) - + int(cfg["FIRE_NUM_TASKS"] * cfg['OMP_NUM_THREADS_FIRE']) - 1) aqm_end = str(int(cfg["LAYOUT_X"]) * int(cfg["LAYOUT_Y"]) - 1) - fire_start = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"])) + fire_start = str(int(cfg["PE_MEMBER01"]) - + int(cfg["FIRE_NUM_TASKS"] * cfg['OMP_NUM_THREADS_FIRE'])) fire_end = str(int(cfg["PE_MEMBER01"]) - 1) atm_petlist_bounds = f'0 {atm_end}' @@ -55,6 +57,8 @@ def create_ufs_configure_file(run_dir,cfg): f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" atm_diag_line = '' fire_petlist_bounds = f'{fire_start} {fire_end}' + fire_omp_num_threads_line = \ + f"\nFIRE_omp_num_threads: {cfg['OMP_NUM_THREADS_FIRE']}" runseq = [ f" @{cfg['DT_ATMOS']}\n", " ATM -> FIRE\n", " FIRE -> ATM :remapmethod=conserve\n", @@ -106,6 +110,7 @@ def create_ufs_configure_file(run_dir,cfg): "runseq": runseq, "AQM_pb": "", "FIRE_pb": "", + "FIRE_omp_num_threads_line": "", "dt_atmos": cfg["DT_ATMOS"], "print_esmf": cfg["PRINT_ESMF"], "cpl_aqm": cfg["CPL_AQM"] @@ -114,6 +119,7 @@ def create_ufs_configure_file(run_dir,cfg): settings["AQM_pb"] = aqm_petlist_bounds if cfg["UFS_FIRE"]: settings["FIRE_pb"] = fire_petlist_bounds + settings["FIRE_omp_num_threads_line"] = fire_omp_num_threads_line print_info_msg( dedent( diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index b59403d54b..0b6337cde0 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -471,6 +471,8 @@ def generate_FV3LAM_wflow( fire_nml_dict["time"]["dt"] = expt_config["fire"][setting] elif setting == "OUTPUT_DT_FIRE": fire_nml_dict["time"]["interval_output"] = expt_config["fire"][setting] + elif setting == "OMP_NUM_THREADS_FIRE": + fire_nml_dict["time"]["num_tiles"] = expt_config["fire"][setting] else: # For all other settings in config.yaml, convert to lowercase # and enter into namelist.fire's &fire section diff --git a/ush/setup.py b/ush/setup.py index 4e9b4c416e..ee61d85283 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1735,6 +1735,8 @@ def _get_location(xcs, fmt, expt_cfg): FV3_HRRR, FV3_HRRR_gf, and RRFS_sas""" )) if fire_conf["FIRE_NUM_TASKS"] < 1: raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True") + if fire_conf["OMP_NUM_THREADS_FIRE"] < 1: + raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True") if fire_conf["FIRE_NUM_TASKS"] > 1: raise ValueError("FIRE_NUM_TASKS > 1 not yet supported")