From f0b89752874ee269828d0219e6dad4c64016de96 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 26 Apr 2024 16:31:17 -0600 Subject: [PATCH 01/65] Updates for upp. --- src/uwtools/cli.py | 58 ++++++++++++++++++++++++++++++++++++++++++ src/uwtools/strings.py | 1 + 2 files changed, 59 insertions(+) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index edf3d8993..7ac979389 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -24,6 +24,7 @@ import uwtools.api.sfc_climo_gen import uwtools.api.template import uwtools.api.ungrib +import uwtools.api.upp import uwtools.config.jinja2 import uwtools.rocoto from uwtools.exceptions import UWConfigRealizeError, UWError, UWTemplateRenderError @@ -71,6 +72,7 @@ def main() -> None: STR.sfcclimogen: _dispatch_sfc_climo_gen, STR.template: _dispatch_template, STR.ungrib: _dispatch_ungrib, + STR.upp: _dispatch_upp, } sys.exit(0 if modes[args[STR.mode]](args) else 1) except UWError as e: @@ -903,6 +905,61 @@ def _dispatch_ungrib(args: Args) -> bool: ) +# Mode upp + + +def _add_subparser_upp(subparsers: Subparsers) -> ModeChecks: + """ + Subparser for mode: upp + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.upp, "Execute UPP tasks") + _basic_setup(parser) + subparsers = _add_subparsers(parser, STR.action, STR.task.upper()) + return { + task: _add_subparser_upp_task(subparsers, task, helpmsg) + for task, helpmsg in uwtools.api.upp.tasks().items() + } + + +def _add_subparser_upp_task(subparsers: Subparsers, task: str, helpmsg: str) -> ActionChecks: + """ + Subparser for mode: UPP + + :param subparsers: Parent parser's subparsers, to add this subparser to. + :param task: The task to add a subparser for. + :param helpmsg: Help message for task. + """ + parser = _add_subparser(subparsers, task, helpmsg.rstrip(".")) + required = parser.add_argument_group(TITLE_REQ_ARG) + _add_arg_cycle(required) + optional = _basic_setup(parser) + _add_arg_config_file(group=optional) + _add_arg_batch(optional) + _add_arg_dry_run(optional) + _add_arg_graph_file(optional) + checks = _add_args_verbosity(optional) + return checks + + +def _dispatch_upp(args: Args) -> bool: + """ + Dispatch logic for upp mode. + + :param args: Parsed command-line args. + """ + return uwtools.api.upp.execute( + task=args[STR.action], + config=args[STR.cfgfile], + cycle=args[STR.cycle], + batch=args[STR.batch], + dry_run=args[STR.dryrun], + graph_file=args[STR.graphfile], + stdin_ok=True, + ) + + # Arguments # pylint: disable=missing-function-docstring @@ -1289,6 +1346,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: STR.sfcclimogen: _add_subparser_sfc_climo_gen(subparsers), STR.template: _add_subparser_template(subparsers), STR.ungrib: _add_subparser_ungrib(subparsers), + STR.upp: _add_subparser_upp(subparsers), } return vars(parser.parse_args(raw_args)), checks diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 27ea4bd64..b6cfbd31f 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -108,6 +108,7 @@ class STR: total: str = "total" translate: str = "translate" ungrib: str = "ungrib" + upp: str = "upp" validate: str = "validate" valsfile: str = "values_file" valsfmt: str = "values_format" From 3b0b612a1825610fddab38d00d819d1a3cbad9ad Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 26 Apr 2024 16:32:22 -0600 Subject: [PATCH 02/65] Add new files for UPP driver. --- docs/shared/upp.yaml | 34 ++++++++ src/uwtools/api/upp.py | 59 +++++++++++++ src/uwtools/drivers/upp.py | 164 +++++++++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 docs/shared/upp.yaml create mode 100644 src/uwtools/api/upp.py create mode 100644 src/uwtools/drivers/upp.py diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml new file mode 100644 index 000000000..843621491 --- /dev/null +++ b/docs/shared/upp.yaml @@ -0,0 +1,34 @@ +upp: + execution: + executable: upp.x + use_crtm: false + crtm_fix: /path/to/crtm/files + files_to_copy: + eta_micro_lookup.dat: /path/to/lookup.dat + postxconfig-NT.txt: /path/to/post/config + params_grib2_tbl_new: /path/to/params_grib2_tbl_new + crtm_files: + - Nalli.IRwater.EmisCoeff.bin + - FAST*.bin + - NPOESS.IRland.EmisCoeff.bin + - NPOESS.IRsnow.EmisCoeff.bin + - NPOESS.IRice.EmisCoeff.bin + - AerosolCoeff.bin + - CloudCoeff.bin + - '*.SpcCoeff.bin' + - '*.TauCoeff.bin' + namelist: # named itag; taken as input so arbitrary + base_file: + update_values: + model_inputs: + datestr: "{{ valid_time.strftime('%Y-%m-%d_%H::%M:%S') }}" + filename: /path/to/forecast/dynf{{ fcst_time.strftime("%H:%M:%S") }}.nc + filenameflat: postxconfig-NT.txt + filenameflux: /path/to/forecast/phyf{{ fcst_time.strftime("%H:%M:%S") }}.nc + grib: grib2 + ioform: netcdf + modelname: FV3R + nampgb: + kpo: 47 + po: [1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,] + numx: 1 diff --git a/src/uwtools/api/upp.py b/src/uwtools/api/upp.py new file mode 100644 index 000000000..4d7b9e624 --- /dev/null +++ b/src/uwtools/api/upp.py @@ -0,0 +1,59 @@ +""" +API access to the ``uwtools`` ``upp`` driver. +""" + +import datetime as dt +from pathlib import Path +from typing import Dict, Optional, Union + +import uwtools.drivers.support as _support +from uwtools.drivers.upp import UPP as _UPP +from uwtools.utils.api import ensure_data_source as _ensure_data_source + + +def execute( + task: str, + cycle: dt.datetime, + config: Optional[Union[Path, str]] = None, + batch: bool = False, + dry_run: bool = False, + graph_file: Optional[Path] = None, + stdin_ok: bool = False, +) -> bool: + """ + Execute an ``upp`` task. + + If ``batch`` is specified, a runscript will be written and submitted to the batch system. + Otherwise, the executable will be run directly on the current system. + + :param task: The task to execute. + :param cycle: The cycle. + :param config: Path to config file (read stdin if missing or None). + :param batch: Submit run to the batch system? + :param dry_run: Do not run the executable, just report what would have been done. + :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? + :return: ``True`` if task completes without raising an exception. + """ + obj = _UPP( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) + getattr(obj, task)() + if graph_file: + with open(graph_file, "w", encoding="utf-8") as f: + print(graph(), file=f) + return True + + +def graph() -> str: + """ + Returns Graphviz DOT code for the most recently executed task. + """ + return _support.graph() + + +def tasks() -> Dict[str, str]: + """ + Returns a mapping from task names to their one-line descriptions. + """ + return _support.tasks(_UPP) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py new file mode 100644 index 000000000..30795c147 --- /dev/null +++ b/src/uwtools/drivers/upp.py @@ -0,0 +1,164 @@ +""" +A driver for the UPP component. +""" + +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from iotaa import asset, dryrun, task, tasks + +from uwtools.config.formats.nml import NMLConfig +from uwtools.drivers.driver import Driver +from uwtools.strings import STR +from uwtools.utils.tasks import file + + +class UPP(Driver): + """ + A driver for UPP. + """ + + def __init__( + self, + cycle: datetime, + lead_time: timedelta, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + ): + """ + The driver. + + :param cycle: The cycle. + :param lead_time: The length of the forecast. + :param config: Path to config file (read stdin if missing or None). + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + """ + super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle) + if self._dry_run: + dryrun() + self._cycle = cycle + fcst_time = datetime.min() + lead_time + valid_time = cycle + lead_time + self._config.dereference( + context={**({"cycle": cycle, "fcst_time": fcst_time, "valid_time": valid_time}), **self._config.data} + ) + + # Workflow tasks + + @tasks + def files_copied(self): + """ + Files copied for run. + """ + yield self._taskname("files copied") + if self._config + yield [ + filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() + ] + + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self._taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] + + @task + def namelist_file(self): + """ + The namelist file. + """ + path = self._rundir / f"itag.{fcst_time.strftime("%H:%M:%S")}" + yield self._taskname(str(path)) + yield asset(path, path.is_file) + yield None + path.parent.mkdir(parents=True, exist_ok=True) + self._create_user_updated_config( + config_class=NMLConfig, + config_values=d, + path=path, + ) + + @tasks + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [ + self.files_copied(), + self.files_linked(), + self.namelist_file(), + self.runscript(), + ] + + @task + def runscript(self): + """ + The runscript. + """ + path = self._runscript_path + yield self._taskname(path.name) + yield asset(path, path.is_file) + yield None + self._write_runscript(path=path, envvars={}) + + @task + def vtable(self): + """ + A symlink to the Vtable file. + """ + path = self._rundir / "Vtable" + yield self._taskname(str(path)) + yield asset(path, path.is_symlink) + infile = Path(self._driver_config["vtable"]) + yield file(path=infile) + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(Path(self._driver_config["vtable"])) + + # Private helper methods + + @property + def _driver_name(self) -> str: + """ + Returns the name of this driver. + """ + return STR.upp + + @task + def _gribfile(self, infile: Path, link: Path): + """ + A symlink to an input GRIB file. + + :param link: Link name. + :param infile: File to link. + """ + yield self._taskname(str(link)) + yield asset(link, link.is_symlink) + yield file(path=infile) + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(infile) + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return "%s %s %s" % (self._cycle.strftime("%Y%m%d %HZ"), self._driver_name, suffix) + + +def _ext(n): + """ + Maps integers to 3-letter string. + """ + b = 26 + return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] From 727282d8eed337a80b64a1315701b4b078611c14 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 30 Apr 2024 18:38:44 -0600 Subject: [PATCH 03/65] Remove old vtable task from new upp driver. --- src/uwtools/drivers/upp.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 30795c147..8d1a655f9 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -111,19 +111,6 @@ def runscript(self): yield None self._write_runscript(path=path, envvars={}) - @task - def vtable(self): - """ - A symlink to the Vtable file. - """ - path = self._rundir / "Vtable" - yield self._taskname(str(path)) - yield asset(path, path.is_symlink) - infile = Path(self._driver_config["vtable"]) - yield file(path=infile) - path.parent.mkdir(parents=True, exist_ok=True) - path.symlink_to(Path(self._driver_config["vtable"])) - # Private helper methods @property From 4e03aa4b2aa2dc6d626e5f0377e156329fdc896f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 2 May 2024 15:16:30 +0000 Subject: [PATCH 04/65] WIP --- src/uwtools/api/upp.py | 63 ++++---------------------------- src/uwtools/cli.py | 2 +- src/uwtools/drivers/upp.py | 75 ++++++++++++++++++++------------------ 3 files changed, 48 insertions(+), 92 deletions(-) diff --git a/src/uwtools/api/upp.py b/src/uwtools/api/upp.py index 4d7b9e624..53dd8f6f9 100644 --- a/src/uwtools/api/upp.py +++ b/src/uwtools/api/upp.py @@ -2,58 +2,11 @@ API access to the ``uwtools`` ``upp`` driver. """ -import datetime as dt -from pathlib import Path -from typing import Dict, Optional, Union - -import uwtools.drivers.support as _support -from uwtools.drivers.upp import UPP as _UPP -from uwtools.utils.api import ensure_data_source as _ensure_data_source - - -def execute( - task: str, - cycle: dt.datetime, - config: Optional[Union[Path, str]] = None, - batch: bool = False, - dry_run: bool = False, - graph_file: Optional[Path] = None, - stdin_ok: bool = False, -) -> bool: - """ - Execute an ``upp`` task. - - If ``batch`` is specified, a runscript will be written and submitted to the batch system. - Otherwise, the executable will be run directly on the current system. - - :param task: The task to execute. - :param cycle: The cycle. - :param config: Path to config file (read stdin if missing or None). - :param batch: Submit run to the batch system? - :param dry_run: Do not run the executable, just report what would have been done. - :param graph_file: Write Graphviz DOT output here. - :param stdin_ok: OK to read from stdin? - :return: ``True`` if task completes without raising an exception. - """ - obj = _UPP( - cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run - ) - getattr(obj, task)() - if graph_file: - with open(graph_file, "w", encoding="utf-8") as f: - print(graph(), file=f) - return True - - -def graph() -> str: - """ - Returns Graphviz DOT code for the most recently executed task. - """ - return _support.graph() - - -def tasks() -> Dict[str, str]: - """ - Returns a mapping from task names to their one-line descriptions. - """ - return _support.tasks(_UPP) +from uwtools.drivers.support import graph +from uwtools.drivers.upp import UPP as _Driver +from uwtools.utils.api import make_execute as _make_execute +from uwtools.utils.api import make_tasks as _make_tasks + +execute = _make_execute(_Driver, with_cycle=True) +tasks = _make_tasks(_Driver) +__all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index ee62087c2..c3b980c3e 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -975,7 +975,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: (STR.mpasinit, True), (STR.sfcclimogen, False), (STR.ungrib, True), - (STR.upp, False), + (STR.upp, True), ] } modes = {**tools, **drivers} diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 8d1a655f9..0cb5a9f1e 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -1,5 +1,5 @@ """ -A driver for the UPP component. +A driver for UPP. """ from datetime import datetime, timedelta @@ -8,10 +8,10 @@ from iotaa import asset, dryrun, task, tasks -from uwtools.config.formats.nml import NMLConfig +# from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import Driver from uwtools.strings import STR -from uwtools.utils.tasks import file +from uwtools.utils.tasks import file, symlink class UPP(Driver): @@ -22,7 +22,7 @@ class UPP(Driver): def __init__( self, cycle: datetime, - lead_time: timedelta, + leadtime: timedelta, config: Optional[Path] = None, dry_run: bool = False, batch: bool = False, @@ -31,7 +31,7 @@ def __init__( The driver. :param cycle: The cycle. - :param lead_time: The length of the forecast. + :param leadtime: The length of the forecast. :param config: Path to config file (read stdin if missing or None). :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? @@ -40,25 +40,28 @@ def __init__( if self._dry_run: dryrun() self._cycle = cycle - fcst_time = datetime.min() + lead_time - valid_time = cycle + lead_time + fcst_time = cycle # datetime.min() + leadtime + valid_time = cycle + leadtime self._config.dereference( - context={**({"cycle": cycle, "fcst_time": fcst_time, "valid_time": valid_time}), **self._config.data} + context={ + **({"cycle": cycle, "fcst_time": fcst_time, "valid_time": valid_time}), + **self._config.data, + } ) # Workflow tasks - @tasks - def files_copied(self): - """ - Files copied for run. - """ - yield self._taskname("files copied") - if self._config - yield [ - filecopy(src=Path(src), dst=self._rundir / dst) - for dst, src in self._driver_config.get("files_to_copy", {}).items() - ] + # @tasks + # def files_copied(self): + # """ + # Files copied for run. + # """ + # yield self._taskname("files copied") + # if self._config + # yield [ + # filecopy(src=Path(src), dst=self._rundir / dst) + # for dst, src in self._driver_config.get("files_to_copy", {}).items() + # ] @tasks def files_linked(self): @@ -71,21 +74,21 @@ def files_linked(self): for linkname, target in self._driver_config.get("files_to_link", {}).items() ] - @task - def namelist_file(self): - """ - The namelist file. - """ - path = self._rundir / f"itag.{fcst_time.strftime("%H:%M:%S")}" - yield self._taskname(str(path)) - yield asset(path, path.is_file) - yield None - path.parent.mkdir(parents=True, exist_ok=True) - self._create_user_updated_config( - config_class=NMLConfig, - config_values=d, - path=path, - ) + # @task + # def namelist_file(self): + # """ + # The namelist file. + # """ + # path = self._rundir / f"itag.{fcst_time.strftime("%H:%M:%S")}" + # yield self._taskname(str(path)) + # yield asset(path, path.is_file) + # yield None + # path.parent.mkdir(parents=True, exist_ok=True) + # self._create_user_updated_config( + # config_class=NMLConfig, + # config_values=d, + # path=path, + # ) @tasks def provisioned_run_directory(self): @@ -94,9 +97,9 @@ def provisioned_run_directory(self): """ yield self._taskname("provisioned run directory") yield [ - self.files_copied(), + # self.files_copied(), self.files_linked(), - self.namelist_file(), + # self.namelist_file(), self.runscript(), ] From 114454478407f829ad70ee4091eda60af15d41a6 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 2 May 2024 19:08:12 +0000 Subject: [PATCH 05/65] Add leadtime support to CLI --- src/uwtools/cli.py | 56 ++++++++++++++++++++++++++++-------------- src/uwtools/strings.py | 1 + 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index c3b980c3e..e823aad8c 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -12,7 +12,7 @@ from functools import partial from importlib import import_module from pathlib import Path -from typing import Any, Callable, Dict, List, NoReturn, Tuple +from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple import uwtools.api import uwtools.api.config @@ -642,6 +642,15 @@ def _add_arg_keys(group: Group) -> None: ) +def _add_arg_leadtime(group: Group) -> None: + group.add_argument( + _switch(STR.leadtime), + help="Leadtime in hours", + required=True, + type=int, + ) + + def _add_arg_output_block(group: Group): group.add_argument( _switch(STR.outblock), @@ -810,25 +819,35 @@ def _add_subparser(subparsers: Subparsers, name: str, helpmsg: str) -> Parser: return parser -def _add_subparser_for_driver(name: str, subparsers: Subparsers, with_cycle: bool) -> ModeChecks: +def _add_subparser_for_driver( + name: str, + subparsers: Subparsers, + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, +) -> ModeChecks: """ Subparser for a driver mode. :param name: Name of the driver whose subparser to configure. :param subparsers: Parent parser's subparsers, to add this subparser to. :param with_cycle: Does this driver require a cycle? + :param with_leadtime: Does this driver require a leadtime? """ parser = _add_subparser(subparsers, name, "Execute %s tasks" % name) _basic_setup(parser) subparsers = _add_subparsers(parser, STR.action, STR.task.upper()) return { - task: _add_subparser_for_driver_task(subparsers, task, helpmsg, with_cycle) + task: _add_subparser_for_driver_task(subparsers, task, helpmsg, with_cycle, with_leadtime) for task, helpmsg in import_module("uwtools.api.%s" % name).tasks().items() } def _add_subparser_for_driver_task( - subparsers: Subparsers, task: str, helpmsg: str, with_cycle: bool + subparsers: Subparsers, + task: str, + helpmsg: str, + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, ) -> ActionChecks: """ Subparser for a driver action. @@ -837,11 +856,14 @@ def _add_subparser_for_driver_task( :param task: The task to add a subparser for. :param helpmsg: Help message for task. :param with_cycle: Does this driver require a cycle? + :param with_leadtime: Does this driver require a leadtime? """ parser = _add_subparser(subparsers, task, helpmsg.rstrip(".")) required = parser.add_argument_group(TITLE_REQ_ARG) if with_cycle: _add_arg_cycle(required) + if with_leadtime: + _add_arg_leadtime(required) optional = _basic_setup(parser) _add_arg_config_file(group=optional) _add_arg_batch(optional) @@ -963,22 +985,18 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: STR.template: partial(_add_subparser_template, subparsers), } drivers = { - x: partial(_add_subparser_for_driver, x, subparsers, with_cycle) - for x, with_cycle in [ - (STR.chgrescube, True), - (STR.esggrid, False), - (STR.fv3, True), - (STR.globalequivresol, False), - (STR.jedi, True), - (STR.makehgrid, False), - (STR.mpas, True), - (STR.mpasinit, True), - (STR.sfcclimogen, False), - (STR.ungrib, True), - (STR.upp, True), - ] + x: partial(_add_subparser_for_driver, x, subparsers) + for x in [STR.esggrid, STR.globalequivresol, STR.makehgrid, STR.sfcclimogen] + } + drivers_with_cycle = { + x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True) + for x in [STR.chgrescube, STR.fv3, STR.jedi, STR.mpas, STR.mpasinit, STR.ungrib, STR.upp] + } + drivers_with_cycle_and_leadtime = { + x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True, with_leadtime=True) + for x in [STR.upp] } - modes = {**tools, **drivers} + modes = {**tools, **drivers, **drivers_with_cycle, **drivers_with_cycle_and_leadtime} checks = {k: modes[k]() for k in sorted(modes.keys())} return vars(parser.parse_args(raw_args)), checks diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index a443b29fc..d7eaac5c4 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -86,6 +86,7 @@ class STR: jedi: str = "jedi" keys: str = "keys" keyvalpairs: str = "key_eq_val_pairs" + leadtime: str = "leadtime" link: str = "link" makehgrid: str = "make_hgrid" mode: str = "mode" From 0461b93ace564c64b4b1444be09952abc5d740c0 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 2 May 2024 19:18:34 +0000 Subject: [PATCH 06/65] Handle leadtime in API --- src/uwtools/api/esg_grid.py | 2 +- src/uwtools/api/global_equiv_resol.py | 2 +- src/uwtools/api/make_hgrid.py | 2 +- src/uwtools/api/sfc_climo_gen.py | 2 +- src/uwtools/api/upp.py | 2 +- src/uwtools/utils/api.py | 43 +++++++++++++++++++++++++-- 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/uwtools/api/esg_grid.py b/src/uwtools/api/esg_grid.py index 5ddded361..4485432f3 100644 --- a/src/uwtools/api/esg_grid.py +++ b/src/uwtools/api/esg_grid.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/global_equiv_resol.py b/src/uwtools/api/global_equiv_resol.py index 0c1b394ab..0095c4f6c 100644 --- a/src/uwtools/api/global_equiv_resol.py +++ b/src/uwtools/api/global_equiv_resol.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/make_hgrid.py b/src/uwtools/api/make_hgrid.py index 3fb5ebb16..e241632b1 100644 --- a/src/uwtools/api/make_hgrid.py +++ b/src/uwtools/api/make_hgrid.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/sfc_climo_gen.py b/src/uwtools/api/sfc_climo_gen.py index 6d643554c..628d5e739 100644 --- a/src/uwtools/api/sfc_climo_gen.py +++ b/src/uwtools/api/sfc_climo_gen.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/upp.py b/src/uwtools/api/upp.py index 53dd8f6f9..08b993f78 100644 --- a/src/uwtools/api/upp.py +++ b/src/uwtools/api/upp.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=True) +execute = _make_execute(_Driver, with_cycle=True, with_leadtime=True) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 166f0d70f..487844381 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -32,12 +32,17 @@ def ensure_data_source( return str2path(data_source) -def make_execute(driver_class: type[Driver], with_cycle: bool) -> Callable[..., bool]: +def make_execute( + driver_class: type[Driver], + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, +) -> Callable[..., bool]: """ Returns a function that executes tasks for the given driver. :param driver_class: The driver class whose tasks to execute. - :param with_cycle: Does the driver's constructor take the 'cycle' parameter? + :param with_cycle: Does the driver's constructor take a 'cycle' parameter? + :param with_leadtime: Does the driver's constructor take a 'leadtime' parameter? """ def execute( # pylint: disable=unused-argument @@ -79,12 +84,40 @@ def execute_cycle( # pylint: disable=unused-argument stdin_ok=stdin_ok, ) + def execute_cycle_leadtime( # pylint: disable=unused-argument + task: str, + cycle: dt.datetime, + leadtime: int, + config: Optional[Union[Path, str]] = None, + batch: bool = False, + dry_run: bool = False, + graph_file: Optional[Union[Path, str]] = None, + stdin_ok: bool = False, + ) -> bool: + return _execute( + driver_class=driver_class, + task=task, + cycle=cycle, + leadtime=leadtime, + config=config, + batch=batch, + dry_run=dry_run, + graph_file=graph_file, + stdin_ok=stdin_ok, + ) + + execute_cycle_leadtime.__name__ = "execute" execute_cycle.__name__ = "execute" assert _execute.__doc__ is not None - execute_cycle.__doc__ = re.sub(r"\n *:param driver_class:.*\n", "\n", _execute.__doc__) + execute_cycle_leadtime.__doc__ = re.sub(r"\n *:param driver_class:.*\n", "\n", _execute.__doc__) + execute_cycle.__doc__ = re.sub( + r"\n *:param leadtime:.*\n", "\n", execute_cycle_leadtime.__doc__ + ) execute.__doc__ = re.sub(r"\n *:param cycle:.*\n", "\n", execute_cycle.__doc__) if with_cycle: + if with_leadtime: + return execute_cycle_leadtime return execute_cycle return execute @@ -121,6 +154,7 @@ def _execute( driver_class: type[Driver], task: str, cycle: Optional[dt.datetime] = None, + leadtime: Optional[int] = None, config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, @@ -136,6 +170,7 @@ def _execute( :param driver_class: Class of driver object to instantiate. :param task: The task to execute. :param cycle: The cycle. + :param leadtime: The leadtime. :param config: Path to config file (read stdin if missing or None). :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. @@ -150,6 +185,8 @@ def _execute( ) if cycle: kwargs["cycle"] = cycle + if leadtime: + kwargs["leadtime"] = leadtime obj = driver_class(**kwargs) getattr(obj, task)() if graph_file: From c8437da27f519f3a405b7ff6ac607f0a444e97f4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 2 May 2024 19:23:05 +0000 Subject: [PATCH 07/65] Require cycle when leadtime provided in API --- src/uwtools/utils/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 487844381..4ab063964 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -115,6 +115,9 @@ def execute_cycle_leadtime( # pylint: disable=unused-argument ) execute.__doc__ = re.sub(r"\n *:param cycle:.*\n", "\n", execute_cycle.__doc__) + if with_leadtime and not with_cycle: + raise UWError("When leadtime is specified, cycle is required") + if with_cycle: if with_leadtime: return execute_cycle_leadtime From c4d5e2d48c06d7571fc7ffd54716813a01e30a21 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Thu, 2 May 2024 21:21:01 +0000 Subject: [PATCH 08/65] Add cycle/leadtime check in Driver --- src/uwtools/drivers/driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 3dde13ce0..65f4efe4c 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -16,7 +16,7 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.validator import validate_internal -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.scheduler import JobScheduler from uwtools.utils.processing import execute @@ -33,6 +33,7 @@ def __init__( dry_run: bool = False, batch: bool = False, cycle: Optional[datetime] = None, + leadtime: Optional[int] = None, ) -> None: """ A component driver. @@ -42,6 +43,8 @@ def __init__( :param batch: Run component via the batch system? :param cycle: The cycle. """ + if leadtime and not cycle: + raise UWError("When leadtime is specified, cycle is required") self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch From 234be3fb49a0a7f62d3771beedcd7547afd2c60e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 15:50:41 +0000 Subject: [PATCH 09/65] Render leadtime info configs --- src/uwtools/drivers/driver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 65f4efe4c..2d229c3f1 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -42,6 +42,7 @@ def __init__( :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? :param cycle: The cycle. + :param leadtime: The leadtime. """ if leadtime and not cycle: raise UWError("When leadtime is specified, cycle is required") @@ -50,7 +51,11 @@ def __init__( self._batch = batch self._config.dereference() self._config.dereference( - context={**({"cycle": cycle} if cycle else {}), **self._config.data} + context={ + **({"cycle": cycle} if cycle else {}), + **({"leadtime": leadtime} if leadtime else {}), + **self._config.data, + } ) self._validate() From 14d4c53354d2fd116fc56bba7376be177a60aa9d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 15:52:02 +0000 Subject: [PATCH 10/65] Can we do a single driver-config dereference? --- src/uwtools/drivers/driver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 2d229c3f1..760b97f91 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -49,7 +49,6 @@ def __init__( self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch - self._config.dereference() self._config.dereference( context={ **({"cycle": cycle} if cycle else {}), From fb7814a58c56d3e859a9ad8a1900d1d098dc2f82 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 22:14:48 +0000 Subject: [PATCH 11/65] WIP --- src/uwtools/cli.py | 4 +- src/uwtools/drivers/upp.py | 61 +++++++++---------- .../resources/jsonschema/upp.jsonschema | 24 ++++++++ 3 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 src/uwtools/resources/jsonschema/upp.jsonschema diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 7c3ee5c99..33ccc81f0 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -957,6 +957,8 @@ def _dispatch_to_driver(name: str, args: Args) -> bool: } if cycle := args.get(STR.cycle): kwargs[STR.cycle] = cycle + if leadtime := args.get(STR.leadtime): + kwargs[STR.leadtime] = leadtime return execute(**kwargs) @@ -992,7 +994,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: } drivers_with_cycle = { x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True) - for x in [STR.chgrescube, STR.fv3, STR.jedi, STR.mpas, STR.mpasinit, STR.ungrib, STR.upp] + for x in [STR.chgrescube, STR.fv3, STR.jedi, STR.mpas, STR.mpasinit, STR.ungrib] } drivers_with_cycle_and_leadtime = { x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True, with_leadtime=True) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 0cb5a9f1e..3b61ac137 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -11,7 +11,7 @@ # from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import Driver from uwtools.strings import STR -from uwtools.utils.tasks import file, symlink +from uwtools.utils.tasks import file, filecopy class UPP(Driver): @@ -31,7 +31,7 @@ def __init__( The driver. :param cycle: The cycle. - :param leadtime: The length of the forecast. + :param leadtime: The leadtime. :param config: Path to config file (read stdin if missing or None). :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? @@ -40,40 +40,34 @@ def __init__( if self._dry_run: dryrun() self._cycle = cycle - fcst_time = cycle # datetime.min() + leadtime - valid_time = cycle + leadtime - self._config.dereference( - context={ - **({"cycle": cycle, "fcst_time": fcst_time, "valid_time": valid_time}), - **self._config.data, - } - ) + self._leadtime = leadtime + # fcst_time = cycle # datetime.min() + leadtime + # valid_time = cycle + leadtime # Workflow tasks - # @tasks - # def files_copied(self): - # """ - # Files copied for run. - # """ - # yield self._taskname("files copied") - # if self._config - # yield [ - # filecopy(src=Path(src), dst=self._rundir / dst) - # for dst, src in self._driver_config.get("files_to_copy", {}).items() - # ] - @tasks - def files_linked(self): + def files_copied(self): """ - Files linked for run. + Files copied for run. """ - yield self._taskname("files linked") + yield self._taskname("files copied") yield [ - symlink(target=Path(target), linkname=self._rundir / linkname) - for linkname, target in self._driver_config.get("files_to_link", {}).items() + filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() ] + # @tasks + # def files_linked(self): + # """ + # Files linked for run. + # """ + # yield self._taskname("files linked") + # yield [ + # symlink(target=Path(target), linkname=self._rundir / linkname) + # for linkname, target in self._driver_config.get("files_to_link", {}).items() + # ] + # @task # def namelist_file(self): # """ @@ -97,10 +91,10 @@ def provisioned_run_directory(self): """ yield self._taskname("provisioned run directory") yield [ - # self.files_copied(), - self.files_linked(), + self.files_copied(), + # self.files_linked(), # self.namelist_file(), - self.runscript(), + # self.runscript(), ] @task @@ -143,7 +137,12 @@ def _taskname(self, suffix: str) -> str: :param suffix: Log-string suffix. """ - return "%s %s %s" % (self._cycle.strftime("%Y%m%d %HZ"), self._driver_name, suffix) + return "%s f%03d %s %s" % ( + self._cycle.strftime("%Y%m%d %HZ"), + self._leadtime, + self._driver_name, + suffix, + ) def _ext(n): diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema new file mode 100644 index 000000000..7248e6cd9 --- /dev/null +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -0,0 +1,24 @@ +{ + "properties": { + "upp": { + "additionalProperties": false, + "properties": { + "execution": { + "$ref": "urn:uwtools:execution" + }, + "files_to_copy": { + "$ref": "urn:uwtools:files-to-stage" + }, + "run_dir": { + "type": "string" + } + }, + "required": [ + "execution", + "run_dir" + ], + "type": "object" + } + }, + "type": "object" +} From a1e5a3ef44728d29dfd0b9bf48a40153580ad12e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 22:57:34 +0000 Subject: [PATCH 12/65] WIP --- src/uwtools/drivers/driver.py | 5 +- src/uwtools/drivers/upp.py | 38 +++++----- .../resources/jsonschema/upp.jsonschema | 69 +++++++++++++++++++ 3 files changed, 90 insertions(+), 22 deletions(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 760b97f91..6af6e7fb1 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -6,7 +6,7 @@ import re import stat from abc import ABC, abstractmethod -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from textwrap import dedent from typing import Any, Dict, List, Optional, Type @@ -46,13 +46,14 @@ def __init__( """ if leadtime and not cycle: raise UWError("When leadtime is specified, cycle is required") + assert cycle self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch self._config.dereference( context={ **({"cycle": cycle} if cycle else {}), - **({"leadtime": leadtime} if leadtime else {}), + **({"valid": cycle + timedelta(hours=leadtime)} if leadtime else {}), **self._config.data, } ) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 3b61ac137..ec8b6f2cc 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -2,13 +2,13 @@ A driver for UPP. """ -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path from typing import Optional from iotaa import asset, dryrun, task, tasks -# from uwtools.config.formats.nml import NMLConfig +from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import Driver from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy @@ -22,7 +22,7 @@ class UPP(Driver): def __init__( self, cycle: datetime, - leadtime: timedelta, + leadtime: int, config: Optional[Path] = None, dry_run: bool = False, batch: bool = False, @@ -41,8 +41,6 @@ def __init__( dryrun() self._cycle = cycle self._leadtime = leadtime - # fcst_time = cycle # datetime.min() + leadtime - # valid_time = cycle + leadtime # Workflow tasks @@ -68,21 +66,21 @@ def files_copied(self): # for linkname, target in self._driver_config.get("files_to_link", {}).items() # ] - # @task - # def namelist_file(self): - # """ - # The namelist file. - # """ - # path = self._rundir / f"itag.{fcst_time.strftime("%H:%M:%S")}" - # yield self._taskname(str(path)) - # yield asset(path, path.is_file) - # yield None - # path.parent.mkdir(parents=True, exist_ok=True) - # self._create_user_updated_config( - # config_class=NMLConfig, - # config_values=d, - # path=path, - # ) + @task + def namelist_file(self): + """ + The namelist file. + """ + path = self._rundir / f"itag.f{self._leadtime:03}" + yield self._taskname(str(path)) + yield asset(path, path.is_file) + yield None + path.parent.mkdir(parents=True, exist_ok=True) + self._create_user_updated_config( + config_class=NMLConfig, + config_values=self._driver_config["namelist"], + path=path, + ) @tasks def provisioned_run_directory(self): diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index 7248e6cd9..fc0eb3294 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -9,12 +9,81 @@ "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" }, + "namelist": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { + "base_file": { + "type": "string" + }, + "update_values": { + "properties": { + "model_inputs": { + "additionalProperties": false, + "properties": { + "datestr": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "filenameflux": { + "type": "string" + }, + "grib": { + "type": "string" + }, + "ioform": { + "type": "string" + }, + "modelname": { + "type": "string" + } + }, + "type": "object" + }, + "namepgb": { + "additionalProperties": false, + "properties": { + "kpo": { + "type": "integer" + }, + "numx": { + "type": "integer" + }, + "po": { + "items": { + "type": "number" + }, + "type": "array" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, "run_dir": { "type": "string" } }, "required": [ "execution", + "namelist", "run_dir" ], "type": "object" From 9a8f423bb531270926ee1ce473b0b65c59997d20 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 23:00:09 +0000 Subject: [PATCH 13/65] WIP --- src/uwtools/drivers/upp.py | 26 +++++++++---------- .../resources/jsonschema/upp.jsonschema | 3 +++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index ec8b6f2cc..b070b7f00 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -11,7 +11,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import Driver from uwtools.strings import STR -from uwtools.utils.tasks import file, filecopy +from uwtools.utils.tasks import file, filecopy, symlink class UPP(Driver): @@ -55,16 +55,16 @@ def files_copied(self): for dst, src in self._driver_config.get("files_to_copy", {}).items() ] - # @tasks - # def files_linked(self): - # """ - # Files linked for run. - # """ - # yield self._taskname("files linked") - # yield [ - # symlink(target=Path(target), linkname=self._rundir / linkname) - # for linkname, target in self._driver_config.get("files_to_link", {}).items() - # ] + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self._taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] @task def namelist_file(self): @@ -90,8 +90,8 @@ def provisioned_run_directory(self): yield self._taskname("provisioned run directory") yield [ self.files_copied(), - # self.files_linked(), - # self.namelist_file(), + self.files_linked(), + self.namelist_file(), # self.runscript(), ] diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index fc0eb3294..e52cdfb6c 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -9,6 +9,9 @@ "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" }, + "files_to_link": { + "$ref": "urn:uwtools:files-to-stage" + }, "namelist": { "additionalProperties": false, "anyOf": [ From 8319970a47d1a4d8a73c6ad6c0ee619431d64597 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 3 May 2024 23:02:41 +0000 Subject: [PATCH 14/65] WIP --- src/uwtools/drivers/upp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index b070b7f00..50a960b9d 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -36,7 +36,7 @@ def __init__( :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? """ - super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle) + super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle, leadtime=leadtime) if self._dry_run: dryrun() self._cycle = cycle From 4d32c7bb1ae00ed58bd0f6ef452ecfeb9aac1a4a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 4 May 2024 03:18:47 +0000 Subject: [PATCH 15/65] WIP --- src/uwtools/drivers/upp.py | 46 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 50a960b9d..ef138792c 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -11,7 +11,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import Driver from uwtools.strings import STR -from uwtools.utils.tasks import file, filecopy, symlink +from uwtools.utils.tasks import filecopy, symlink class UPP(Driver): @@ -36,7 +36,13 @@ def __init__( :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? """ - super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle, leadtime=leadtime) + super().__init__( + config=config, + dry_run=dry_run, + batch=batch, + cycle=cycle, + leadtime=leadtime, + ) if self._dry_run: dryrun() self._cycle = cycle @@ -115,19 +121,19 @@ def _driver_name(self) -> str: """ return STR.upp - @task - def _gribfile(self, infile: Path, link: Path): - """ - A symlink to an input GRIB file. + # @task + # def _gribfile(self, infile: Path, link: Path): + # """ + # A symlink to an input GRIB file. - :param link: Link name. - :param infile: File to link. - """ - yield self._taskname(str(link)) - yield asset(link, link.is_symlink) - yield file(path=infile) - link.parent.mkdir(parents=True, exist_ok=True) - link.symlink_to(infile) + # :param link: Link name. + # :param infile: File to link. + # """ + # yield self._taskname(str(link)) + # yield asset(link, link.is_symlink) + # yield file(path=infile) + # link.parent.mkdir(parents=True, exist_ok=True) + # link.symlink_to(infile) def _taskname(self, suffix: str) -> str: """ @@ -143,9 +149,9 @@ def _taskname(self, suffix: str) -> str: ) -def _ext(n): - """ - Maps integers to 3-letter string. - """ - b = 26 - return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] +# def _ext(n): +# """ +# Maps integers to 3-letter string. +# """ +# b = 26 +# return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] From 7b68dab936b447b8f74e5c94e287ea06f3a97326 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 4 May 2024 03:58:50 +0000 Subject: [PATCH 16/65] WIP --- src/uwtools/drivers/driver.py | 11 ++++++----- src/uwtools/drivers/upp.py | 36 +++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 6af6e7fb1..b05133904 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -44,16 +44,17 @@ def __init__( :param cycle: The cycle. :param leadtime: The leadtime. """ - if leadtime and not cycle: - raise UWError("When leadtime is specified, cycle is required") - assert cycle self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch + if leadtime: + if not cycle: + raise UWError("When leadtime is specified, cycle is required") + self._validtime = cycle + timedelta(hours=leadtime) self._config.dereference( context={ **({"cycle": cycle} if cycle else {}), - **({"valid": cycle + timedelta(hours=leadtime)} if leadtime else {}), + **({"validtime": self._validtime} if leadtime else {}), **self._config.data, } ) @@ -262,7 +263,7 @@ def _write_runscript(self, path: Path, envvars: Dict[str, str]) -> None: envvars=envvars, execution=[ "time %s" % self._runcmd, - "test $? -eq 0 && touch %s/done.%s" % (self._rundir, self._driver_name), + "test $? -eq 0 && touch %s.done" % self._runscript_path.name, ], scheduler=self._scheduler if self._batch else None, ) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index ef138792c..971a30b1c 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -77,7 +77,7 @@ def namelist_file(self): """ The namelist file. """ - path = self._rundir / f"itag.f{self._leadtime:03}" + path = self._namelist_path yield self._taskname(str(path)) yield asset(path, path.is_file) yield None @@ -98,7 +98,7 @@ def provisioned_run_directory(self): self.files_copied(), self.files_linked(), self.namelist_file(), - # self.runscript(), + self.runscript(), ] @task @@ -135,6 +135,34 @@ def _driver_name(self) -> str: # link.parent.mkdir(parents=True, exist_ok=True) # link.symlink_to(infile) + @property + def _namelist_path(self) -> Path: + """ + Path to the namelist file. + """ + return self._rundir / f"{self._driver_name}-{self._validtime_str}.nml" + + @property + def _runscript_path(self) -> Path: + """ + Path to the runscript. + """ + return self._rundir / f"runscript.{self._driver_name}-{self._validtime_str}" + + @property + def _runcmd(self) -> str: + """ + Returns the full command-line component invocation. + """ + execution = self._driver_config.get("execution", {}) + mpiargs = execution.get("mpiargs", []) + components = [ + execution.get("mpicmd"), + *[str(x) for x in mpiargs], + "%s <%s" % (execution["executable"], self._namelist_path.name), + ] + return " ".join(filter(None, components)) + def _taskname(self, suffix: str) -> str: """ Returns a common tag for graph-task log messages. @@ -148,6 +176,10 @@ def _taskname(self, suffix: str) -> str: suffix, ) + @property + def _validtime_str(self) -> str: + return self._validtime.strftime("%Y-%m-%dT%H:%M:%S") + # def _ext(n): # """ From e42193a86727dad743e5d0deaf4581c9608ca6ec Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 4 May 2024 04:02:23 +0000 Subject: [PATCH 17/65] Fix unit test --- src/uwtools/drivers/upp.py | 2 +- src/uwtools/tests/drivers/test_driver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 971a30b1c..5c40f5c56 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -159,7 +159,7 @@ def _runcmd(self) -> str: components = [ execution.get("mpicmd"), *[str(x) for x in mpiargs], - "%s <%s" % (execution["executable"], self._namelist_path.name), + "%s < %s" % (execution["executable"], self._namelist_path.name), ] return " ".join(filter(None, components)) diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index fa54f57f0..d5881fff8 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -301,7 +301,7 @@ def test_Driver__write_runscript(driverobj, tmp_path): export BAZ=qux time foo bar baz {executable} - test $? -eq 0 && touch /path/to/2024032218/run/done.concrete + test $? -eq 0 && touch runscript.concrete.done """ with open(path, "r", encoding="utf-8") as f: actual = f.read() From 0d6e413147655d61aff4b33c980868ee6a609ead Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 4 May 2024 04:30:15 +0000 Subject: [PATCH 18/65] Provide default batch-job output path --- src/uwtools/drivers/driver.py | 1 + src/uwtools/tests/drivers/test_driver.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index b05133904..db278d24b 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -172,6 +172,7 @@ def _resources(self) -> Dict[str, Any]: "account": platform["account"], "rundir": self._rundir, "scheduler": platform["scheduler"], + "stdout": "%s.out" % self._runscript_path.name, # config may override **self._driver_config.get("execution", {}).get("batchargs", {}), } diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index d5881fff8..dd0d4249a 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -207,6 +207,7 @@ def test_Driver__resources_pass(driverobj): "account": account, "rundir": driverobj._rundir, "scheduler": scheduler, + "stdout": "runscript.concrete.out", "walltime": walltime, } From 64c5769a373dd038246135e1659dc5f49a579fea Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sat, 4 May 2024 05:06:10 +0000 Subject: [PATCH 19/65] Remove dead code --- src/uwtools/drivers/upp.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 5c40f5c56..e009995e3 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -121,20 +121,6 @@ def _driver_name(self) -> str: """ return STR.upp - # @task - # def _gribfile(self, infile: Path, link: Path): - # """ - # A symlink to an input GRIB file. - - # :param link: Link name. - # :param infile: File to link. - # """ - # yield self._taskname(str(link)) - # yield asset(link, link.is_symlink) - # yield file(path=infile) - # link.parent.mkdir(parents=True, exist_ok=True) - # link.symlink_to(infile) - @property def _namelist_path(self) -> Path: """ @@ -179,11 +165,3 @@ def _taskname(self, suffix: str) -> str: @property def _validtime_str(self) -> str: return self._validtime.strftime("%Y-%m-%dT%H:%M:%S") - - -# def _ext(n): -# """ -# Maps integers to 3-letter string. -# """ -# b = 26 -# return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] From e4938749ad063831799c0c60dcffd6d4edf5a133 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sun, 5 May 2024 15:19:15 +0000 Subject: [PATCH 20/65] Work on schema --- .../resources/jsonschema/upp.jsonschema | 89 ++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index e52cdfb6c..92e2b5c35 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -36,22 +36,38 @@ "additionalProperties": false, "properties": { "datestr": { + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}:[0-9]{2}:[0-9]{2}$", "type": "string" }, "filename": { + "maxlength": 256, + "type": "string" + }, + "filenameflat": { + "maxlength": 256, "type": "string" }, "filenameflux": { + "maxlength": 256, "type": "string" }, "grib": { - "type": "string" + "enum": "grib2" }, "ioform": { - "type": "string" + "enum": [ + "binarynemsio", + "netcdf" + ] }, "modelname": { - "type": "string" + "enum": [ + "FV3R", + "3DRTMA", + "GFS", + "RAPR", + "NMM" + ] } }, "type": "object" @@ -59,9 +75,46 @@ "namepgb": { "additionalProperties": false, "properties": { + "aqf_on": { + "type": "boolean" + }, + "d2d_chem": { + "type": "boolean" + }, + "d3d_on": { + "type": "boolean" + }, + "filenameaer": { + "maxlength": 256, + "type": "string" + }, + "gccpp_on": { + "type": "boolean" + }, + "gocart_on": { + "type": "boolean" + }, + "gtg_on": { + "type": "boolean" + }, + "hyb_sigp": { + "type": "boolean" + }, "kpo": { "type": "integer" }, + "kpv": { + "type": "integer" + }, + "kth": { + "type": "integer" + }, + "method_blsn": { + "type": "boolean" + }, + "nasa_on": { + "type": "boolean" + }, "numx": { "type": "integer" }, @@ -69,7 +122,37 @@ "items": { "type": "number" }, + "maxItems": 70, "type": "array" + }, + "popascal": { + "type": "boolean" + }, + "pv": { + "items": { + "type": "number" + }, + "maxItems": 70, + "type": "array" + }, + "rdaod": { + "type": "boolean" + }, + "slrutah_on": { + "type": "boolean" + }, + "th": { + "items": { + "type": "number" + }, + "maxItems": 70, + "type": "array" + }, + "vtimeunits": { + "enum": "FMIN" + }, + "write_ifi_debug_files": { + "type": "boolean" } }, "type": "object" From 4b87f50fe22e909ae496eb08c8408cb927907b64 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sun, 5 May 2024 15:24:05 +0000 Subject: [PATCH 21/65] Work on schema --- src/uwtools/resources/jsonschema/upp.jsonschema | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index 92e2b5c35..9a3983531 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -52,13 +52,17 @@ "type": "string" }, "grib": { - "enum": "grib2" + "enum": [ + "grib2" + ], + "type": "string" }, "ioform": { "enum": [ "binarynemsio", "netcdf" - ] + ], + "type": "string" }, "modelname": { "enum": [ @@ -67,7 +71,8 @@ "GFS", "RAPR", "NMM" - ] + ], + "type": "string" } }, "type": "object" @@ -149,7 +154,10 @@ "type": "array" }, "vtimeunits": { - "enum": "FMIN" + "enum": [ + "FMIN" + ], + "type": "string" }, "write_ifi_debug_files": { "type": "boolean" From 32ccad9a0699094ce0994eb20fbe9e2439f4ee75 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Sun, 5 May 2024 15:27:19 +0000 Subject: [PATCH 22/65] WIP --- src/uwtools/drivers/upp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index e009995e3..aa548c6ba 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -126,7 +126,7 @@ def _namelist_path(self) -> Path: """ Path to the namelist file. """ - return self._rundir / f"{self._driver_name}-{self._validtime_str}.nml" + return self._rundir / "itag" # f"{self._driver_name}-{self._validtime_str}.nml" @property def _runscript_path(self) -> Path: From 36c1e90cea09affdd244759630afab8f3e54747a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 17:09:04 +0000 Subject: [PATCH 23/65] Simplify namelist and runscript filenames --- src/uwtools/drivers/upp.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index aa548c6ba..0692ff433 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -126,14 +126,14 @@ def _namelist_path(self) -> Path: """ Path to the namelist file. """ - return self._rundir / "itag" # f"{self._driver_name}-{self._validtime_str}.nml" + return self._rundir / "itag" @property def _runscript_path(self) -> Path: """ Path to the runscript. """ - return self._rundir / f"runscript.{self._driver_name}-{self._validtime_str}" + return self._rundir / f"runscript.{self._driver_name}" @property def _runcmd(self) -> str: @@ -161,7 +161,3 @@ def _taskname(self, suffix: str) -> str: self._driver_name, suffix, ) - - @property - def _validtime_str(self) -> str: - return self._validtime.strftime("%Y-%m-%dT%H:%M:%S") From 95ea35017a9044384dda31bf571f0228efb05f1b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 17:27:02 +0000 Subject: [PATCH 24/65] Work on unit tests --- src/uwtools/tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 77c0d3c62..f22904096 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -558,11 +558,13 @@ def test__dispatch_template_translate_no_optional(): def test__dispatch_to_driver(): name = "adriver" cycle = dt.datetime.now() + leadtime = 24 args: dict = { "action": "foo", "batch": True, "config_file": "config.yaml", "cycle": cycle, + "leadtime": leadtime, "dry_run": False, "graph_file": None, "stdin_ok": True, @@ -574,6 +576,7 @@ def test__dispatch_to_driver(): batch=True, config="config.yaml", cycle=cycle, + leadtime=leadtime, dry_run=False, graph_file=None, task="foo", From 5ad1a8035c90dedbde158a3e50109825c991f738 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 17:36:17 +0000 Subject: [PATCH 25/65] Work on unit tests --- src/uwtools/tests/drivers/test_driver.py | 59 ++++++++++++++---------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index dd0d4249a..2691bc580 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -16,7 +16,7 @@ from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers import driver -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.tests.support import regex_logged @@ -57,34 +57,37 @@ def write(path, s): @fixture -def driverobj(tmp_path): - cf = write( - tmp_path / "good.yaml", - { - "concrete": { - "base_file": str(write(tmp_path / "base.yaml", {"a": 11, "b": 22})), - "execution": { - "batchargs": { - "export": "NONE", - "nodes": 1, - "stdout": "{{ concrete.run_dir }}/out", - "walltime": "00:05:00", - }, - "executable": str(tmp_path / "qux"), - "mpiargs": ["bar", "baz"], - "mpicmd": "foo", +def config(tmp_path): + return { + "concrete": { + "base_file": str(write(tmp_path / "base.yaml", {"a": 11, "b": 22})), + "execution": { + "batchargs": { + "export": "NONE", + "nodes": 1, + "stdout": "{{ concrete.run_dir }}/out", + "walltime": "00:05:00", }, - "run_dir": "{{ rootdir }}/{{ cycle.strftime('%Y%m%d%H') }}/run", - "update_values": {"a": 33}, + "executable": str(tmp_path / "qux"), + "mpiargs": ["bar", "baz"], + "mpicmd": "foo", }, - "platform": { - "account": "me", - "scheduler": "slurm", - }, - "rootdir": "/path/to", + "run_dir": "{{ rootdir }}/{{ cycle.strftime('%Y%m%d%H') }}/run", + "update_values": {"a": 33}, + }, + "platform": { + "account": "me", + "scheduler": "slurm", }, + "rootdir": "/path/to", + } + + +@fixture +def driverobj(config): + return ConcreteDriver( + config=config, dry_run=True, batch=True, cycle=datetime(2024, 3, 22, 18), leadtime=24 ) - return ConcreteDriver(config=cf, dry_run=True, batch=True, cycle=datetime(2024, 3, 22, 18)) # Tests @@ -96,6 +99,12 @@ def test_Driver(driverobj): assert driverobj._batch is True +def test_Driver_cycle_leadtime_error(config): + with raises(UWError) as e: + ConcreteDriver(config=config, leadtime=24) + assert "When leadtime is specified, cycle is required" in str(e) + + # Tests for workflow methods From 11bfa73117db108eb65134d98b5cdf5020378120 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 17:45:54 +0000 Subject: [PATCH 26/65] Work on unit tests --- src/uwtools/tests/utils/test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index 0cbe3985b..060f72c4d 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -72,6 +72,28 @@ def test_make_execute_cycle(execute_kwargs): _execute.assert_called_once_with(driver_class=ConcreteDriver, **execute_kwargs) +def test_make_execute_cycle_leadtime(execute_kwargs): + execute_kwargs["cycle"] = dt.now() + execute_kwargs["leadtime"] = 24 + func = api.make_execute(driver_class=ConcreteDriver, with_cycle=True, with_leadtime=True) + assert func.__name__ == "execute" + assert func.__doc__ is not None + assert ":param cycle:" in func.__doc__ + assert ":param leadtime:" in func.__doc__ + assert ":param driver_class:" not in func.__doc__ + assert ":param task:" in func.__doc__ + with patch.object(api, "_execute", return_value=True) as _execute: + assert func(**execute_kwargs) is True + _execute.assert_called_once_with(driver_class=ConcreteDriver, **execute_kwargs) + + +def test_make_execute_leadtime_no_cycle_error(execute_kwargs): + execute_kwargs["leadtime"] = 24 + with raises(UWError) as e: + api.make_execute(driver_class=ConcreteDriver, with_leadtime=True) + assert "When leadtime is specified, cycle is required" in str(e) + + def test_make_tasks(): func = api.make_tasks(driver_class=ConcreteDriver) assert func.__name__ == "tasks" @@ -101,6 +123,7 @@ def test__execute(execute_kwargs, tmp_path): "driver_class": ConcreteDriver, "config": config, "cycle": dt.now(), + "leadtime": 24, "graph_file": graph_file, } assert not graph_file.is_file() From 781911e3965f8e57dfc8bc1ccf64c2279e773e40 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:03:21 +0000 Subject: [PATCH 27/65] Fix schema --- src/uwtools/resources/jsonschema/upp.jsonschema | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index 9a3983531..cf6713150 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -31,6 +31,7 @@ "type": "string" }, "update_values": { + "additionalProperties": false, "properties": { "model_inputs": { "additionalProperties": false, @@ -77,7 +78,7 @@ }, "type": "object" }, - "namepgb": { + "nampgb": { "additionalProperties": false, "properties": { "aqf_on": { From 78414b2838afb0440dc5746332f299099e75651a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:03:39 +0000 Subject: [PATCH 28/65] Work on unit tests --- src/uwtools/tests/drivers/test_upp.py | 195 ++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 src/uwtools/tests/drivers/test_upp.py diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py new file mode 100644 index 000000000..67152088d --- /dev/null +++ b/src/uwtools/tests/drivers/test_upp.py @@ -0,0 +1,195 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +UPP driver tests. +""" +import datetime as dt + +# import f90nml # type: ignore +import yaml +from pytest import fixture + +from uwtools.drivers import upp + +# from unittest.mock import DEFAULT as D +# from unittest.mock import patch + + +# from uwtools.scheduler import Slurm + +# Fixtures + + +@fixture +def config(tmp_path): + return { + "upp": { + "execution": { + "batchargs": { + "cores": 1, + "walltime": "00:01:00", + }, + "executable": str(tmp_path / "upp.exe"), + }, + "files_to_copy": { + "foo": str(tmp_path / "foo"), + "bar": str(tmp_path / "bar"), + }, + "files_to_link": { + "baz": str(tmp_path / "baz"), + "qux": str(tmp_path / "qux"), + }, + "namelist": { + "base_file": str(tmp_path / "base.nml"), + "update_values": { + "model_inputs": { + "grib": "grib2", + }, + "nampgb": { + "kpo": 3, + }, + }, + }, + "run_dir": str(tmp_path), + }, + "platform": { + "account": "me", + "scheduler": "slurm", + }, + } + + +@fixture +def config_file(config, tmp_path): + path = tmp_path / "config.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.dump(config, f) + return path + + +@fixture +def cycle(): + return dt.datetime(2024, 5, 6, 12) + + +@fixture +def driverobj(config_file, cycle, leadtime): + return upp.UPP(config=config_file, cycle=cycle, leadtime=leadtime, batch=True) + + +@fixture +def leadtime(): + return 24 + + +# Driver tests + + +def test_UPP(driverobj): + assert isinstance(driverobj, upp.UPP) + + +# def test_UPP_dry_run(config_file, cycle): +# with patch.object(upp, "dryrun") as dryrun: +# driverobj = upp.UPP(config=config_file, cycle=cycle, batch=True, dry_run=True) +# assert driverobj._dry_run is True +# dryrun.assert_called_once_with() + + +# def test_UPP__gribfile(driverobj): +# src = driverobj._rundir / "GRIBFILE.AAA.in" +# src.touch() +# dst = driverobj._rundir / "GRIBFILE.AAA" +# assert not dst.is_symlink() +# driverobj._gribfile(src, dst) +# assert dst.is_symlink() + + +# def test_UPP_gribfiles(driverobj, tmp_path): +# links = [] +# cycle_hr = 12 +# for n, forecast_hour in enumerate((6, 12, 18)): +# links = [driverobj._rundir / f"GRIBFILE.{upp._ext(n)}"] +# infile = tmp_path / "gfs.t{cycle_hr:02d}z.pgrb2.0p25.f{forecast_hour:03d}".format( +# cycle_hr=cycle_hr, forecast_hour=forecast_hour +# ) +# infile.touch() +# assert not any(link.is_file() for link in links) +# driverobj.gribfiles() +# assert all(link.is_symlink() for link in links) + + +# def test_UPP_namelist_file(driverobj): +# dst = driverobj._rundir / "namelist.wps" +# assert not dst.is_file() +# driverobj.namelist_file() +# assert dst.is_file() +# nml = f90nml.read(dst) +# assert isinstance(nml, f90nml.Namelist) +# assert nml["share"]["interval_seconds"] == 21600 +# assert nml["share"]["end_date"] == "2024-02-02_06:00:00" + + +# def test_UPP_provisioned_run_directory(driverobj): +# with patch.multiple( +# driverobj, +# gribfiles=D, +# namelist_file=D, +# runscript=D, +# vtable=D, +# ) as mocks: +# driverobj.provisioned_run_directory() +# for m in mocks: +# mocks[m].assert_called_once_with() + + +# def test_UPP_run_batch(driverobj): +# with patch.object(driverobj, "_run_via_batch_submission") as func: +# driverobj.run() +# func.assert_called_once_with() + + +# def test_UPP_run_local(driverobj): +# driverobj._batch = False +# with patch.object(driverobj, "_run_via_local_execution") as func: +# driverobj.run() +# func.assert_called_once_with() + + +# def test_UPP_runscript(driverobj): +# with patch.object(driverobj, "_runscript") as runscript: +# driverobj.runscript() +# runscript.assert_called_once() +# args = ("envcmds", "envvars", "execution", "scheduler") +# types = [list, dict, list, Slurm] +# assert [type(runscript.call_args.kwargs[x]) for x in args] == types + + +# def test_UPP_vtable(driverobj): +# src = driverobj._rundir / "Vtable.GFS.in" +# src.touch() +# driverobj._driver_config["vtable"] = src +# dst = driverobj._rundir / "Vtable" +# assert not dst.is_symlink() +# driverobj.vtable() +# assert dst.is_symlink() + + +# def test_UPP__driver_config(driverobj): +# assert driverobj._driver_config == driverobj._config["upp"] + + +# def test_UPP__runscript_path(driverobj): +# assert driverobj._runscript_path == driverobj._rundir / "runscript.upp" + + +# def test_UPP__taskname(driverobj): +# assert driverobj._taskname("foo") == "20240201 18Z upp foo" + + +# def test_UPP__validate(driverobj): +# driverobj._validate() + + +# def test__ext(): +# assert upp._ext(0) == "AAA" +# assert upp._ext(26) == "ABA" From ad00c6404375789eac7565e47b179f17112f428d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:10:26 +0000 Subject: [PATCH 29/65] Work on unit tests --- src/uwtools/tests/drivers/test_chgres_cube.py | 2 +- src/uwtools/tests/drivers/test_esg_grid.py | 4 ++-- src/uwtools/tests/drivers/test_fv3.py | 2 +- .../tests/drivers/test_global_equiv_resol.py | 4 ++-- src/uwtools/tests/drivers/test_jedi.py | 15 ++++++--------- src/uwtools/tests/drivers/test_make_hgrid.py | 4 ++-- src/uwtools/tests/drivers/test_mpas.py | 15 ++++++--------- src/uwtools/tests/drivers/test_mpas_init.py | 15 ++++++--------- src/uwtools/tests/drivers/test_sfc_climo_gen.py | 4 +++- src/uwtools/tests/drivers/test_ungrib.py | 17 +++++++---------- src/uwtools/tests/drivers/test_upp.py | 2 +- 11 files changed, 37 insertions(+), 47 deletions(-) diff --git a/src/uwtools/tests/drivers/test_chgres_cube.py b/src/uwtools/tests/drivers/test_chgres_cube.py index 4d01455f6..2add920fd 100644 --- a/src/uwtools/tests/drivers/test_chgres_cube.py +++ b/src/uwtools/tests/drivers/test_chgres_cube.py @@ -80,7 +80,7 @@ def driverobj(config_file, cycle): return chgres_cube.ChgresCube(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_ChgresCube(driverobj): diff --git a/src/uwtools/tests/drivers/test_esg_grid.py b/src/uwtools/tests/drivers/test_esg_grid.py index 2b9d8e905..017629ef1 100644 --- a/src/uwtools/tests/drivers/test_esg_grid.py +++ b/src/uwtools/tests/drivers/test_esg_grid.py @@ -12,7 +12,7 @@ from uwtools.drivers import esg_grid from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -63,7 +63,7 @@ def driverobj(config_file): return esg_grid.ESGGrid(config=config_file, batch=True) -# Driver tests +# Tests def test_ESGGrid(driverobj): diff --git a/src/uwtools/tests/drivers/test_fv3.py b/src/uwtools/tests/drivers/test_fv3.py index ee44be6d2..4cf361fb0 100644 --- a/src/uwtools/tests/drivers/test_fv3.py +++ b/src/uwtools/tests/drivers/test_fv3.py @@ -84,7 +84,7 @@ def true(): return true -# Driver tests +# Tests def test_FV3(driverobj): diff --git a/src/uwtools/tests/drivers/test_global_equiv_resol.py b/src/uwtools/tests/drivers/test_global_equiv_resol.py index 818174f0e..f6afef1b7 100644 --- a/src/uwtools/tests/drivers/test_global_equiv_resol.py +++ b/src/uwtools/tests/drivers/test_global_equiv_resol.py @@ -12,7 +12,7 @@ from uwtools.drivers import global_equiv_resol from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -49,7 +49,7 @@ def driverobj(config_file): return global_equiv_resol.GlobalEquivResol(config=config_file, batch=True) -# Driver tests +# Tests def test_GlobalEquivResol(driverobj): diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index f1ccb91c3..79d9718c4 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -76,12 +68,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return jedi.JEDI(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_JEDI(driverobj): diff --git a/src/uwtools/tests/drivers/test_make_hgrid.py b/src/uwtools/tests/drivers/test_make_hgrid.py index 1644d3e45..5b63bac25 100644 --- a/src/uwtools/tests/drivers/test_make_hgrid.py +++ b/src/uwtools/tests/drivers/test_make_hgrid.py @@ -11,7 +11,7 @@ from uwtools.drivers import make_hgrid from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -54,7 +54,7 @@ def driverobj(config_file): return make_hgrid.MakeHgrid(config=config_file, batch=True) -# Driver tests +# Tests def test_MakeHgrid(driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas.py b/src/uwtools/tests/drivers/test_mpas.py index 3e88ced0b..88dc08dde 100644 --- a/src/uwtools/tests/drivers/test_mpas.py +++ b/src/uwtools/tests/drivers/test_mpas.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 3, 22, 6) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -77,12 +69,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 3, 22, 6) + + @fixture def driverobj(config_file, cycle): return mpas.MPAS(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_MPAS(driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas_init.py b/src/uwtools/tests/drivers/test_mpas_init.py index ac60028ec..863d1bb64 100644 --- a/src/uwtools/tests/drivers/test_mpas_init.py +++ b/src/uwtools/tests/drivers/test_mpas_init.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -88,12 +80,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return mpas_init.MPASInit(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_MPASInit(driverobj): diff --git a/src/uwtools/tests/drivers/test_sfc_climo_gen.py b/src/uwtools/tests/drivers/test_sfc_climo_gen.py index 3c98f5d5d..b6de46322 100644 --- a/src/uwtools/tests/drivers/test_sfc_climo_gen.py +++ b/src/uwtools/tests/drivers/test_sfc_climo_gen.py @@ -13,6 +13,8 @@ from uwtools.drivers import sfc_climo_gen from uwtools.scheduler import Slurm +# Fixtures + config: dict = { "sfc_climo_gen": { "execution": { @@ -71,7 +73,7 @@ def driverobj(config_file): return sfc_climo_gen.SfcClimoGen(config=config_file, batch=True) -# Driver tests +# Tests def test_SfcClimoGen(driverobj): diff --git a/src/uwtools/tests/drivers/test_ungrib.py b/src/uwtools/tests/drivers/test_ungrib.py index d96270120..9dca1ed1c 100644 --- a/src/uwtools/tests/drivers/test_ungrib.py +++ b/src/uwtools/tests/drivers/test_ungrib.py @@ -16,14 +16,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -45,7 +37,7 @@ def config(tmp_path): "vtable": str(tmp_path / "Vtable.GFS"), }, "platform": { - "account": "wrfruc", + "account": "me", "scheduler": "slurm", }, } @@ -59,12 +51,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return ungrib.Ungrib(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_Ungrib(driverobj): diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index 67152088d..e7e5ce2b5 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -81,7 +81,7 @@ def leadtime(): return 24 -# Driver tests +# Tests def test_UPP(driverobj): From b1bf25c3fb91ff5877c0ec25f60ca8f33b9110bb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:24:06 +0000 Subject: [PATCH 30/65] Work on unit tests --- src/uwtools/tests/drivers/test_upp.py | 65 +++++++++++++-------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index e7e5ce2b5..2acd24140 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -3,6 +3,10 @@ UPP driver tests. """ import datetime as dt +from pathlib import Path + +# from unittest.mock import DEFAULT as D +from unittest.mock import patch # import f90nml # type: ignore import yaml @@ -10,10 +14,6 @@ from uwtools.drivers import upp -# from unittest.mock import DEFAULT as D -# from unittest.mock import patch - - # from uwtools.scheduler import Slurm # Fixtures @@ -49,7 +49,7 @@ def config(tmp_path): }, }, }, - "run_dir": str(tmp_path), + "run_dir": str(tmp_path / "run"), }, "platform": { "account": "me", @@ -88,34 +88,33 @@ def test_UPP(driverobj): assert isinstance(driverobj, upp.UPP) -# def test_UPP_dry_run(config_file, cycle): -# with patch.object(upp, "dryrun") as dryrun: -# driverobj = upp.UPP(config=config_file, cycle=cycle, batch=True, dry_run=True) -# assert driverobj._dry_run is True -# dryrun.assert_called_once_with() - - -# def test_UPP__gribfile(driverobj): -# src = driverobj._rundir / "GRIBFILE.AAA.in" -# src.touch() -# dst = driverobj._rundir / "GRIBFILE.AAA" -# assert not dst.is_symlink() -# driverobj._gribfile(src, dst) -# assert dst.is_symlink() - - -# def test_UPP_gribfiles(driverobj, tmp_path): -# links = [] -# cycle_hr = 12 -# for n, forecast_hour in enumerate((6, 12, 18)): -# links = [driverobj._rundir / f"GRIBFILE.{upp._ext(n)}"] -# infile = tmp_path / "gfs.t{cycle_hr:02d}z.pgrb2.0p25.f{forecast_hour:03d}".format( -# cycle_hr=cycle_hr, forecast_hour=forecast_hour -# ) -# infile.touch() -# assert not any(link.is_file() for link in links) -# driverobj.gribfiles() -# assert all(link.is_symlink() for link in links) +def test_UPP_dry_run(config_file, cycle, leadtime): + with patch.object(upp, "dryrun") as dryrun: + driverobj = upp.UPP( + config=config_file, cycle=cycle, leadtime=leadtime, batch=True, dry_run=True + ) + assert driverobj._dry_run is True + dryrun.assert_called_once_with() + + +def test_UPP_files_copied(driverobj): + for _, src in driverobj._driver_config["files_to_copy"].items(): + Path(src).touch() + for dst, _ in driverobj._driver_config["files_to_copy"].items(): + assert not Path(driverobj._rundir / dst).is_file() + driverobj.files_copied() + for dst, _ in driverobj._driver_config["files_to_copy"].items(): + assert Path(driverobj._rundir / dst).is_file() + + +def test_UPP_files_linked(driverobj): + for _, src in driverobj._driver_config["files_to_link"].items(): + Path(src).touch() + for dst, _ in driverobj._driver_config["files_to_link"].items(): + assert not Path(driverobj._rundir / dst).is_file() + driverobj.files_linked() + for dst, _ in driverobj._driver_config["files_to_link"].items(): + assert Path(driverobj._rundir / dst).is_symlink() # def test_UPP_namelist_file(driverobj): From 2da7da95ea46823762873c6ef60cea084311c2f4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:31:34 +0000 Subject: [PATCH 31/65] Work on unit tests --- src/uwtools/tests/drivers/test_upp.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index 2acd24140..fc64bc948 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -8,7 +8,7 @@ # from unittest.mock import DEFAULT as D from unittest.mock import patch -# import f90nml # type: ignore +import f90nml # type: ignore import yaml from pytest import fixture @@ -117,15 +117,17 @@ def test_UPP_files_linked(driverobj): assert Path(driverobj._rundir / dst).is_symlink() -# def test_UPP_namelist_file(driverobj): -# dst = driverobj._rundir / "namelist.wps" -# assert not dst.is_file() -# driverobj.namelist_file() -# assert dst.is_file() -# nml = f90nml.read(dst) -# assert isinstance(nml, f90nml.Namelist) -# assert nml["share"]["interval_seconds"] == 21600 -# assert nml["share"]["end_date"] == "2024-02-02_06:00:00" +def test_UPP_namelist_file(driverobj): + datestr = "2024-05-05_12:00:00" + with open(driverobj._driver_config["namelist"]["base_file"], "w", encoding="utf-8") as f: + print("&model_inputs datestr='%s' / &nampgb kpv=88 /" % datestr, file=f) + dst = driverobj._rundir / "itag" + assert not dst.is_file() + driverobj.namelist_file() + assert dst.is_file() + nml = f90nml.read(dst) + assert isinstance(nml, f90nml.Namelist) + assert nml["model_inputs"]["datestr"] == datestr # def test_UPP_provisioned_run_directory(driverobj): From eb13878c56e4589b82bd8002af8fcbe35fada45f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:33:24 +0000 Subject: [PATCH 32/65] Work on unit tests --- src/uwtools/tests/drivers/test_upp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index fc64bc948..b71bcd706 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -128,6 +128,9 @@ def test_UPP_namelist_file(driverobj): nml = f90nml.read(dst) assert isinstance(nml, f90nml.Namelist) assert nml["model_inputs"]["datestr"] == datestr + assert nml["model_inputs"]["grib"] == "grib2" + assert nml["nampgb"]["kpo"] == 3 + assert nml["nampgb"]["kpv"] == 88 # def test_UPP_provisioned_run_directory(driverobj): From d1dfa9fb27fbfe56cc2d1933cf490fa5a02a9453 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:37:46 +0000 Subject: [PATCH 33/65] Work on unit tests --- src/uwtools/tests/drivers/test_upp.py | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index b71bcd706..ff8479bbb 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -4,8 +4,7 @@ """ import datetime as dt from pathlib import Path - -# from unittest.mock import DEFAULT as D +from unittest.mock import DEFAULT as D from unittest.mock import patch import f90nml # type: ignore @@ -133,17 +132,17 @@ def test_UPP_namelist_file(driverobj): assert nml["nampgb"]["kpv"] == 88 -# def test_UPP_provisioned_run_directory(driverobj): -# with patch.multiple( -# driverobj, -# gribfiles=D, -# namelist_file=D, -# runscript=D, -# vtable=D, -# ) as mocks: -# driverobj.provisioned_run_directory() -# for m in mocks: -# mocks[m].assert_called_once_with() +def test_UPP_provisioned_run_directory(driverobj): + with patch.multiple( + driverobj, + files_copied=D, + files_linked=D, + namelist_file=D, + runscript=D, + ) as mocks: + driverobj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() # def test_UPP_run_batch(driverobj): @@ -197,3 +196,11 @@ def test_UPP_namelist_file(driverobj): # def test__ext(): # assert upp._ext(0) == "AAA" # assert upp._ext(26) == "ABA" + + +def test_UPP__namelist_path(driverobj): + assert driverobj._namelist_path == driverobj._rundir / "itag" + + +def test_UPP__runscript_path(driverobj): + assert driverobj._runscript_path == driverobj._rundir / "runscript.upp" From 746898bd7d50276ddd88db619cce82218ade45cc Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 18:42:58 +0000 Subject: [PATCH 34/65] Unit tests @ 100% --- src/uwtools/tests/drivers/test_upp.py | 66 ++++++++++----------------- 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index ff8479bbb..7f77d06cb 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -12,8 +12,7 @@ from pytest import fixture from uwtools.drivers import upp - -# from uwtools.scheduler import Slurm +from uwtools.scheduler import Slurm # Fixtures @@ -145,57 +144,38 @@ def test_UPP_provisioned_run_directory(driverobj): mocks[m].assert_called_once_with() -# def test_UPP_run_batch(driverobj): -# with patch.object(driverobj, "_run_via_batch_submission") as func: -# driverobj.run() -# func.assert_called_once_with() - - -# def test_UPP_run_local(driverobj): -# driverobj._batch = False -# with patch.object(driverobj, "_run_via_local_execution") as func: -# driverobj.run() -# func.assert_called_once_with() - - -# def test_UPP_runscript(driverobj): -# with patch.object(driverobj, "_runscript") as runscript: -# driverobj.runscript() -# runscript.assert_called_once() -# args = ("envcmds", "envvars", "execution", "scheduler") -# types = [list, dict, list, Slurm] -# assert [type(runscript.call_args.kwargs[x]) for x in args] == types - - -# def test_UPP_vtable(driverobj): -# src = driverobj._rundir / "Vtable.GFS.in" -# src.touch() -# driverobj._driver_config["vtable"] = src -# dst = driverobj._rundir / "Vtable" -# assert not dst.is_symlink() -# driverobj.vtable() -# assert dst.is_symlink() +def test_UPP_run_batch(driverobj): + with patch.object(driverobj, "_run_via_batch_submission") as func: + driverobj.run() + func.assert_called_once_with() -# def test_UPP__driver_config(driverobj): -# assert driverobj._driver_config == driverobj._config["upp"] +def test_UPP_run_local(driverobj): + driverobj._batch = False + with patch.object(driverobj, "_run_via_local_execution") as func: + driverobj.run() + func.assert_called_once_with() -# def test_UPP__runscript_path(driverobj): -# assert driverobj._runscript_path == driverobj._rundir / "runscript.upp" +def test_UPP_runscript(driverobj): + with patch.object(driverobj, "_runscript") as runscript: + driverobj.runscript() + runscript.assert_called_once() + args = ("envcmds", "envvars", "execution", "scheduler") + types = [list, dict, list, Slurm] + assert [type(runscript.call_args.kwargs[x]) for x in args] == types -# def test_UPP__taskname(driverobj): -# assert driverobj._taskname("foo") == "20240201 18Z upp foo" +def test_UPP__driver_config(driverobj): + assert driverobj._driver_config == driverobj._config["upp"] -# def test_UPP__validate(driverobj): -# driverobj._validate() +def test_UPP__taskname(driverobj): + assert driverobj._taskname("foo") == "20240506 12Z f024 upp foo" -# def test__ext(): -# assert upp._ext(0) == "AAA" -# assert upp._ext(26) == "ABA" +def test_UPP__validate(driverobj): + driverobj._validate() def test_UPP__namelist_path(driverobj): From e09ce4fb1e080626fc11ef043a2b5c7bf5a428af Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 19:07:30 +0000 Subject: [PATCH 35/65] Work on schema tests --- src/uwtools/tests/test_schemas.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 5a2146eda..065a179d8 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -73,6 +73,11 @@ def ungrib_prop(): return partial(schema_validator, "ungrib", "properties", "ungrib", "properties") +@fixture +def upp_prop(): + return partial(schema_validator, "upp", "properties", "upp", "properties") + + # chgres-cube @@ -804,3 +809,49 @@ def test_schema_ungrib_run_dir(ungrib_prop): # Must be a string: assert not errors("/some/path") assert "88 is not of type 'string'" in errors(88) + + +# upp + + +def test_schema_upp(): + config = { + "execution": { + "batchargs": { + "cores": 1, + "walltime": "00:01:00", + }, + "executable": "/path/to/upp.exe", + }, + "namelist": { + "base_file": "/path/to/base.nml", + "update_values": { + "model_inputs": { + "grib": "grib2", + }, + "nampgb": { + "kpo": 3, + }, + }, + }, + "run_dir": "/path/to/run", + } + errors = schema_validator("upp", "properties", "upp") + # Basic correctness: + assert not errors(config) + # Some top-level keys are required: + for key in ("execution", "namelist", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Other top-level keys are optional: + assert not errors({**config, "files_to_copy": {"dst": "src"}}) + assert not errors({**config, "files_to_link": {"dst": "src"}}) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) + + +# def test_schema_upp_namelist +def test_schema_upp_run_dir(upp_prop): + errors = upp_prop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) From eb19b062d9ddd63083abd480583bc49f36fc409e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 19:12:58 +0000 Subject: [PATCH 36/65] Update docs/shared/upp.yaml --- docs/shared/upp.yaml | 53 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 843621491..ee6f66864 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -1,34 +1,41 @@ upp: execution: - executable: upp.x - use_crtm: false - crtm_fix: /path/to/crtm/files + batchargs: + nodes: 1 + export: NONE + walltime: 00:05:00 + envcmds: + - module use /path/to/modulefiles + - module load runtime-module + - source /etc/profile.d/slurm.sh + executable: /path/to/upp.x + mpiargs: + - "--ntasks $SLURM_CPUS_ON_NODE" + mpicmd: srun files_to_copy: - eta_micro_lookup.dat: /path/to/lookup.dat - postxconfig-NT.txt: /path/to/post/config + postxconfig-NT.txt: /path/to/postxconfig-NT.txt + files_to_link: + eta_micro_lookup.dat: /path/to/nam_micro_lookup.dat params_grib2_tbl_new: /path/to/params_grib2_tbl_new - crtm_files: - - Nalli.IRwater.EmisCoeff.bin - - FAST*.bin - - NPOESS.IRland.EmisCoeff.bin - - NPOESS.IRsnow.EmisCoeff.bin - - NPOESS.IRice.EmisCoeff.bin - - AerosolCoeff.bin - - CloudCoeff.bin - - '*.SpcCoeff.bin' - - '*.TauCoeff.bin' - namelist: # named itag; taken as input so arbitrary - base_file: + namelist: update_values: model_inputs: - datestr: "{{ valid_time.strftime('%Y-%m-%d_%H::%M:%S') }}" - filename: /path/to/forecast/dynf{{ fcst_time.strftime("%H:%M:%S") }}.nc - filenameflat: postxconfig-NT.txt - filenameflux: /path/to/forecast/phyf{{ fcst_time.strftime("%H:%M:%S") }}.nc + datestr: "{{ validtime.strftime('%Y-%m-%d_%H:%M:%S') }}" + filename: /path/to/dynf{{ user.leadtime }}.nc + filenameflux: /path/to/phyf{{ user.leadtime }}.nc grib: grib2 ioform: netcdf modelname: FV3R nampgb: - kpo: 47 - po: [1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,] + kpo: 3 numx: 1 + po: + - 1000. + - 100. + - 1. + run_dir: /path/to/run +platform: + account: me + scheduler: slurm +user: + leadtime: "{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }}" From 194d1a0ea4fd8ad8d79f66bad6678d5e1cc5c64a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 20:05:55 +0000 Subject: [PATCH 37/65] Work on schema tests --- .../resources/jsonschema/upp.jsonschema | 8 +- src/uwtools/tests/test_schemas.py | 80 ++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema index cf6713150..ad66caf83 100644 --- a/src/uwtools/resources/jsonschema/upp.jsonschema +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -41,15 +41,15 @@ "type": "string" }, "filename": { - "maxlength": 256, + "maxLength": 256, "type": "string" }, "filenameflat": { - "maxlength": 256, + "maxLength": 256, "type": "string" }, "filenameflux": { - "maxlength": 256, + "maxLength": 256, "type": "string" }, "grib": { @@ -91,7 +91,7 @@ "type": "boolean" }, "filenameaer": { - "maxlength": 256, + "maxLength": 256, "type": "string" }, "gccpp_on": { diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 065a179d8..8a06002ac 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -849,7 +849,85 @@ def test_schema_upp(): assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) -# def test_schema_upp_namelist +def test_schema_upp_namelist(upp_prop): + maxpathlen = 256 + errors = upp_prop("namelist") + # At least one of base_file or update_values is required: + assert "is not valid" in errors({}) + # Just base_file is ok: + assert not errors({"base_file": "/path/to/base.nml"}) + # Just update_values is ok: + assert not errors({"update_values": {"model_inputs": {"grib": "grib2"}}}) + # Both base_file and update_values are ok: + assert not errors( + {"base_file": "/path/to/base.nml", "update_values": {"model_inputs": {"grib": "grib2"}}} + ) + # model_inputs: datestr requires a specific format: + assert not errors({"update_values": {"model_inputs": {"datestr": "2024-05-06_12:00:00"}}}) + assert "does not match" in errors( + {"update_values": {"model_inputs": {"datestr": "2024-05-06T12:00:00"}}} + ) + # model_inputs: String pathnames have a max length: + for key in ["filename", "filenameflat", "filenameflux"]: + assert not errors({"update_values": {"model_inputs": {key: "c" * maxpathlen}}}) + assert "too long" in errors( + {"update_values": {"model_inputs": {key: "c" * (maxpathlen + 1)}}} + ) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {key: 88}}}) + # model_inputs: Only one grib value is supported: + assert not errors({"update_values": {"model_inputs": {"grib": "grib2"}}}) + assert "not one of ['grib2']" in errors({"update_values": {"model_inputs": {"grib": "grib1"}}}) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {"grib": 88}}}) + # model_inputs: Only certain ioform values are supported: + assert "not one of ['binarynemsio', 'netcdf']" in errors( + {"update_values": {"model_inputs": {"ioform": "jpg"}}} + ) + # model_inputs: Only certain modelname values are supported: + assert "not one of ['FV3R', '3DRTMA', 'GFS', 'RAPR', 'NMM']" in errors( + {"update_values": {"model_inputs": {"modelname": "foo"}}} + ) + # model_inputs: No other keys are supported: + assert "Additional properties are not allowed" in errors( + {"update_values": {"model_inputs": {"something": "else"}}} + ) + # nampgb: Some boolean keys are supported: + for key in [ + "aqf_on", + "d2d_chem", + "gccpp_on", + "gocart_on", + "gtg_on", + "hyb_sigp", + "method_blsn", + "nasa_on", + "popascal", + "rdaod", + "slrutah_on", + "write_ifi_debug_files", + ]: + assert not errors({"update_values": {"nampgb": {key: True}}}) + assert "not of type 'boolean'" in errors({"update_values": {"nampgb": {key: 88}}}) + # nampgb: String pathnames have a max length: + for key in ["filenameaer"]: + assert not errors({"update_values": {"nampgb": {key: "c" * maxpathlen}}}) + assert "too long" in errors({"update_values": {"nampgb": {key: "c" * (maxpathlen + 1)}}}) + assert "not of type 'string'" in errors({"update_values": {"nampgb": {key: 88}}}) + # nampgb: Some integer keys are supported: + for key in ["kpo", "kpv", "kth", "numx"]: + assert not errors({"update_values": {"nampgb": {key: 88}}}) + assert "not of type 'integer'" in errors({"update_values": {"nampgb": {key: True}}}) + # nampgb: Some arrays of numbers are supported: + nitems = 70 + for key in ["po", "pv", "th"]: + assert not errors({"update_values": {"nampgb": {key: [3.14] * nitems}}}) + assert "too long" in errors({"update_values": {"nampgb": {key: [3.14] * (nitems + 1)}}}) + assert "not of type 'number'" in errors( + {"update_values": {"nampgb": {key: [True] * nitems}}} + ) + # nampgb: Only one vtimeunits value is supported: + assert "not one of ['FMIN']" in errors({"update_values": {"nampgb": {"vtimeunits": "FOO"}}}) + + def test_schema_upp_run_dir(upp_prop): errors = upp_prop("run_dir") # Must be a string: From f0d808531fe4e2590ca8bb40bdc613f909842d20 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 20:37:07 +0000 Subject: [PATCH 38/65] Work on docs --- docs/index.rst | 36 +++--- docs/sections/user_guide/api/chgres_cube.rst | 2 +- docs/sections/user_guide/api/esg_grid.rst | 2 +- docs/sections/user_guide/api/fv3.rst | 2 +- docs/sections/user_guide/api/index.rst | 1 + docs/sections/user_guide/api/jedi.rst | 2 +- docs/sections/user_guide/api/mpas.rst | 2 +- docs/sections/user_guide/api/mpas_init.rst | 2 +- docs/sections/user_guide/api/ungrib.rst | 4 +- docs/sections/user_guide/api/upp.rst | 5 + .../sections/user_guide/cli/drivers/index.rst | 1 + docs/sections/user_guide/cli/drivers/upp.rst | 106 ++++++++++++++++++ .../user_guide/yaml/components/index.rst | 1 + .../user_guide/yaml/components/jedi.rst | 2 +- .../user_guide/yaml/components/ungrib.rst | 2 +- .../user_guide/yaml/components/upp.rst | 39 +++++++ 16 files changed, 183 insertions(+), 26 deletions(-) create mode 100644 docs/sections/user_guide/api/upp.rst create mode 100644 docs/sections/user_guide/cli/drivers/upp.rst create mode 100644 docs/sections/user_guide/yaml/components/upp.rst diff --git a/docs/index.rst b/docs/index.rst index 370ed0ed8..9a3a63a70 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -110,49 +110,55 @@ Drivers for UFS To prepare a complete forecast, drivers would typically be run in the order shown here (along with additional drivers still in development). esg_grid -""""""""""" +"""""""" | **CLI**: ``uw esg_grid -h`` -| **API**: ``import uwtools.api.drivers.esg_grid`` +| **API**: ``import uwtools.api.esg_grid`` global_equiv_resol """""""""""""""""" | **CLI**: ``uw global_equiv_resol -h`` -| **API**: ``import uwtools.api.drivers.global_equiv_resol`` +| **API**: ``import uwtools.api.global_equiv_resol`` make_hgrid """""""""" | **CLI**: ``uw make_hgrid -h`` -| **API**: ``import uwtools.api.drivers.make_hgrid`` +| **API**: ``import uwtools.api.make_hgrid`` sfc_climo_gen """"""""""""" | **CLI**: ``uw sfc_climo_gen -h`` -| **API**: ``import uwtools.api.drivers.sfc_climo_gen`` +| **API**: ``import uwtools.api.sfc_climo_gen`` chgres_cube """"""""""" | **CLI**: ``uw chgres_cube -h`` -| **API**: ``import uwtools.api.drivers.chgres_cube`` +| **API**: ``import uwtools.api.chgres_cube`` FV3 """ | **CLI**: ``uw fv3 -h`` -| **API**: ``import uwtools.api.drivers.fv3`` +| **API**: ``import uwtools.api.fv3`` -JEDI -"""" +UPP +""" -| **CLI**: ``uw jedi -h`` -| **API**: ``import uwtools.api.drivers.jedi`` +| **CLI**: ``uw upp -h`` +| **API**: ``import uwtools.api.upp`` +Driver for JEDI +^^^^^^^^^^^^^^^ +| **CLI**: ``uw jedi -h`` +| **API**: ``import uwtools.api.jedi`` +JEDI +"""" Drivers for MPAS ^^^^^^^^^^^^^^^^ @@ -163,22 +169,20 @@ ungrib """""" | **CLI**: ``uw ungrib -h`` -| **API**: ``import uwtools.api.drivers.ungrib`` +| **API**: ``import uwtools.api.ungrib`` mpas_init """"""""" | **CLI**: ``uw mpas_init -h`` -| **API**: ``import uwtools.api.drivers.mpas_init`` +| **API**: ``import uwtools.api.mpas_init`` mpas """" | **CLI**: ``uw mpas -h`` -| **API**: ``import uwtools.api.drivers.mpas`` - - +| **API**: ``import uwtools.api.mpas`` ------------------ diff --git a/docs/sections/user_guide/api/chgres_cube.rst b/docs/sections/user_guide/api/chgres_cube.rst index 4d2faf746..b36a673e4 100644 --- a/docs/sections/user_guide/api/chgres_cube.rst +++ b/docs/sections/user_guide/api/chgres_cube.rst @@ -1,5 +1,5 @@ ``uwtools.api.chgres_cube`` -============================= +=========================== .. automodule:: uwtools.api.chgres_cube :members: diff --git a/docs/sections/user_guide/api/esg_grid.rst b/docs/sections/user_guide/api/esg_grid.rst index 7205ec941..c706d377e 100644 --- a/docs/sections/user_guide/api/esg_grid.rst +++ b/docs/sections/user_guide/api/esg_grid.rst @@ -1,5 +1,5 @@ ``uwtools.api.esg_grid`` -============================= +======================== .. automodule:: uwtools.api.esg_grid :members: diff --git a/docs/sections/user_guide/api/fv3.rst b/docs/sections/user_guide/api/fv3.rst index 4a5e70954..3e447a20b 100644 --- a/docs/sections/user_guide/api/fv3.rst +++ b/docs/sections/user_guide/api/fv3.rst @@ -1,5 +1,5 @@ ``uwtools.api.fv3`` -======================== +=================== .. automodule:: uwtools.api.fv3 :members: diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 02e19276d..37d6b9f7e 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -17,3 +17,4 @@ API sfc_climo_gen template ungrib + upp diff --git a/docs/sections/user_guide/api/jedi.rst b/docs/sections/user_guide/api/jedi.rst index f0eb7acce..a08cc8467 100644 --- a/docs/sections/user_guide/api/jedi.rst +++ b/docs/sections/user_guide/api/jedi.rst @@ -1,5 +1,5 @@ ``uwtools.api.jedi`` -========================== +==================== .. automodule:: uwtools.api.jedi :members: diff --git a/docs/sections/user_guide/api/mpas.rst b/docs/sections/user_guide/api/mpas.rst index 6cda7b22b..032b76029 100644 --- a/docs/sections/user_guide/api/mpas.rst +++ b/docs/sections/user_guide/api/mpas.rst @@ -1,5 +1,5 @@ ``uwtools.api.mpas`` -============================= +==================== .. automodule:: uwtools.api.mpas :members: diff --git a/docs/sections/user_guide/api/mpas_init.rst b/docs/sections/user_guide/api/mpas_init.rst index 0eb2a87bc..d20095e73 100644 --- a/docs/sections/user_guide/api/mpas_init.rst +++ b/docs/sections/user_guide/api/mpas_init.rst @@ -1,5 +1,5 @@ ``uwtools.api.mpas_init`` -============================= +========================= .. automodule:: uwtools.api.mpas_init :members: diff --git a/docs/sections/user_guide/api/ungrib.rst b/docs/sections/user_guide/api/ungrib.rst index 6db41352b..e149acaf2 100644 --- a/docs/sections/user_guide/api/ungrib.rst +++ b/docs/sections/user_guide/api/ungrib.rst @@ -1,5 +1,5 @@ ``uwtools.api.ungrib`` -============================= +====================== .. automodule:: uwtools.api.ungrib - :members: \ No newline at end of file + :members: diff --git a/docs/sections/user_guide/api/upp.rst b/docs/sections/user_guide/api/upp.rst new file mode 100644 index 000000000..1130fcd27 --- /dev/null +++ b/docs/sections/user_guide/api/upp.rst @@ -0,0 +1,5 @@ +``uwtools.api.upp`` +=================== + +.. automodule:: uwtools.api.upp + :members: diff --git a/docs/sections/user_guide/cli/drivers/index.rst b/docs/sections/user_guide/cli/drivers/index.rst index 9ee2861ce..08fc8acf8 100644 --- a/docs/sections/user_guide/cli/drivers/index.rst +++ b/docs/sections/user_guide/cli/drivers/index.rst @@ -14,3 +14,4 @@ Drivers mpas_init sfc_climo_gen ungrib + upp diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp.rst new file mode 100644 index 000000000..371ff4c1f --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/upp.rst @@ -0,0 +1,106 @@ +``upp`` +======= + +The ``uw`` mode for configuring and running the `UPP `_ component. + +.. code-block:: text + + $ uw upp --help + usage: uw upp [-h] [--version] TASK ... + + Execute upp tasks + + Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + + Positional arguments: + TASK + files_copied + Files copied for run + files_linked + Files linked for run + namelist_file + The namelist file + provisioned_run_directory + Run directory provisioned with all required content + run + A run + runscript + The runscript + validate + Validate the UW driver config + +All tasks take the same arguments. For example: + +.. code-block:: text + + $ uw upp run --help + usage: uw upp run --cycle CYCLE --leadtime LEADTIME [-h] [--version] [--config-file PATH] + [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] + + A run + + Required arguments: + --cycle CYCLE + The cycle in ISO8601 format + --leadtime LEADTIME + Leadtime in hours + + Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with content similar to: + +.. highlight:: yaml +.. literalinclude:: ../../../../shared/upp.yaml + +Its contents are described in depth in section :ref:`upp_yaml`. + +* Run ``upp`` on an interactive node + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 + + The driver creates a ``runscript.upp`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``upp``. + +* Run ``upp`` via a batch job + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch + + The driver creates a ``runscript.upp`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``upp:``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch --dry-run + +* The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``upp`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file: + + .. code-block:: text + + $ uw upp provisioned_run_directory --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index a07dccc3b..a9dfb49da 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -14,3 +14,4 @@ UW YAML for Components mpas_init sfc_climo_gen ungrib + upp diff --git a/docs/sections/user_guide/yaml/components/jedi.rst b/docs/sections/user_guide/yaml/components/jedi.rst index 5408445e6..06d804c98 100644 --- a/docs/sections/user_guide/yaml/components/jedi.rst +++ b/docs/sections/user_guide/yaml/components/jedi.rst @@ -11,7 +11,7 @@ Here is a prototype UW YAML jedi: block, explained in detail below: .. literalinclude:: ../../../../shared/jedi.yaml UW YAML for the ``jedi:`` Block ------------------------------------- +------------------------------- execution: ^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/ungrib.rst b/docs/sections/user_guide/yaml/components/ungrib.rst index a2b7f3d88..ef3f60289 100644 --- a/docs/sections/user_guide/yaml/components/ungrib.rst +++ b/docs/sections/user_guide/yaml/components/ungrib.rst @@ -11,7 +11,7 @@ Here is a prototype UW YAML ``ungrib:`` block, explained in detail below: .. literalinclude:: ../../../../shared/ungrib.yaml UW YAML for the ``ungrib:`` Block ----------------------------------------- +--------------------------------- execution: ^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst new file mode 100644 index 000000000..87b8e6e32 --- /dev/null +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -0,0 +1,39 @@ +.. _upp_yaml: + +upp +=== + +Structured YAML to run the UPP post-processor is validated by JSON Schema and requires the ``upp:`` block, described below. If UPP is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. + +Here is a prototype UW YAML ``upp:`` block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: ../../../../shared/upp.yaml + +UW YAML for the ``upp:`` Block +------------------------------ + +execution: +^^^^^^^^^^ + +See :ref:`here ` for details. + +files_to_copy: +^^^^^^^^^^^^^^ + +TODO + +files_to_link: +"""""""""""""" + +TODO + +namelist_file: +"""""""""""""" + +TODO + +run_dir: +^^^^^^^^ + +The path to the UPP run directory. From 51a5654b8d13e6ba60cc628a12012a71facc332d Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 21:03:51 +0000 Subject: [PATCH 39/65] Work on docs --- .../user_guide/yaml/components/chgres_cube.rst | 2 ++ docs/sections/user_guide/yaml/components/fv3.rst | 16 +++------------- .../sections/user_guide/yaml/components/jedi.rst | 2 ++ .../sections/user_guide/yaml/components/mpas.rst | 2 ++ .../user_guide/yaml/components/mpas_init.rst | 2 ++ .../user_guide/yaml/components/ungrib.rst | 2 ++ docs/sections/user_guide/yaml/components/upp.rst | 13 +++++++++++-- docs/shared/injected_cycle.rst | 13 +++++++++++++ docs/shared/injected_leadtime.rst | 15 +++++++++++++++ 9 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 docs/shared/injected_cycle.rst create mode 100644 docs/shared/injected_leadtime.rst diff --git a/docs/sections/user_guide/yaml/components/chgres_cube.rst b/docs/sections/user_guide/yaml/components/chgres_cube.rst index f857e66d1..7a0bb68b3 100644 --- a/docs/sections/user_guide/yaml/components/chgres_cube.rst +++ b/docs/sections/user_guide/yaml/components/chgres_cube.rst @@ -5,6 +5,8 @@ chgres_cube Structured YAML to run :ufs-utils:`chgres_cube` is validated by JSON Schema and requires the ``chgres_cube:`` block, described below. If ``chgres_cube`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``chgres_cube:`` block, explained in detail below: .. highlight:: yaml diff --git a/docs/sections/user_guide/yaml/components/fv3.rst b/docs/sections/user_guide/yaml/components/fv3.rst index 164497a57..5627a9d25 100644 --- a/docs/sections/user_guide/yaml/components/fv3.rst +++ b/docs/sections/user_guide/yaml/components/fv3.rst @@ -5,6 +5,8 @@ fv3 Structured YAML to run FV3 is validated by JSON Schema and requires the ``fv3:`` block, described below. If FV3 is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. The configuration files required by the UFS Weather Model are documented :weather-model-io:`here`. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``fv3:`` block, explained in detail below: .. highlight:: yaml @@ -36,19 +38,7 @@ The path to a :weather-model-io:`valid field-table file` to be files_to_copy: ^^^^^^^^^^^^^^ -See :ref:`this page ` for details. For the ``fv3`` driver, both keys and values may contain Jinja2 variables/expressions using a ``cycle`` variable, which is a Python ``datetime`` object corresponding to the FV3 cycle being run. This supports specification of cycle-specific filenames/paths. For example, a key-value pair - -.. code-block:: yaml - - gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc: /some/path/{{ cycle.strftime('%Y%m%d')}}/{{ cycle.strftime('%H') }}/gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc - -would be rendered as - -.. code-block:: yaml - - gfs.t18z.atmanl.nc: /some/path/20240212/18/gfs.t18z.atmanl.nc - -for the ``2024-02-12T18`` cycle. +See :ref:`this page ` for details. files_to_link: ^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/jedi.rst b/docs/sections/user_guide/yaml/components/jedi.rst index 06d804c98..0a806faf1 100644 --- a/docs/sections/user_guide/yaml/components/jedi.rst +++ b/docs/sections/user_guide/yaml/components/jedi.rst @@ -5,6 +5,8 @@ jedi Structured YAML to run JEDI is validated by JSON Schema and requires the ``jedi:`` block, described below. If ``jedi`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML jedi: block, explained in detail below: .. highlight:: yaml diff --git a/docs/sections/user_guide/yaml/components/mpas.rst b/docs/sections/user_guide/yaml/components/mpas.rst index 2431c1824..b2a2a22d0 100644 --- a/docs/sections/user_guide/yaml/components/mpas.rst +++ b/docs/sections/user_guide/yaml/components/mpas.rst @@ -5,6 +5,8 @@ mpas Structured YAML to run MPAS is validated by JSON Schema and requires the ``mpas:`` block, described below. If ``mpas`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``mpas:`` block, explained in detail below: .. highlight:: yaml diff --git a/docs/sections/user_guide/yaml/components/mpas_init.rst b/docs/sections/user_guide/yaml/components/mpas_init.rst index b2f324f04..2ba34a322 100644 --- a/docs/sections/user_guide/yaml/components/mpas_init.rst +++ b/docs/sections/user_guide/yaml/components/mpas_init.rst @@ -5,6 +5,8 @@ mpas_init Structured YAML to run MPAS Init is validated by JSON Schema and requires the ``mpas_init:`` block, described below. If ``mpas_init`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``mpas_init:`` block, explained in detail below: .. highlight:: yaml diff --git a/docs/sections/user_guide/yaml/components/ungrib.rst b/docs/sections/user_guide/yaml/components/ungrib.rst index ef3f60289..7084369ca 100644 --- a/docs/sections/user_guide/yaml/components/ungrib.rst +++ b/docs/sections/user_guide/yaml/components/ungrib.rst @@ -5,6 +5,8 @@ ungrib Structured YAML to run the WRF preprocessing component ``ungrib`` is validated by JSON Schema and requires the ``ungrib:`` block, described below. If ``ungrib`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``ungrib:`` block, explained in detail below: .. highlight:: yaml diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index 87b8e6e32..22124f5d4 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -5,6 +5,9 @@ upp Structured YAML to run the UPP post-processor is validated by JSON Schema and requires the ``upp:`` block, described below. If UPP is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst +.. include:: ../../../../shared/injected_leadtime.rst + Here is a prototype UW YAML ``upp:`` block, explained in detail below: .. highlight:: yaml @@ -21,12 +24,18 @@ See :ref:`here ` for details. files_to_copy: ^^^^^^^^^^^^^^ -TODO +See :ref:`this page ` for details. + +.. code-block:: text + + upp: + files_to_copy: + f1: /path/to/f1 files_to_link: """""""""""""" -TODO +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. namelist_file: """""""""""""" diff --git a/docs/shared/injected_cycle.rst b/docs/shared/injected_cycle.rst new file mode 100644 index 000000000..5d979e823 --- /dev/null +++ b/docs/shared/injected_cycle.rst @@ -0,0 +1,13 @@ +* This driver receives a ``cycle`` argument, which it makes available as a Python ``datetime`` to Jinja2 when realizing its input config. This supports specification of cycle-specific values. For example, the key-value pair + + .. code-block:: yaml + + gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc: /some/path/{{ cycle.strftime('%Y%m%d%H') }}/gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc + + would be rendered as + + .. code-block:: yaml + + gfs.t18z.atmanl.nc: /some/path/2024021218/gfs.t18z.atmanl.nc + + for cycle ``2024-02-12T18``. diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_leadtime.rst new file mode 100644 index 000000000..e624576ce --- /dev/null +++ b/docs/shared/injected_leadtime.rst @@ -0,0 +1,15 @@ +* This driver receives a ``leadtime`` argument, from which it computes a ``validtime`` Python ``datetime`` that it makes available to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the values + + .. code-block:: yaml + + validstr: "{{ validtime.strftime('%Y-%m-%d %H:%M') }}" + suffix: f{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }} + + would be rendered as + + .. code-block:: yaml + + validstr: 2024-02-12 18:00 + suffix: f018 + + for cycle ``2024-02-12T18`` and leadtime ``18``. From d6dcd5b7e6c74da9bfc8e73a1f1793f44a2d18a3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 21:07:37 +0000 Subject: [PATCH 40/65] Work on docs --- docs/sections/user_guide/yaml/components/chgres_cube.rst | 2 +- docs/sections/user_guide/yaml/components/esg_grid.rst | 2 +- docs/sections/user_guide/yaml/components/fv3.rst | 2 +- .../user_guide/yaml/components/global_equiv_resol.rst | 2 +- docs/sections/user_guide/yaml/components/jedi.rst | 8 +------- docs/sections/user_guide/yaml/components/make_hgrid.rst | 4 ++-- docs/sections/user_guide/yaml/components/mpas.rst | 2 +- docs/sections/user_guide/yaml/components/mpas_init.rst | 2 +- .../sections/user_guide/yaml/components/sfc_climo_gen.rst | 2 +- docs/sections/user_guide/yaml/components/ungrib.rst | 2 +- docs/sections/user_guide/yaml/components/upp.rst | 8 +------- 11 files changed, 12 insertions(+), 24 deletions(-) diff --git a/docs/sections/user_guide/yaml/components/chgres_cube.rst b/docs/sections/user_guide/yaml/components/chgres_cube.rst index 7a0bb68b3..dde09414e 100644 --- a/docs/sections/user_guide/yaml/components/chgres_cube.rst +++ b/docs/sections/user_guide/yaml/components/chgres_cube.rst @@ -18,7 +18,7 @@ UW YAML for the ``chgres_cube:`` Block execution ^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/esg_grid.rst b/docs/sections/user_guide/yaml/components/esg_grid.rst index 29b3106d6..3ce8738c1 100644 --- a/docs/sections/user_guide/yaml/components/esg_grid.rst +++ b/docs/sections/user_guide/yaml/components/esg_grid.rst @@ -16,7 +16,7 @@ UW YAML for the ``esg_grid:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist: ^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/fv3.rst b/docs/sections/user_guide/yaml/components/fv3.rst index 5627a9d25..0aa7c0701 100644 --- a/docs/sections/user_guide/yaml/components/fv3.rst +++ b/docs/sections/user_guide/yaml/components/fv3.rst @@ -28,7 +28,7 @@ Accepted values are ``global`` and ``regional``. execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. field_table: ^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst index 5754ba73c..3da9e70bc 100644 --- a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst +++ b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst @@ -18,7 +18,7 @@ UW YAML for the ``global_equiv_resol:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. input_grid_file: ^^^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/jedi.rst b/docs/sections/user_guide/yaml/components/jedi.rst index 0a806faf1..f12175e7b 100644 --- a/docs/sections/user_guide/yaml/components/jedi.rst +++ b/docs/sections/user_guide/yaml/components/jedi.rst @@ -18,7 +18,7 @@ UW YAML for the ``jedi:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. configuration_file: ^^^^^^^^^^^^^^^^^^^ @@ -29,12 +29,6 @@ files_to_copy: See :ref:`this page ` for details. -.. code-block:: text - - jedi: - files_to_copy: - f1: /path/to/f1 - files_to_link: ^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/make_hgrid.rst b/docs/sections/user_guide/yaml/components/make_hgrid.rst index ef791b67e..42430c132 100644 --- a/docs/sections/user_guide/yaml/components/make_hgrid.rst +++ b/docs/sections/user_guide/yaml/components/make_hgrid.rst @@ -19,7 +19,7 @@ UW YAML for the ``make_hgrid:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. run_dir: ^^^^^^^^ @@ -239,4 +239,4 @@ Specify boundaries for defining zonal regions of varying resolution. When ``trip ybnds: """""" -Specify boundaries for defining meridional regions of varying resolution. \ No newline at end of file +Specify boundaries for defining meridional regions of varying resolution. diff --git a/docs/sections/user_guide/yaml/components/mpas.rst b/docs/sections/user_guide/yaml/components/mpas.rst index b2a2a22d0..9093e392f 100644 --- a/docs/sections/user_guide/yaml/components/mpas.rst +++ b/docs/sections/user_guide/yaml/components/mpas.rst @@ -20,7 +20,7 @@ UW YAML for the ``mpas:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/mpas_init.rst b/docs/sections/user_guide/yaml/components/mpas_init.rst index 2ba34a322..b7dce112d 100644 --- a/docs/sections/user_guide/yaml/components/mpas_init.rst +++ b/docs/sections/user_guide/yaml/components/mpas_init.rst @@ -20,7 +20,7 @@ UW YAML for the ``mpas_init:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst index 373324b1f..af4aebf62 100644 --- a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst +++ b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst @@ -16,7 +16,7 @@ UW YAML for the ``sfc_climo_gen:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist: ^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/ungrib.rst b/docs/sections/user_guide/yaml/components/ungrib.rst index 7084369ca..201bc237b 100644 --- a/docs/sections/user_guide/yaml/components/ungrib.rst +++ b/docs/sections/user_guide/yaml/components/ungrib.rst @@ -18,7 +18,7 @@ UW YAML for the ``ungrib:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. gfs_files: ^^^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index 22124f5d4..d74736107 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -19,19 +19,13 @@ UW YAML for the ``upp:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. files_to_copy: ^^^^^^^^^^^^^^ See :ref:`this page ` for details. -.. code-block:: text - - upp: - files_to_copy: - f1: /path/to/f1 - files_to_link: """""""""""""" From 1a43a1df14f1e04da108df1a1b281e159b6be607 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 21:37:50 +0000 Subject: [PATCH 41/65] Work on docs --- .../user_guide/yaml/components/make_hgrid.rst | 3 --- .../user_guide/yaml/components/upp.rst | 18 +++++++++++++++++- docs/static/custom.css | 6 ++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/sections/user_guide/yaml/components/make_hgrid.rst b/docs/sections/user_guide/yaml/components/make_hgrid.rst index 42430c132..ef9a41b58 100644 --- a/docs/sections/user_guide/yaml/components/make_hgrid.rst +++ b/docs/sections/user_guide/yaml/components/make_hgrid.rst @@ -31,7 +31,6 @@ config: Describes the required parameters to run a ``make_hgrid`` configuration. - grid_type: """""""""" @@ -62,8 +61,6 @@ grid_type: * - ``tripolar_grid`` - ``nxbnds``, ``nybnds``, ``xbnds``, ``ybnds`` must be specified to define the grid bounds. - - angular_midpoint: """"""""""""""""" diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index d74736107..218dc570a 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -34,7 +34,23 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th namelist_file: """""""""""""" -TODO +Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_values` for details). + +The following namelists and variables can be customized: + +.. list-table:: + :widths: 10 95 + :header-rows: 1 + + * - Namelist + - Variables + * - ``model_inputs`` + - ``datestr``, ``filename``, ``filenameflat``, ``filenameflux``, ``grib``, ``ioform``, ``modelname`` + * - ``nampgb`` + - ``aqf_on``, ``d2d_chem``, ``d3d_on``, ``filenameaer``, ``gccpp_on``, ``gocart_on``, ``gtg_on``, ``hyb_sigp``, ``kpo``, ``kpv``, ``kth``, ``method_blsn``, ``nasa_on``, ``numx``, ``po``, ``popascal``, ``pv``, ``rdaod``, ``slrutah_on``, ``th``, ``vtimeunits``, ``write_ifi_debug_files`` + +Read more on the UPP namelists, including variable meanings and appropriate values, `here `_. + run_dir: ^^^^^^^^ diff --git a/docs/static/custom.css b/docs/static/custom.css index 859fe191f..cb22e007e 100644 --- a/docs/static/custom.css +++ b/docs/static/custom.css @@ -15,3 +15,9 @@ html.writer-html5 .rst-content table.docutils td>p { .wy-table-responsive { overflow-x: hidden; } + +/* Let table content word-wrap. */ + +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: inherit; +} From 597eae2aa4957027808c857a5d566bb320df6534 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 21:38:13 +0000 Subject: [PATCH 42/65] Work on docs --- docs/sections/user_guide/cli/drivers/jedi.rst | 12 ++++++------ docs/sections/user_guide/cli/drivers/upp.rst | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index 7308e6300..eec11150a 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -7,15 +7,15 @@ The ``uw`` mode for configuring and running the JEDI framework. $ uw jedi -help usage: uw jedi [-h] [--version] TASK ... - + Execute JEDI tasks - + Optional arguments: -h, --help Show help and exit --version Show version info and exit - + Positional arguments: TASK configuration_file @@ -42,13 +42,13 @@ All tasks take the same arguments. For example: $ uw jedi run -help usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] - + A run - + Required arguments: --cycle CYCLE The cycle in ISO8601 format - + Optional arguments: -h, --help Show help and exit diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp.rst index 371ff4c1f..7ed8996c1 100644 --- a/docs/sections/user_guide/cli/drivers/upp.rst +++ b/docs/sections/user_guide/cli/drivers/upp.rst @@ -7,15 +7,15 @@ The ``uw`` mode for configuring and running the `UPP Date: Mon, 6 May 2024 21:47:31 +0000 Subject: [PATCH 43/65] Harmonize --help switches in CLI examples --- docs/sections/user_guide/cli/drivers/global_equiv_resol.rst | 4 ++-- docs/sections/user_guide/cli/drivers/jedi.rst | 4 ++-- docs/sections/user_guide/cli/drivers/make_hgrid.rst | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst index 7e2f71d6b..98746de42 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst @@ -5,7 +5,7 @@ The ``uw`` mode for configuring and running the UFS Utils preprocessing componen .. code-block:: text - $ uw global_equiv_resol -h + $ uw global_equiv_resol --help usage: uw global_equiv_resol [-h] [--version] TASK ... Execute global_equiv_resol tasks @@ -33,7 +33,7 @@ All tasks take the same arguments. For example: .. code-block:: text - $ uw global_equiv_resol run -h + $ uw global_equiv_resol run --help usage: uw global_equiv_resol run --config-file PATH [-h] [--version] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index eec11150a..15086d0de 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -5,7 +5,7 @@ The ``uw`` mode for configuring and running the JEDI framework. .. code-block:: text - $ uw jedi -help + $ uw jedi --help usage: uw jedi [-h] [--version] TASK ... Execute JEDI tasks @@ -39,7 +39,7 @@ All tasks take the same arguments. For example: .. code-block:: text - $ uw jedi run -help + $ uw jedi run --help usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid.rst b/docs/sections/user_guide/cli/drivers/make_hgrid.rst index 36b4d29ae..2bdb11adc 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid.rst +++ b/docs/sections/user_guide/cli/drivers/make_hgrid.rst @@ -5,7 +5,7 @@ The ``uw`` mode for configuring and running the UFS Utils preprocessing componen .. code-block:: text - $ uw make_hgrid -h + $ uw make_hgrid --help usage: uw make_hgrid [-h] [--version] TASK ... Execute make_hgrid tasks @@ -31,7 +31,7 @@ All tasks take the same arguments. For example: .. code-block:: text - $ uw make_hgrid run -h + $ uw make_hgrid run --help usage: uw make_hgrid run --config-file PATH [-h] [--version] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] @@ -92,4 +92,4 @@ Its contents are described in section :ref:`make_hgrid_yaml`. .. code-block:: text - $ uw make_hgrid run --config-file config.yaml --batch --dry-run \ No newline at end of file + $ uw make_hgrid run --config-file config.yaml --batch --dry-run From 1e031f3e6b780be50cf8157ea7cd470c7777d6cd Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 22:03:09 +0000 Subject: [PATCH 44/65] Fix PR template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5a90bf43f..7ef7fee1e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,7 +21,7 @@ INSTRUCTIONS - [ ] Bug fix (corrects a known issue) - [ ] Code maintenance (refactoring, etc. without behavior change) - [ ] Documentation -- [ ] Enhancement (adds a new functionality) +- [ ] Enhancement (adds new functionality) - [ ] Tooling (CI, code-quality, packaging, revision-control, etc.) **Impact** From d8ba52a790fd056705e93acd083bfdef3cc067c5 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 22:15:59 +0000 Subject: [PATCH 45/65] injected_leadtime -> injected_validtime --- docs/sections/user_guide/yaml/components/upp.rst | 2 +- docs/shared/{injected_leadtime.rst => injected_validtime.rst} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/shared/{injected_leadtime.rst => injected_validtime.rst} (100%) diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index 218dc570a..b0753554d 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -6,7 +6,7 @@ upp Structured YAML to run the UPP post-processor is validated by JSON Schema and requires the ``upp:`` block, described below. If UPP is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. .. include:: ../../../../shared/injected_cycle.rst -.. include:: ../../../../shared/injected_leadtime.rst +.. include:: ../../../../shared/injected_validtime.rst Here is a prototype UW YAML ``upp:`` block, explained in detail below: diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_validtime.rst similarity index 100% rename from docs/shared/injected_leadtime.rst rename to docs/shared/injected_validtime.rst From f1a74a645e41ffd0114e9617d00f3c0fbbc88d6e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 22:42:55 +0000 Subject: [PATCH 46/65] More generalization related to leadtime --- src/uwtools/tests/api/test_drivers.py | 27 +++++++++++++++++++++++---- src/uwtools/tests/utils/test_api.py | 8 ++++++-- src/uwtools/utils/api.py | 2 ++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index 2ff55b2cd..de6877645 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -17,12 +17,26 @@ mpas_init, sfc_climo_gen, ungrib, + upp, ) from uwtools.drivers import support from uwtools.utils import api -modules = [chgres_cube, esg_grid, fv3, jedi, make_hgrid, mpas, mpas_init, sfc_climo_gen, ungrib] -nocycle = [esg_grid, global_equiv_resol, make_hgrid, sfc_climo_gen] +modules = [ + chgres_cube, + esg_grid, + fv3, + global_equiv_resol, + jedi, + make_hgrid, + mpas, + mpas_init, + sfc_climo_gen, + ungrib, + upp, +] +with_cycle = [chgres_cube, fv3, jedi, mpas, mpas_init, ungrib, upp] +with_leadtime = [upp] @pytest.mark.parametrize("module", modules) @@ -35,12 +49,17 @@ def test_api_execute(module): "stdin_ok": True, "task": "foo", } - kwargs = kwbase if module in nocycle else {"cycle": dt.now(), **kwbase} + kwargs = { + **kwbase, + **({"cycle": dt.now()} if module in with_cycle else {}), + **({"leadtime": 24} if module in with_leadtime else {}), + } with patch.object(api, "_execute") as _execute: module.execute(**kwargs) _execute.assert_called_once_with( driver_class=module._Driver, - cycle=None if module in nocycle else kwargs["cycle"], + cycle=kwargs["cycle"] if module in with_cycle else None, + leadtime=kwargs["leadtime"] if module in with_leadtime else None, **kwbase ) diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index 060f72c4d..e901ffd41 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -56,7 +56,9 @@ def test_make_execute(execute_kwargs): assert ":param task:" in func.__doc__ with patch.object(api, "_execute", return_value=True) as _execute: assert func(**execute_kwargs) is True - _execute.assert_called_once_with(driver_class=ConcreteDriver, cycle=None, **execute_kwargs) + _execute.assert_called_once_with( + driver_class=ConcreteDriver, cycle=None, leadtime=None, **execute_kwargs + ) def test_make_execute_cycle(execute_kwargs): @@ -69,7 +71,9 @@ def test_make_execute_cycle(execute_kwargs): assert ":param task:" in func.__doc__ with patch.object(api, "_execute", return_value=True) as _execute: assert func(**execute_kwargs) is True - _execute.assert_called_once_with(driver_class=ConcreteDriver, **execute_kwargs) + _execute.assert_called_once_with( + driver_class=ConcreteDriver, leadtime=None, **execute_kwargs + ) def test_make_execute_cycle_leadtime(execute_kwargs): diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 4ab063964..8d03b75e2 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -57,6 +57,7 @@ def execute( # pylint: disable=unused-argument driver_class=driver_class, task=task, cycle=None, + leadtime=None, config=config, batch=batch, dry_run=dry_run, @@ -76,6 +77,7 @@ def execute_cycle( # pylint: disable=unused-argument return _execute( driver_class=driver_class, task=task, + leadtime=None, cycle=cycle, config=config, batch=batch, From 65599330f631dcfa042f694f6550ec6bb54b94d1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Mon, 6 May 2024 23:08:57 +0000 Subject: [PATCH 47/65] Update --- src/uwtools/cli.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 33ccc81f0..a8a3931b6 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -989,16 +989,32 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: STR.template: partial(_add_subparser_template, subparsers), } drivers = { - x: partial(_add_subparser_for_driver, x, subparsers) - for x in [STR.esggrid, STR.globalequivresol, STR.makehgrid, STR.sfcclimogen] + component: partial(_add_subparser_for_driver, component, subparsers) + for component in [ + STR.esggrid, + STR.globalequivresol, + STR.makehgrid, + STR.sfcclimogen, + ] } drivers_with_cycle = { - x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True) - for x in [STR.chgrescube, STR.fv3, STR.jedi, STR.mpas, STR.mpasinit, STR.ungrib] + component: partial(_add_subparser_for_driver, component, subparsers, with_cycle=True) + for component in [ + STR.chgrescube, + STR.fv3, + STR.jedi, + STR.mpas, + STR.mpasinit, + STR.ungrib, + ] } drivers_with_cycle_and_leadtime = { - x: partial(_add_subparser_for_driver, x, subparsers, with_cycle=True, with_leadtime=True) - for x in [STR.upp] + component: partial( + _add_subparser_for_driver, component, subparsers, with_cycle=True, with_leadtime=True + ) + for component in [ + STR.upp, + ] } modes = {**tools, **drivers, **drivers_with_cycle, **drivers_with_cycle_and_leadtime} checks = {k: modes[k]() for k in sorted(modes.keys())} From c0297df40ef16a19a423625100312a820933c9ee Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 16:58:35 +0000 Subject: [PATCH 48/65] Formatting --- src/uwtools/tests/api/test_drivers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index ac27f05c6..3742694ef 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -40,6 +40,7 @@ with_cycle = [chgres_cube, fv3, jedi, mpas, mpas_init, ungrib, upp] with_leadtime = [upp] + @pytest.mark.parametrize("module", modules) def test_api_execute(module): kwbase = { From 15d26bbb57183efb8dcdb6d082543176e2203f36 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 16:59:56 +0000 Subject: [PATCH 49/65] Doc update --- docs/sections/user_guide/cli/drivers/make_hgrid.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid.rst b/docs/sections/user_guide/cli/drivers/make_hgrid.rst index c3367ca61..7421ae62d 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid.rst +++ b/docs/sections/user_guide/cli/drivers/make_hgrid.rst @@ -32,18 +32,18 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw make_hgrid run --help - usage: uw make_hgrid run --config-file PATH [-h] [--version] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] - + usage: uw make_hgrid run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--quiet] [--verbose] + A run - + Optional arguments: - --config-file PATH, -c PATH - Path to config file -h, --help Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run @@ -53,7 +53,6 @@ All tasks take the same arguments. For example: --quiet, -q Print no logging messages --verbose, -v - Print all logging messages Examples ^^^^^^^^ From e1aa6ee141c9bcc03212bf94ba8171700c28918b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 17:00:47 +0000 Subject: [PATCH 50/65] Doc whitespace fixes --- docs/index.rst | 1 - docs/sections/user_guide/cli/drivers/chgres_cube.rst | 2 -- docs/sections/user_guide/cli/drivers/global_equiv_resol.rst | 1 - docs/sections/user_guide/cli/drivers/jedi.rst | 1 - docs/sections/user_guide/cli/drivers/make_hgrid.rst | 4 ++-- docs/sections/user_guide/cli/drivers/mpas.rst | 2 -- docs/sections/user_guide/cli/drivers/mpas_init.rst | 1 - docs/sections/user_guide/cli/drivers/ungrib.rst | 1 - docs/sections/user_guide/yaml/components/make_hgrid.rst | 2 -- docs/sections/user_guide/yaml/components/shave.rst | 2 -- docs/sections/user_guide/yaml/components/upp.rst | 1 - docs/sections/user_guide/yaml/tags.rst | 3 --- 12 files changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c681f7bf1..72ce6c15c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -177,7 +177,6 @@ ungrib | **CLI**: ``uw ungrib -h`` | **API**: ``import uwtools.api.ungrib`` - mpas_init """"""""" diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube.rst index 592fb8905..b52219188 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube.rst @@ -69,10 +69,8 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/chgres_cube.yaml - Its contents are described in depth in section :ref:`chgres_cube_yaml`. Each of the values in the ``chgres_cube`` YAML may contain Jinja2 variables/expressions using a ``cycle`` variable, which is a Python ``datetime`` object corresponding to the FV3 cycle being run. - * Run ``chgres_cube`` on an interactive node .. code-block:: text diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst index 98746de42..fb9fc79b2 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst @@ -67,7 +67,6 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/global_equiv_resol.yaml - Its contents are described in section :ref:`global_equiv_resol_yaml`. * Run ``global_equiv_resol`` on an interactive node diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index 15086d0de..7d469d006 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -75,7 +75,6 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/jedi.yaml - Its contents are described in section :ref:`jedi_yaml`. * Run ``jedi`` on an interactive node diff --git a/docs/sections/user_guide/cli/drivers/make_hgrid.rst b/docs/sections/user_guide/cli/drivers/make_hgrid.rst index 7421ae62d..d35512865 100644 --- a/docs/sections/user_guide/cli/drivers/make_hgrid.rst +++ b/docs/sections/user_guide/cli/drivers/make_hgrid.rst @@ -34,9 +34,9 @@ All tasks take the same arguments. For example: $ uw make_hgrid run --help usage: uw make_hgrid run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] - + A run - + Optional arguments: -h, --help Show help and exit diff --git a/docs/sections/user_guide/cli/drivers/mpas.rst b/docs/sections/user_guide/cli/drivers/mpas.rst index f482cd669..3085c49e8 100644 --- a/docs/sections/user_guide/cli/drivers/mpas.rst +++ b/docs/sections/user_guide/cli/drivers/mpas.rst @@ -69,7 +69,6 @@ All tasks take the same arguments. For example: --verbose, -v Print all logging messages - Examples ^^^^^^^^ @@ -78,7 +77,6 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/mpas.yaml - Its contents are described in depth in section :ref:`mpas_yaml`. * Run MPAS on an interactive node diff --git a/docs/sections/user_guide/cli/drivers/mpas_init.rst b/docs/sections/user_guide/cli/drivers/mpas_init.rst index 450e8fcba..0c59cc71f 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init.rst +++ b/docs/sections/user_guide/cli/drivers/mpas_init.rst @@ -77,7 +77,6 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/mpas_init.yaml - Its contents are described in depth in section :ref:`mpas_init_yaml`. * Run ``init_atmosphere`` on an interactive node diff --git a/docs/sections/user_guide/cli/drivers/ungrib.rst b/docs/sections/user_guide/cli/drivers/ungrib.rst index 5beed79ba..15fe36148 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib.rst +++ b/docs/sections/user_guide/cli/drivers/ungrib.rst @@ -73,7 +73,6 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/ungrib.yaml - Its contents are described in depth in section :ref:`ungrib_yaml`. * Run ``ungrib`` on an interactive node diff --git a/docs/sections/user_guide/yaml/components/make_hgrid.rst b/docs/sections/user_guide/yaml/components/make_hgrid.rst index ef9a41b58..ff059ed16 100644 --- a/docs/sections/user_guide/yaml/components/make_hgrid.rst +++ b/docs/sections/user_guide/yaml/components/make_hgrid.rst @@ -156,13 +156,11 @@ nlon: Number of model grid points(supergrid) for each zonal regions of varying resolution. - nxbnds: """"""" Specify number of zonal regions for varying resolution. - nybnds: """"""" diff --git a/docs/sections/user_guide/yaml/components/shave.rst b/docs/sections/user_guide/yaml/components/shave.rst index 25f7aa385..c7f07b910 100644 --- a/docs/sections/user_guide/yaml/components/shave.rst +++ b/docs/sections/user_guide/yaml/components/shave.rst @@ -33,13 +33,11 @@ nh4: The number of halo rows/columns. - nx: """ The i/x dimensions of the compute domain (not including halo). - ny: """ diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index b0753554d..c4d327d19 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -51,7 +51,6 @@ The following namelists and variables can be customized: Read more on the UPP namelists, including variable meanings and appropriate values, `here `_. - run_dir: ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 1e77b6d1b..248a07eb2 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -37,7 +37,6 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input % uw config realize --input-file input.yaml --output-format yaml f2: 5.859 - ``!int`` ^^^^^^^^ @@ -56,7 +55,6 @@ Converts the tagged node to a Python ``int`` value. For example, given ``input.y f2: 11 f2: 140 - ``!include`` ^^^^^^^^^^^^ @@ -80,7 +78,6 @@ and ``supplemental.yaml``: e: 2.718 pi: 3.141 - ``!remove`` ^^^^^^^^^^^ From d2aa937a9301b1eee626f6d2fd2f79e78e836240 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 17:04:06 +0000 Subject: [PATCH 51/65] Doc fix --- docs/sections/user_guide/cli/drivers/shave.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/sections/user_guide/cli/drivers/shave.rst b/docs/sections/user_guide/cli/drivers/shave.rst index 5b2e37535..7104506dc 100644 --- a/docs/sections/user_guide/cli/drivers/shave.rst +++ b/docs/sections/user_guide/cli/drivers/shave.rst @@ -33,7 +33,7 @@ All tasks take the same arguments. For example: $ uw shave run --help usage: uw shave run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + [--graph-file PATH] [--quiet] [--verbose] A run @@ -42,6 +42,8 @@ All tasks take the same arguments. For example: Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run From a5e13a7577f84a2591d499b3072833be6c7d6b8e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 17:11:24 +0000 Subject: [PATCH 52/65] Update CLI doc examples --- .../user_guide/cli/drivers/esg_grid.rst | 47 +++++++++---------- .../cli/drivers/global_equiv_resol.rst | 8 ++-- docs/sections/user_guide/cli/drivers/jedi.rst | 4 +- docs/sections/user_guide/cli/drivers/mpas.rst | 8 ++-- .../user_guide/cli/drivers/mpas_init.rst | 2 +- .../user_guide/cli/drivers/sfc_climo_gen.rst | 8 ++-- .../user_guide/cli/drivers/ungrib.rst | 4 +- 7 files changed, 38 insertions(+), 43 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/esg_grid.rst b/docs/sections/user_guide/cli/drivers/esg_grid.rst index e9cfb8104..05c697194 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid.rst +++ b/docs/sections/user_guide/cli/drivers/esg_grid.rst @@ -33,30 +33,29 @@ All tasks take the same arguments. For example: .. code-block:: text - $ uw esg_grid run --help - usage: uw esg_grid run --config-file PATH [-h] [--version] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] - - A run - - Required arguments: - --config-file PATH, -c PATH - Path to UW YAML config file - - Optional arguments: - -h, --help - Show help and exit - --version - Show version info and exit - --batch - Submit run to batch scheduler - --dry-run - Only log info, making no changes - --graph-file PATH - Path to Graphviz DOT output [experimental] - --quiet, -q - Print no logging messages - --verbose, -v - Print all logging messages + $ uw esg_grid run --help + usage: uw esg_grid run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--quiet] [--verbose] + + A run + + Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages Examples ^^^^^^^^ diff --git a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst index fb9fc79b2..c04f4b964 100644 --- a/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst +++ b/docs/sections/user_guide/cli/drivers/global_equiv_resol.rst @@ -34,20 +34,18 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw global_equiv_resol run --help - usage: uw global_equiv_resol run --config-file PATH [-h] [--version] [--batch] [--dry-run] + usage: uw global_equiv_resol run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] A run - Required arguments: - --config-file PATH, -c PATH - Path to config file - Optional arguments: -h, --help Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index 7d469d006..8d32ef5dd 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -41,7 +41,7 @@ All tasks take the same arguments. For example: $ uw jedi run --help usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + [--graph-file PATH] [--quiet] [--verbose] A run @@ -55,7 +55,7 @@ All tasks take the same arguments. For example: --version Show version info and exit --config-file PATH, -c PATH - Path to config file (default: read from stdin) + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run diff --git a/docs/sections/user_guide/cli/drivers/mpas.rst b/docs/sections/user_guide/cli/drivers/mpas.rst index 3085c49e8..d5b570c39 100644 --- a/docs/sections/user_guide/cli/drivers/mpas.rst +++ b/docs/sections/user_guide/cli/drivers/mpas.rst @@ -42,14 +42,12 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw mpas run --help - usage: uw mpas run --config-file PATH --cycle CYCLE [-h] [--version] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + usage: uw mpas run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--quiet] [--verbose] A run Required arguments: - --config-file PATH, -c PATH - Path to UW YAML config file --cycle CYCLE The cycle in ISO8601 format @@ -58,6 +56,8 @@ All tasks take the same arguments. For example: Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run diff --git a/docs/sections/user_guide/cli/drivers/mpas_init.rst b/docs/sections/user_guide/cli/drivers/mpas_init.rst index 0c59cc71f..17b56d425 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init.rst +++ b/docs/sections/user_guide/cli/drivers/mpas_init.rst @@ -43,7 +43,7 @@ All tasks take the same arguments. For example: $ uw mpas_init run --help usage: uw mpas_init run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + [--graph-file PATH] [--quiet] [--verbose] A run diff --git a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst b/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst index 28cc875a5..946821745 100644 --- a/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst +++ b/docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst @@ -34,20 +34,18 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw sfc_climo_gen run --help - usage: uw sfc_climo_gen run --config-file PATH [-h] [--version] [--batch] [--dry-run] + usage: uw sfc_climo_gen run [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] A run - Required arguments: - --config-file PATH, -c PATH - Path to UW YAML config file - Optional arguments: -h, --help Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run diff --git a/docs/sections/user_guide/cli/drivers/ungrib.rst b/docs/sections/user_guide/cli/drivers/ungrib.rst index 15fe36148..c11e50137 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib.rst +++ b/docs/sections/user_guide/cli/drivers/ungrib.rst @@ -38,8 +38,8 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw ungrib run --help - usage: uw ungrib run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] - [--quiet] [--verbose] + usage: uw ungrib run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] + [--graph-file PATH] [--quiet] [--verbose] A run From 440ce1f418e3bf1c74db68bc6ed69cd769fb8282 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 17:21:23 +0000 Subject: [PATCH 53/65] Update CLI examples in docs --- .../user_guide/cli/drivers/chgres_cube.rst | 8 +-- .../user_guide/cli/drivers/esg_grid.rst | 44 +++++++------- docs/sections/user_guide/cli/drivers/fv3.rst | 2 +- docs/sections/user_guide/cli/drivers/jedi.rst | 58 +++++++++---------- docs/sections/user_guide/cli/drivers/mpas.rst | 4 +- .../user_guide/cli/drivers/mpas_init.rst | 4 +- .../user_guide/cli/drivers/ungrib.rst | 6 +- 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube.rst index b52219188..ed5e7a88e 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube.rst @@ -34,14 +34,12 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw chgres_cube run --help - usage: uw chgres_cube run --config-file PATH --cycle CYCLE [-h] [--version] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + usage: uw chgres_cube run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] + [--dry-run] [--graph-file PATH] [--quiet] [--verbose] A run Required arguments: - --config-file PATH, -c PATH - Path to UW YAML config file --cycle CYCLE The cycle in ISO8601 format @@ -50,6 +48,8 @@ All tasks take the same arguments. For example: Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run diff --git a/docs/sections/user_guide/cli/drivers/esg_grid.rst b/docs/sections/user_guide/cli/drivers/esg_grid.rst index 05c697194..a40590285 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid.rst +++ b/docs/sections/user_guide/cli/drivers/esg_grid.rst @@ -6,28 +6,28 @@ The ``uw`` mode for configuring and running the :ufs-utils:`regional_esg_grid Date: Tue, 7 May 2024 17:27:58 +0000 Subject: [PATCH 54/65] Run yaml-sort --- docs/sections/user_guide/api/shave.rst | 2 +- docs/shared/shave.yaml | 10 +++++----- docs/shared/upp.yaml | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/sections/user_guide/api/shave.rst b/docs/sections/user_guide/api/shave.rst index e189ae222..395231d98 100644 --- a/docs/sections/user_guide/api/shave.rst +++ b/docs/sections/user_guide/api/shave.rst @@ -1,5 +1,5 @@ ``uwtools.api.shave`` -====================== +===================== .. automodule:: uwtools.api.shave :members: diff --git a/docs/shared/shave.yaml b/docs/shared/shave.yaml index 70fc752c2..927706f3f 100644 --- a/docs/shared/shave.yaml +++ b/docs/shared/shave.yaml @@ -1,14 +1,14 @@ shave: - execution: - batchargs: - cores: 1 - walltime: "00:01:00" - executable: /path/to/shave config: input_grid_file: /path/to/input/grid/file nh4: 1 nx: 214 ny: 128 + execution: + batchargs: + cores: 1 + walltime: "00:01:00" + executable: /path/to/shave run_dir: /path/to/dir/run platform: account: me diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index ee6f66864..4f45f3923 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -1,9 +1,9 @@ upp: execution: batchargs: - nodes: 1 export: NONE - walltime: 00:05:00 + nodes: 1 + walltime: "00:05:00" envcmds: - module use /path/to/modulefiles - module load runtime-module @@ -30,12 +30,12 @@ upp: kpo: 3 numx: 1 po: - - 1000. - - 100. - - 1. + - 1000 + - 100 + - 1 run_dir: /path/to/run +user: + leadtime: "{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }}" platform: account: me scheduler: slurm -user: - leadtime: "{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }}" From 3413f8a5e786449c9eec07770d58baf3fbefa2a8 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 17:30:28 +0000 Subject: [PATCH 55/65] Doc update for consistency --- docs/sections/user_guide/yaml/components/shave.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sections/user_guide/yaml/components/shave.rst b/docs/sections/user_guide/yaml/components/shave.rst index c7f07b910..f550d17eb 100644 --- a/docs/sections/user_guide/yaml/components/shave.rst +++ b/docs/sections/user_guide/yaml/components/shave.rst @@ -16,7 +16,7 @@ UW YAML for the ``shave:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. config: ^^^^^^^ @@ -41,4 +41,4 @@ The i/x dimensions of the compute domain (not including halo). ny: """ -The j/y dimensions of the compute domain (not including halo) \ No newline at end of file +The j/y dimensions of the compute domain (not including halo) From b5abed632d2f97ed9f4a5036dad2a2c8cf4b055a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 18:06:36 +0000 Subject: [PATCH 56/65] Fix JEDI doc section on main page --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 72ce6c15c..671f60ce4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -160,12 +160,12 @@ UPP Driver for JEDI ^^^^^^^^^^^^^^^ -| **CLI**: ``uw jedi -h`` -| **API**: ``import uwtools.api.jedi`` - JEDI """" +| **CLI**: ``uw jedi -h`` +| **API**: ``import uwtools.api.jedi`` + Drivers for MPAS ^^^^^^^^^^^^^^^^ From 8966304d8aabf462ada56aae2d84c6e184c8d7ac Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 18:22:14 +0000 Subject: [PATCH 57/65] Add namelist.base_file to upp.yaml --- docs/shared/upp.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 4f45f3923..3c4e490d5 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -18,6 +18,7 @@ upp: eta_micro_lookup.dat: /path/to/nam_micro_lookup.dat params_grib2_tbl_new: /path/to/params_grib2_tbl_new namelist: + base_file: /path/to/base.nml update_values: model_inputs: datestr: "{{ validtime.strftime('%Y-%m-%d_%H:%M:%S') }}" From 2870a4c63a6243512779f8742bc65503af149c7a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 7 May 2024 19:54:07 +0000 Subject: [PATCH 58/65] Harmonize run_dir: descriptions in UW YAML docs --- .../user_guide/yaml/components/chgres_cube.rst | 2 +- docs/sections/user_guide/yaml/components/esg_grid.rst | 2 +- docs/sections/user_guide/yaml/components/fv3.rst | 2 +- .../user_guide/yaml/components/global_equiv_resol.rst | 2 +- docs/sections/user_guide/yaml/components/jedi.rst | 2 +- .../sections/user_guide/yaml/components/make_hgrid.rst | 10 +++++----- docs/sections/user_guide/yaml/components/mpas.rst | 7 ++----- docs/sections/user_guide/yaml/components/mpas_init.rst | 6 ++---- .../user_guide/yaml/components/sfc_climo_gen.rst | 2 +- docs/sections/user_guide/yaml/components/shave.rst | 5 +++++ docs/sections/user_guide/yaml/components/ungrib.rst | 2 +- docs/sections/user_guide/yaml/components/upp.rst | 2 +- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/sections/user_guide/yaml/components/chgres_cube.rst b/docs/sections/user_guide/yaml/components/chgres_cube.rst index dde09414e..6f6923636 100644 --- a/docs/sections/user_guide/yaml/components/chgres_cube.rst +++ b/docs/sections/user_guide/yaml/components/chgres_cube.rst @@ -28,4 +28,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir ^^^^^^^ -The path to the directory where ``chgres_cube`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/esg_grid.rst b/docs/sections/user_guide/yaml/components/esg_grid.rst index 3ce8738c1..072c293c2 100644 --- a/docs/sections/user_guide/yaml/components/esg_grid.rst +++ b/docs/sections/user_guide/yaml/components/esg_grid.rst @@ -25,4 +25,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where ``esg_grid`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/fv3.rst b/docs/sections/user_guide/yaml/components/fv3.rst index 0aa7c0701..38dcc9382 100644 --- a/docs/sections/user_guide/yaml/components/fv3.rst +++ b/docs/sections/user_guide/yaml/components/fv3.rst @@ -83,4 +83,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where FV3 will find its inputs, configuration files, etc., and where it will write its output. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst index 3da9e70bc..635b5461c 100644 --- a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst +++ b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst @@ -28,4 +28,4 @@ The path to the input grid file required by the program. run_dir: ^^^^^^^^ -The path to the directory where ``global_equiv_resol`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/jedi.rst b/docs/sections/user_guide/yaml/components/jedi.rst index f12175e7b..c2da4630b 100644 --- a/docs/sections/user_guide/yaml/components/jedi.rst +++ b/docs/sections/user_guide/yaml/components/jedi.rst @@ -37,4 +37,4 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``jedi`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/make_hgrid.rst b/docs/sections/user_guide/yaml/components/make_hgrid.rst index ff059ed16..3ad7f7548 100644 --- a/docs/sections/user_guide/yaml/components/make_hgrid.rst +++ b/docs/sections/user_guide/yaml/components/make_hgrid.rst @@ -21,11 +21,6 @@ execution: See :ref:`this page ` for details. -run_dir: -^^^^^^^^ - -The path to the directory where ``make_hgrid`` will write its outputs. - config: ^^^^^^^ @@ -235,3 +230,8 @@ ybnds: """""" Specify boundaries for defining meridional regions of varying resolution. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/mpas.rst b/docs/sections/user_guide/yaml/components/mpas.rst index 9093e392f..1614b94f5 100644 --- a/docs/sections/user_guide/yaml/components/mpas.rst +++ b/docs/sections/user_guide/yaml/components/mpas.rst @@ -25,9 +25,7 @@ See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ -Describes the boundary condition files needed for the forecast. These will be the output from the -``init_atmosphere`` executable, which may be run using the ``mpas_init`` UW driver. Please see its -documentation :ref:`here `. +Describes the boundary condition files needed for the forecast. These will be the output from the ``init_atmosphere`` executable, which may be run using the ``mpas_init`` UW driver. Please see its documentation :ref:`here `. interval_hours: """"""""""""""" @@ -78,8 +76,7 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``mpas`` will find its namelist, streams file, and necessary data -files and write its outputs. +The path to the run directory. streams: ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/mpas_init.rst b/docs/sections/user_guide/yaml/components/mpas_init.rst index b7dce112d..b6419cd4e 100644 --- a/docs/sections/user_guide/yaml/components/mpas_init.rst +++ b/docs/sections/user_guide/yaml/components/mpas_init.rst @@ -25,8 +25,7 @@ See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ -Describes the boundary condition files needed for the forecast. These will most likely be the output -of the ``ungrib`` tool. +Describes the boundary condition files needed for the forecast. These will most likely be the output of the ``ungrib`` tool. interval_hours: """"""""""""""" @@ -77,8 +76,7 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``mpas_init`` will find its namelist, streams file, and necessary data -files and write its outputs. +The path to the run directory. streams: ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst index af4aebf62..6b8ab91d7 100644 --- a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst +++ b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst @@ -26,4 +26,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where ``sfc_climo_gen`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/shave.rst b/docs/sections/user_guide/yaml/components/shave.rst index f550d17eb..0bb818083 100644 --- a/docs/sections/user_guide/yaml/components/shave.rst +++ b/docs/sections/user_guide/yaml/components/shave.rst @@ -42,3 +42,8 @@ ny: """ The j/y dimensions of the compute domain (not including halo) + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/ungrib.rst b/docs/sections/user_guide/yaml/components/ungrib.rst index 201bc237b..7db1df55d 100644 --- a/docs/sections/user_guide/yaml/components/ungrib.rst +++ b/docs/sections/user_guide/yaml/components/ungrib.rst @@ -48,7 +48,7 @@ An absolute-path template to the GRIB-formatted files to be processed by ``ungri run_dir: ^^^^^^^^ -The path to the directory where ``ungrib`` will find its namelist and write its outputs. +The path to the run directory. vtable: ^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index c4d327d19..8bf17c633 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -54,4 +54,4 @@ Read more on the UPP namelists, including variable meanings and appropriate valu run_dir: ^^^^^^^^ -The path to the UPP run directory. +The path to the run directory. From 6dc1f5671b25daec7aedad15ddfd9743b33508cb Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 01:42:21 +0000 Subject: [PATCH 59/65] Test update --- src/uwtools/api/shave.py | 2 +- src/uwtools/tests/test_schemas.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/uwtools/api/shave.py b/src/uwtools/api/shave.py index 6c82fb053..da1c3f2a1 100644 --- a/src/uwtools/api/shave.py +++ b/src/uwtools/api/shave.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 0c21500fc..29c791212 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -917,6 +917,10 @@ def test_schema_upp_namelist(upp_prop): assert not errors( {"base_file": "/path/to/base.nml", "update_values": {"model_inputs": {"grib": "grib2"}}} ) + # Only two specific namelists are allowed: + assert "Additional properties are not allowed" in errors( + {"udpate_values": {"another_namelist": {}}} + ) # model_inputs: datestr requires a specific format: assert not errors({"update_values": {"model_inputs": {"datestr": "2024-05-06_12:00:00"}}}) assert "does not match" in errors( @@ -930,7 +934,6 @@ def test_schema_upp_namelist(upp_prop): ) assert "not of type 'string'" in errors({"update_values": {"model_inputs": {key: 88}}}) # model_inputs: Only one grib value is supported: - assert not errors({"update_values": {"model_inputs": {"grib": "grib2"}}}) assert "not one of ['grib2']" in errors({"update_values": {"model_inputs": {"grib": "grib1"}}}) assert "not of type 'string'" in errors({"update_values": {"model_inputs": {"grib": 88}}}) # model_inputs: Only certain ioform values are supported: @@ -981,6 +984,10 @@ def test_schema_upp_namelist(upp_prop): ) # nampgb: Only one vtimeunits value is supported: assert "not one of ['FMIN']" in errors({"update_values": {"nampgb": {"vtimeunits": "FOO"}}}) + # nampgb: No other keys are supported: + assert "Additional properties are not allowed" in errors( + {"update_values": {"nampgb": {"something": "else"}}} + ) def test_schema_upp_run_dir(upp_prop): From b040f6a02c5d127c6e707542a83cb3ca785a21fe Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 18:39:05 +0000 Subject: [PATCH 60/65] Support subhourly leadtime --- docs/sections/user_guide/cli/drivers/upp.rst | 2 +- .../user_guide/yaml/components/upp.rst | 2 +- docs/shared/injected_cycle.rst | 2 +- docs/shared/injected_leadtime.rst | 13 ++++++++++++ docs/shared/injected_validtime.rst | 15 ------------- docs/shared/upp.yaml | 8 +++---- src/uwtools/cli.py | 21 +++++++++++++++++-- src/uwtools/drivers/driver.py | 10 ++++----- src/uwtools/drivers/upp.py | 9 ++++---- src/uwtools/tests/api/test_drivers.py | 6 +++--- src/uwtools/tests/drivers/test_driver.py | 10 ++++++--- src/uwtools/tests/drivers/test_upp.py | 4 ++-- src/uwtools/tests/test_cli.py | 11 +++++++++- src/uwtools/tests/utils/test_api.py | 14 ++++++------- src/uwtools/utils/api.py | 4 ++-- 15 files changed, 77 insertions(+), 54 deletions(-) create mode 100644 docs/shared/injected_leadtime.rst delete mode 100644 docs/shared/injected_validtime.rst diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp.rst index 7ed8996c1..f7004d3be 100644 --- a/docs/sections/user_guide/cli/drivers/upp.rst +++ b/docs/sections/user_guide/cli/drivers/upp.rst @@ -47,7 +47,7 @@ All tasks take the same arguments. For example: --cycle CYCLE The cycle in ISO8601 format --leadtime LEADTIME - Leadtime in hours + Leadtime as HH[:MM[:SS]] Optional arguments: -h, --help diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst index 8bf17c633..822d35b85 100644 --- a/docs/sections/user_guide/yaml/components/upp.rst +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -6,7 +6,7 @@ upp Structured YAML to run the UPP post-processor is validated by JSON Schema and requires the ``upp:`` block, described below. If UPP is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. .. include:: ../../../../shared/injected_cycle.rst -.. include:: ../../../../shared/injected_validtime.rst +.. include:: ../../../../shared/injected_leadtime.rst Here is a prototype UW YAML ``upp:`` block, explained in detail below: diff --git a/docs/shared/injected_cycle.rst b/docs/shared/injected_cycle.rst index 5d979e823..b9ec27bc3 100644 --- a/docs/shared/injected_cycle.rst +++ b/docs/shared/injected_cycle.rst @@ -1,4 +1,4 @@ -* This driver receives a ``cycle`` argument, which it makes available as a Python ``datetime`` to Jinja2 when realizing its input config. This supports specification of cycle-specific values. For example, the key-value pair +* This driver receives a ``cycle`` argument, which it makes available as a Python ``datetime`` object to Jinja2 when realizing its input config. This supports specification of cycle-specific values. For example, the key-value pair .. code-block:: yaml diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_leadtime.rst new file mode 100644 index 000000000..a073775f3 --- /dev/null +++ b/docs/shared/injected_leadtime.rst @@ -0,0 +1,13 @@ +* This driver receives a ``leadtime`` argument, which it makes available as a Python ``timedelta`` object to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the key-value pair + + .. code-block:: yaml + + suffix: f{{ '%03d' % leadtime.hours }} + + would be rendered as + + .. code-block:: yaml + + suffix: f018 + + for leadtime 18. diff --git a/docs/shared/injected_validtime.rst b/docs/shared/injected_validtime.rst deleted file mode 100644 index e624576ce..000000000 --- a/docs/shared/injected_validtime.rst +++ /dev/null @@ -1,15 +0,0 @@ -* This driver receives a ``leadtime`` argument, from which it computes a ``validtime`` Python ``datetime`` that it makes available to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the values - - .. code-block:: yaml - - validstr: "{{ validtime.strftime('%Y-%m-%d %H:%M') }}" - suffix: f{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }} - - would be rendered as - - .. code-block:: yaml - - validstr: 2024-02-12 18:00 - suffix: f018 - - for cycle ``2024-02-12T18`` and leadtime ``18``. diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 3c4e490d5..5c8ea99d4 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -21,9 +21,9 @@ upp: base_file: /path/to/base.nml update_values: model_inputs: - datestr: "{{ validtime.strftime('%Y-%m-%d_%H:%M:%S') }}" - filename: /path/to/dynf{{ user.leadtime }}.nc - filenameflux: /path/to/phyf{{ user.leadtime }}.nc + datestr: "{{ (cycle + leadtime).strftime('%Y-%m-%d_%H:%M:%S') }}" + filename: /path/to/dynf{{ '%03d' % (leadtime.total_seconds() / 3600) }}.nc + filenameflux: /path/to/phyf{{ '%03d' % (leadtime.total_seconds() / 3600) }}.nc grib: grib2 ioform: netcdf modelname: FV3R @@ -35,8 +35,6 @@ upp: - 100 - 1 run_dir: /path/to/run -user: - leadtime: "{{ '%03d' % ((validtime - cycle).total_seconds() // 3600) }}" platform: account: me scheduler: slurm diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index f4598f845..49c6977d0 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -27,6 +27,7 @@ from uwtools.utils.file import get_file_format, resource_path FORMATS = FORMAT.extensions() +LEADTIME_DESC = "HH[:MM[:SS]]" TITLE_REQ_ARG = "Required arguments" Args = Dict[str, Any] @@ -646,9 +647,9 @@ def _add_arg_keys(group: Group) -> None: def _add_arg_leadtime(group: Group) -> None: group.add_argument( _switch(STR.leadtime), - help="Leadtime in hours", + help=f"Leadtime as {LEADTIME_DESC}", required=True, - type=int, + type=_timedelta_from_str, ) @@ -1033,6 +1034,22 @@ def _switch(arg: str) -> str: return "--%s" % arg.replace("_", "-") +def _timedelta_from_str(tds: str) -> dt.timedelta: + """ + Return a timedelta parsed from a leadtime string. + + :param tds: The timedelta string to parse. + """ + fmts = ("%H:%M:%S", "%H:%M", "%H") + for fmt in fmts: + try: + t = dt.datetime.strptime(tds, fmt) + return dt.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) + except ValueError: + pass + _abort(f"Specify leadtime as {LEADTIME_DESC}") + + def _version() -> str: """ Return version information. diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index db278d24b..2c56cd0ef 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -33,7 +33,7 @@ def __init__( dry_run: bool = False, batch: bool = False, cycle: Optional[datetime] = None, - leadtime: Optional[int] = None, + leadtime: Optional[timedelta] = None, ) -> None: """ A component driver. @@ -47,14 +47,12 @@ def __init__( self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch - if leadtime: - if not cycle: - raise UWError("When leadtime is specified, cycle is required") - self._validtime = cycle + timedelta(hours=leadtime) + if leadtime and not cycle: + raise UWError("When leadtime is specified, cycle is required") self._config.dereference( context={ **({"cycle": cycle} if cycle else {}), - **({"validtime": self._validtime} if leadtime else {}), + **({"leadtime": leadtime} if leadtime else {}), **self._config.data, } ) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 0692ff433..e4d9d9e93 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -2,7 +2,7 @@ A driver for UPP. """ -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Optional @@ -22,7 +22,7 @@ class UPP(Driver): def __init__( self, cycle: datetime, - leadtime: int, + leadtime: timedelta, config: Optional[Path] = None, dry_run: bool = False, batch: bool = False, @@ -155,9 +155,8 @@ def _taskname(self, suffix: str) -> str: :param suffix: Log-string suffix. """ - return "%s f%03d %s %s" % ( - self._cycle.strftime("%Y%m%d %HZ"), - self._leadtime, + return "%s %s %s" % ( + (self._cycle + self._leadtime).strftime("%Y%m%d %H:%M:%S"), self._driver_name, suffix, ) diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index 3742694ef..d7652f39c 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -1,6 +1,6 @@ # pylint: disable=missing-function-docstring,protected-access -from datetime import datetime as dt +import datetime as dt from unittest.mock import patch import iotaa @@ -53,8 +53,8 @@ def test_api_execute(module): } kwargs = { **kwbase, - **({"cycle": dt.now()} if module in with_cycle else {}), - **({"leadtime": 24} if module in with_leadtime else {}), + **({"cycle": dt.datetime.now()} if module in with_cycle else {}), + **({"leadtime": dt.timedelta(hours=24)} if module in with_leadtime else {}), } with patch.object(api, "_execute") as _execute: module.execute(**kwargs) diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 2691bc580..e0b9aaee1 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -2,9 +2,9 @@ """ Tests for uwtools.drivers.driver module. """ +import datetime as dt import json import logging -from datetime import datetime from pathlib import Path from textwrap import dedent from unittest.mock import Mock, PropertyMock, patch @@ -86,7 +86,11 @@ def config(tmp_path): @fixture def driverobj(config): return ConcreteDriver( - config=config, dry_run=True, batch=True, cycle=datetime(2024, 3, 22, 18), leadtime=24 + config=config, + dry_run=True, + batch=True, + cycle=dt.datetime(2024, 3, 22, 18), + leadtime=dt.timedelta(hours=24), ) @@ -101,7 +105,7 @@ def test_Driver(driverobj): def test_Driver_cycle_leadtime_error(config): with raises(UWError) as e: - ConcreteDriver(config=config, leadtime=24) + ConcreteDriver(config=config, leadtime=dt.timedelta(hours=24)) assert "When leadtime is specified, cycle is required" in str(e) diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py index 7f77d06cb..50487c703 100644 --- a/src/uwtools/tests/drivers/test_upp.py +++ b/src/uwtools/tests/drivers/test_upp.py @@ -76,7 +76,7 @@ def driverobj(config_file, cycle, leadtime): @fixture def leadtime(): - return 24 + return dt.timedelta(hours=24) # Tests @@ -171,7 +171,7 @@ def test_UPP__driver_config(driverobj): def test_UPP__taskname(driverobj): - assert driverobj._taskname("foo") == "20240506 12Z f024 upp foo" + assert driverobj._taskname("foo") == "20240507 12:00:00 upp foo" def test_UPP__validate(driverobj): diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index f22904096..aeb79bae6 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -558,7 +558,7 @@ def test__dispatch_template_translate_no_optional(): def test__dispatch_to_driver(): name = "adriver" cycle = dt.datetime.now() - leadtime = 24 + leadtime = dt.timedelta(hours=24) args: dict = { "action": "foo", "batch": True, @@ -653,5 +653,14 @@ def test__switch(): assert cli._switch("foo_bar") == "--foo-bar" +def test__timedelta_from_str(capsys): + assert cli._timedelta_from_str("11:12:13") == dt.timedelta(hours=11, minutes=12, seconds=13) + assert cli._timedelta_from_str("11:12") == dt.timedelta(hours=11, minutes=12) + assert cli._timedelta_from_str("11") == dt.timedelta(hours=11) + with raises(SystemExit): + cli._timedelta_from_str("foo") + assert f"Specify leadtime as {cli.LEADTIME_DESC}" in capsys.readouterr().err + + def test__version(): assert re.match(r"version \d+\.\d+\.\d+ build \d+", cli._version()) diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index e901ffd41..d35360be6 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -1,6 +1,6 @@ # pylint: disable=missing-function-docstring,protected-access,redefined-outer-name -from datetime import datetime as dt +import datetime as dt from pathlib import Path from unittest.mock import patch @@ -62,7 +62,7 @@ def test_make_execute(execute_kwargs): def test_make_execute_cycle(execute_kwargs): - execute_kwargs["cycle"] = dt.now() + execute_kwargs["cycle"] = dt.datetime.now() func = api.make_execute(driver_class=ConcreteDriver, with_cycle=True) assert func.__name__ == "execute" assert func.__doc__ is not None @@ -77,8 +77,8 @@ def test_make_execute_cycle(execute_kwargs): def test_make_execute_cycle_leadtime(execute_kwargs): - execute_kwargs["cycle"] = dt.now() - execute_kwargs["leadtime"] = 24 + execute_kwargs["cycle"] = dt.datetime.now() + execute_kwargs["leadtime"] = dt.timedelta(hours=24) func = api.make_execute(driver_class=ConcreteDriver, with_cycle=True, with_leadtime=True) assert func.__name__ == "execute" assert func.__doc__ is not None @@ -92,7 +92,7 @@ def test_make_execute_cycle_leadtime(execute_kwargs): def test_make_execute_leadtime_no_cycle_error(execute_kwargs): - execute_kwargs["leadtime"] = 24 + execute_kwargs["leadtime"] = dt.timedelta(hours=24) with raises(UWError) as e: api.make_execute(driver_class=ConcreteDriver, with_leadtime=True) assert "When leadtime is specified, cycle is required" in str(e) @@ -126,8 +126,8 @@ def test__execute(execute_kwargs, tmp_path): **execute_kwargs, "driver_class": ConcreteDriver, "config": config, - "cycle": dt.now(), - "leadtime": 24, + "cycle": dt.datetime.now(), + "leadtime": dt.timedelta(hours=24), "graph_file": graph_file, } assert not graph_file.is_file() diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 8d03b75e2..a3abc0ea5 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -89,7 +89,7 @@ def execute_cycle( # pylint: disable=unused-argument def execute_cycle_leadtime( # pylint: disable=unused-argument task: str, cycle: dt.datetime, - leadtime: int, + leadtime: dt.timedelta, config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, @@ -159,7 +159,7 @@ def _execute( driver_class: type[Driver], task: str, cycle: Optional[dt.datetime] = None, - leadtime: Optional[int] = None, + leadtime: Optional[dt.timedelta] = None, config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, From 0ffe17e46f1a495c3d9520863117ae4d40787c87 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 18:40:41 +0000 Subject: [PATCH 61/65] Give ISO8601 example in CLI help --- docs/sections/user_guide/cli/drivers/chgres_cube.rst | 2 +- docs/sections/user_guide/cli/drivers/fv3.rst | 2 +- docs/sections/user_guide/cli/drivers/jedi.rst | 2 +- docs/sections/user_guide/cli/drivers/mpas.rst | 2 +- docs/sections/user_guide/cli/drivers/mpas_init.rst | 2 +- docs/sections/user_guide/cli/drivers/ungrib.rst | 2 +- docs/sections/user_guide/cli/drivers/upp.rst | 2 +- src/uwtools/cli.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube.rst index ed5e7a88e..0cd20f7c2 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube.rst @@ -41,7 +41,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/fv3.rst b/docs/sections/user_guide/cli/drivers/fv3.rst index 12a5ff632..c98c4abb7 100644 --- a/docs/sections/user_guide/cli/drivers/fv3.rst +++ b/docs/sections/user_guide/cli/drivers/fv3.rst @@ -57,7 +57,7 @@ All tasks take the same arguments. For example: --config-file PATH, -c PATH Path to UW YAML config file --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/jedi.rst b/docs/sections/user_guide/cli/drivers/jedi.rst index 63e1fffe4..620a29f17 100644 --- a/docs/sections/user_guide/cli/drivers/jedi.rst +++ b/docs/sections/user_guide/cli/drivers/jedi.rst @@ -47,7 +47,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/mpas.rst b/docs/sections/user_guide/cli/drivers/mpas.rst index fc72af12c..fe5516e54 100644 --- a/docs/sections/user_guide/cli/drivers/mpas.rst +++ b/docs/sections/user_guide/cli/drivers/mpas.rst @@ -49,7 +49,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/mpas_init.rst b/docs/sections/user_guide/cli/drivers/mpas_init.rst index f7df09cc5..49d63a168 100644 --- a/docs/sections/user_guide/cli/drivers/mpas_init.rst +++ b/docs/sections/user_guide/cli/drivers/mpas_init.rst @@ -49,7 +49,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/ungrib.rst b/docs/sections/user_guide/cli/drivers/ungrib.rst index 78dc537cd..79ba93d1e 100644 --- a/docs/sections/user_guide/cli/drivers/ungrib.rst +++ b/docs/sections/user_guide/cli/drivers/ungrib.rst @@ -45,7 +45,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/upp.rst b/docs/sections/user_guide/cli/drivers/upp.rst index f7004d3be..5817b2555 100644 --- a/docs/sections/user_guide/cli/drivers/upp.rst +++ b/docs/sections/user_guide/cli/drivers/upp.rst @@ -45,7 +45,7 @@ All tasks take the same arguments. For example: Required arguments: --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) --leadtime LEADTIME Leadtime as HH[:MM[:SS]] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 49c6977d0..f0d15bf64 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -552,7 +552,7 @@ def _add_arg_config_file(group: Group, required: bool = False) -> None: def _add_arg_cycle(group: Group) -> None: group.add_argument( _switch(STR.cycle), - help="The cycle in ISO8601 format", + help="The cycle in ISO8601 format (e.g. 2024-05-08T18)", required=True, type=dt.datetime.fromisoformat, ) From 80e59e9e4c9f15f830aa77bc4295f695e047cd84 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 18:49:59 +0000 Subject: [PATCH 62/65] Doc fix --- docs/shared/injected_leadtime.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_leadtime.rst index a073775f3..27bb5a6db 100644 --- a/docs/shared/injected_leadtime.rst +++ b/docs/shared/injected_leadtime.rst @@ -2,7 +2,7 @@ .. code-block:: yaml - suffix: f{{ '%03d' % leadtime.hours }} + suffix: f{{ '%03d' % (leadtime.total_seconds() / 3600) }} would be rendered as From 46432cdb5ae3beb1b5d48096747634c5ebc5399e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 18:59:44 +0000 Subject: [PATCH 63/65] Doc improvement --- docs/shared/injected_leadtime.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_leadtime.rst index 27bb5a6db..c0740f8f8 100644 --- a/docs/shared/injected_leadtime.rst +++ b/docs/shared/injected_leadtime.rst @@ -1,13 +1,15 @@ -* This driver receives a ``leadtime`` argument, which it makes available as a Python ``timedelta`` object to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the key-value pair +* This driver receives a ``leadtime`` argument, which it makes available as a Python ``timedelta`` object to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the key-value pairs .. code-block:: yaml + datestr: "{{ (cycle + leadtime).strftime('%Y-%m-%d_%H:%M:%S') }}" suffix: f{{ '%03d' % (leadtime.total_seconds() / 3600) }} would be rendered as .. code-block:: yaml + datestr: 2024-05-09_06:00:00 suffix: f018 - for leadtime 18. + for cycle ``2024-05-08T12`` and leadtime ``18``. From 5aa2ed95a54b852d36785a56517c787241606fb3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 19:38:55 +0000 Subject: [PATCH 64/65] Tweak CLI help message for --leadtime --- src/uwtools/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index f0d15bf64..3419eca9e 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -647,7 +647,7 @@ def _add_arg_keys(group: Group) -> None: def _add_arg_leadtime(group: Group) -> None: group.add_argument( _switch(STR.leadtime), - help=f"Leadtime as {LEADTIME_DESC}", + help=f"The leadtime as {LEADTIME_DESC}", required=True, type=_timedelta_from_str, ) From 3af186f33cfcfb0251f4a9fa3b9de87c3568fa46 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 8 May 2024 19:59:02 +0000 Subject: [PATCH 65/65] Give current 6-hourly cycle in CLI advice --- src/uwtools/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 3419eca9e..7d75a608c 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -550,9 +550,11 @@ def _add_arg_config_file(group: Group, required: bool = False) -> None: def _add_arg_cycle(group: Group) -> None: + offset = dt.timedelta(hours=(dt.datetime.now(dt.UTC).hour // 6) * 6) + cycle = dt.datetime.combine(dt.date.today(), dt.datetime.min.time()) + offset group.add_argument( _switch(STR.cycle), - help="The cycle in ISO8601 format (e.g. 2024-05-08T18)", + help="The cycle in ISO8601 format (e.g. %s)" % cycle.strftime("%Y-%m-%dT%H"), required=True, type=dt.datetime.fromisoformat, )