diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5a90bf43f..7ef7fee1e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,7 +21,7 @@ INSTRUCTIONS - [ ] Bug fix (corrects a known issue) - [ ] Code maintenance (refactoring, etc. without behavior change) - [ ] Documentation -- [ ] Enhancement (adds a new functionality) +- [ ] Enhancement (adds new functionality) - [ ] Tooling (CI, code-quality, packaging, revision-control, etc.) **Impact** diff --git a/docs/index.rst b/docs/index.rst index e5ace06b3..671f60ce4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -110,28 +110,28 @@ Drivers for UFS To prepare a complete forecast, drivers would typically be run in the order shown here (along with additional drivers still in development). esg_grid -""""""""""" +"""""""" | **CLI**: ``uw esg_grid -h`` -| **API**: ``import uwtools.api.drivers.esg_grid`` +| **API**: ``import uwtools.api.esg_grid`` global_equiv_resol """""""""""""""""" | **CLI**: ``uw global_equiv_resol -h`` -| **API**: ``import uwtools.api.drivers.global_equiv_resol`` +| **API**: ``import uwtools.api.global_equiv_resol`` make_hgrid """""""""" | **CLI**: ``uw make_hgrid -h`` -| **API**: ``import uwtools.api.drivers.make_hgrid`` +| **API**: ``import uwtools.api.make_hgrid`` sfc_climo_gen """"""""""""" | **CLI**: ``uw sfc_climo_gen -h`` -| **API**: ``import uwtools.api.drivers.sfc_climo_gen`` +| **API**: ``import uwtools.api.sfc_climo_gen`` shave """"" @@ -143,22 +143,28 @@ chgres_cube """"""""""" | **CLI**: ``uw chgres_cube -h`` -| **API**: ``import uwtools.api.drivers.chgres_cube`` +| **API**: ``import uwtools.api.chgres_cube`` FV3 """ | **CLI**: ``uw fv3 -h`` -| **API**: ``import uwtools.api.drivers.fv3`` +| **API**: ``import uwtools.api.fv3`` -JEDI -"""" +UPP +""" -| **CLI**: ``uw jedi -h`` -| **API**: ``import uwtools.api.drivers.jedi`` +| **CLI**: ``uw upp -h`` +| **API**: ``import uwtools.api.upp`` +Driver for JEDI +^^^^^^^^^^^^^^^ +JEDI +"""" +| **CLI**: ``uw jedi -h`` +| **API**: ``import uwtools.api.jedi`` Drivers for MPAS ^^^^^^^^^^^^^^^^ @@ -169,22 +175,19 @@ ungrib """""" | **CLI**: ``uw ungrib -h`` -| **API**: ``import uwtools.api.drivers.ungrib`` - +| **API**: ``import uwtools.api.ungrib`` mpas_init """"""""" | **CLI**: ``uw mpas_init -h`` -| **API**: ``import uwtools.api.drivers.mpas_init`` +| **API**: ``import uwtools.api.mpas_init`` mpas """" | **CLI**: ``uw mpas -h`` -| **API**: ``import uwtools.api.drivers.mpas`` - - +| **API**: ``import uwtools.api.mpas`` ------------------ diff --git a/docs/sections/user_guide/api/chgres_cube.rst b/docs/sections/user_guide/api/chgres_cube.rst index 4d2faf746..b36a673e4 100644 --- a/docs/sections/user_guide/api/chgres_cube.rst +++ b/docs/sections/user_guide/api/chgres_cube.rst @@ -1,5 +1,5 @@ ``uwtools.api.chgres_cube`` -============================= +=========================== .. automodule:: uwtools.api.chgres_cube :members: diff --git a/docs/sections/user_guide/api/esg_grid.rst b/docs/sections/user_guide/api/esg_grid.rst index 7205ec941..c706d377e 100644 --- a/docs/sections/user_guide/api/esg_grid.rst +++ b/docs/sections/user_guide/api/esg_grid.rst @@ -1,5 +1,5 @@ ``uwtools.api.esg_grid`` -============================= +======================== .. automodule:: uwtools.api.esg_grid :members: diff --git a/docs/sections/user_guide/api/fv3.rst b/docs/sections/user_guide/api/fv3.rst index 4a5e70954..3e447a20b 100644 --- a/docs/sections/user_guide/api/fv3.rst +++ b/docs/sections/user_guide/api/fv3.rst @@ -1,5 +1,5 @@ ``uwtools.api.fv3`` -======================== +=================== .. automodule:: uwtools.api.fv3 :members: diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 6e4a24401..dec1d76f1 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -18,3 +18,4 @@ API shave template ungrib + upp diff --git a/docs/sections/user_guide/api/jedi.rst b/docs/sections/user_guide/api/jedi.rst index f0eb7acce..a08cc8467 100644 --- a/docs/sections/user_guide/api/jedi.rst +++ b/docs/sections/user_guide/api/jedi.rst @@ -1,5 +1,5 @@ ``uwtools.api.jedi`` -========================== +==================== .. automodule:: uwtools.api.jedi :members: diff --git a/docs/sections/user_guide/api/mpas.rst b/docs/sections/user_guide/api/mpas.rst index 6cda7b22b..032b76029 100644 --- a/docs/sections/user_guide/api/mpas.rst +++ b/docs/sections/user_guide/api/mpas.rst @@ -1,5 +1,5 @@ ``uwtools.api.mpas`` -============================= +==================== .. automodule:: uwtools.api.mpas :members: diff --git a/docs/sections/user_guide/api/mpas_init.rst b/docs/sections/user_guide/api/mpas_init.rst index 0eb2a87bc..d20095e73 100644 --- a/docs/sections/user_guide/api/mpas_init.rst +++ b/docs/sections/user_guide/api/mpas_init.rst @@ -1,5 +1,5 @@ ``uwtools.api.mpas_init`` -============================= +========================= .. automodule:: uwtools.api.mpas_init :members: diff --git a/docs/sections/user_guide/api/shave.rst b/docs/sections/user_guide/api/shave.rst index e189ae222..395231d98 100644 --- a/docs/sections/user_guide/api/shave.rst +++ b/docs/sections/user_guide/api/shave.rst @@ -1,5 +1,5 @@ ``uwtools.api.shave`` -====================== +===================== .. automodule:: uwtools.api.shave :members: diff --git a/docs/sections/user_guide/api/ungrib.rst b/docs/sections/user_guide/api/ungrib.rst index 6db41352b..e149acaf2 100644 --- a/docs/sections/user_guide/api/ungrib.rst +++ b/docs/sections/user_guide/api/ungrib.rst @@ -1,5 +1,5 @@ ``uwtools.api.ungrib`` -============================= +====================== .. automodule:: uwtools.api.ungrib - :members: \ No newline at end of file + :members: diff --git a/docs/sections/user_guide/api/upp.rst b/docs/sections/user_guide/api/upp.rst new file mode 100644 index 000000000..1130fcd27 --- /dev/null +++ b/docs/sections/user_guide/api/upp.rst @@ -0,0 +1,5 @@ +``uwtools.api.upp`` +=================== + +.. automodule:: uwtools.api.upp + :members: diff --git a/docs/sections/user_guide/cli/drivers/chgres_cube.rst b/docs/sections/user_guide/cli/drivers/chgres_cube.rst index 592fb8905..0cd20f7c2 100644 --- a/docs/sections/user_guide/cli/drivers/chgres_cube.rst +++ b/docs/sections/user_guide/cli/drivers/chgres_cube.rst @@ -34,22 +34,22 @@ All tasks take the same arguments. For example: .. code-block:: text $ uw chgres_cube run --help - usage: uw chgres_cube run --config-file PATH --cycle CYCLE [-h] [--version] [--batch] [--dry-run] - [--graph-file PATH] [--quiet] [--verbose] + usage: uw chgres_cube run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] + [--dry-run] [--graph-file PATH] [--quiet] [--verbose] A run Required arguments: - --config-file PATH, -c PATH - Path to UW YAML config file --cycle CYCLE - The cycle in ISO8601 format + The cycle in ISO8601 format (e.g. 2024-05-08T18) Optional arguments: -h, --help Show help and exit --version Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) --batch Submit run to batch scheduler --dry-run @@ -69,10 +69,8 @@ The examples use a configuration file named ``config.yaml`` with content similar .. highlight:: yaml .. literalinclude:: ../../../../shared/chgres_cube.yaml - Its contents are described in depth in section :ref:`chgres_cube_yaml`. Each of the values in the ``chgres_cube`` YAML may contain Jinja2 variables/expressions using a ``cycle`` variable, which is a Python ``datetime`` object corresponding to the FV3 cycle being run. - * Run ``chgres_cube`` on an interactive node .. code-block:: text diff --git a/docs/sections/user_guide/cli/drivers/esg_grid.rst b/docs/sections/user_guide/cli/drivers/esg_grid.rst index e9cfb8104..a40590285 100644 --- a/docs/sections/user_guide/cli/drivers/esg_grid.rst +++ b/docs/sections/user_guide/cli/drivers/esg_grid.rst @@ -6,57 +6,56 @@ The ``uw`` mode for configuring and running the :ufs-utils:`regional_esg_grid`_ component. + +.. code-block:: text + + $ uw upp --help + usage: uw upp [-h] [--version] TASK ... + + Execute upp tasks + + Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + + Positional arguments: + TASK + files_copied + Files copied for run + files_linked + Files linked for run + namelist_file + The namelist file + provisioned_run_directory + Run directory provisioned with all required content + run + A run + runscript + The runscript + validate + Validate the UW driver config + +All tasks take the same arguments. For example: + +.. code-block:: text + + $ uw upp run --help + usage: uw upp run --cycle CYCLE --leadtime LEADTIME [-h] [--version] [--config-file PATH] + [--batch] [--dry-run] [--graph-file PATH] [--quiet] [--verbose] + + A run + + Required arguments: + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-05-08T18) + --leadtime LEADTIME + Leadtime as HH[:MM[:SS]] + + Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with content similar to: + +.. highlight:: yaml +.. literalinclude:: ../../../../shared/upp.yaml + +Its contents are described in depth in section :ref:`upp_yaml`. + +* Run ``upp`` on an interactive node + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 + + The driver creates a ``runscript.upp`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``upp``. + +* Run ``upp`` via a batch job + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch + + The driver creates a ``runscript.upp`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``upp:``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch --dry-run + +* The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``upp`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file: + + .. code-block:: text + + $ uw upp provisioned_run_directory --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch diff --git a/docs/sections/user_guide/yaml/components/chgres_cube.rst b/docs/sections/user_guide/yaml/components/chgres_cube.rst index f857e66d1..6f6923636 100644 --- a/docs/sections/user_guide/yaml/components/chgres_cube.rst +++ b/docs/sections/user_guide/yaml/components/chgres_cube.rst @@ -5,6 +5,8 @@ chgres_cube Structured YAML to run :ufs-utils:`chgres_cube` is validated by JSON Schema and requires the ``chgres_cube:`` block, described below. If ``chgres_cube`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``chgres_cube:`` block, explained in detail below: .. highlight:: yaml @@ -16,7 +18,7 @@ UW YAML for the ``chgres_cube:`` Block execution ^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist ^^^^^^^^ @@ -26,4 +28,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir ^^^^^^^ -The path to the directory where ``chgres_cube`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/esg_grid.rst b/docs/sections/user_guide/yaml/components/esg_grid.rst index 29b3106d6..072c293c2 100644 --- a/docs/sections/user_guide/yaml/components/esg_grid.rst +++ b/docs/sections/user_guide/yaml/components/esg_grid.rst @@ -16,7 +16,7 @@ UW YAML for the ``esg_grid:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist: ^^^^^^^^^ @@ -25,4 +25,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where ``esg_grid`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/fv3.rst b/docs/sections/user_guide/yaml/components/fv3.rst index 164497a57..38dcc9382 100644 --- a/docs/sections/user_guide/yaml/components/fv3.rst +++ b/docs/sections/user_guide/yaml/components/fv3.rst @@ -5,6 +5,8 @@ fv3 Structured YAML to run FV3 is validated by JSON Schema and requires the ``fv3:`` block, described below. If FV3 is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. The configuration files required by the UFS Weather Model are documented :weather-model-io:`here`. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``fv3:`` block, explained in detail below: .. highlight:: yaml @@ -26,7 +28,7 @@ Accepted values are ``global`` and ``regional``. execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. field_table: ^^^^^^^^^^^^ @@ -36,19 +38,7 @@ The path to a :weather-model-io:`valid field-table file` to be files_to_copy: ^^^^^^^^^^^^^^ -See :ref:`this page ` for details. For the ``fv3`` driver, both keys and values may contain Jinja2 variables/expressions using a ``cycle`` variable, which is a Python ``datetime`` object corresponding to the FV3 cycle being run. This supports specification of cycle-specific filenames/paths. For example, a key-value pair - -.. code-block:: yaml - - gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc: /some/path/{{ cycle.strftime('%Y%m%d')}}/{{ cycle.strftime('%H') }}/gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc - -would be rendered as - -.. code-block:: yaml - - gfs.t18z.atmanl.nc: /some/path/20240212/18/gfs.t18z.atmanl.nc - -for the ``2024-02-12T18`` cycle. +See :ref:`this page ` for details. files_to_link: ^^^^^^^^^^^^^^ @@ -93,4 +83,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where FV3 will find its inputs, configuration files, etc., and where it will write its output. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst index 5754ba73c..635b5461c 100644 --- a/docs/sections/user_guide/yaml/components/global_equiv_resol.rst +++ b/docs/sections/user_guide/yaml/components/global_equiv_resol.rst @@ -18,7 +18,7 @@ UW YAML for the ``global_equiv_resol:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. input_grid_file: ^^^^^^^^^^^^^^^^ @@ -28,4 +28,4 @@ The path to the input grid file required by the program. run_dir: ^^^^^^^^ -The path to the directory where ``global_equiv_resol`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index d3250d18c..2a714b547 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -15,3 +15,4 @@ UW YAML for Components sfc_climo_gen shave ungrib + upp diff --git a/docs/sections/user_guide/yaml/components/jedi.rst b/docs/sections/user_guide/yaml/components/jedi.rst index 5408445e6..c2da4630b 100644 --- a/docs/sections/user_guide/yaml/components/jedi.rst +++ b/docs/sections/user_guide/yaml/components/jedi.rst @@ -5,18 +5,20 @@ jedi Structured YAML to run JEDI is validated by JSON Schema and requires the ``jedi:`` block, described below. If ``jedi`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML jedi: block, explained in detail below: .. highlight:: yaml .. literalinclude:: ../../../../shared/jedi.yaml UW YAML for the ``jedi:`` Block ------------------------------------- +------------------------------- execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. configuration_file: ^^^^^^^^^^^^^^^^^^^ @@ -27,12 +29,6 @@ files_to_copy: See :ref:`this page ` for details. -.. code-block:: text - - jedi: - files_to_copy: - f1: /path/to/f1 - files_to_link: ^^^^^^^^^^^^^^ @@ -41,4 +37,4 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``jedi`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/make_hgrid.rst b/docs/sections/user_guide/yaml/components/make_hgrid.rst index ef791b67e..3ad7f7548 100644 --- a/docs/sections/user_guide/yaml/components/make_hgrid.rst +++ b/docs/sections/user_guide/yaml/components/make_hgrid.rst @@ -19,19 +19,13 @@ UW YAML for the ``make_hgrid:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. - -run_dir: -^^^^^^^^ - -The path to the directory where ``make_hgrid`` will write its outputs. +See :ref:`this page ` for details. config: ^^^^^^^ Describes the required parameters to run a ``make_hgrid`` configuration. - grid_type: """""""""" @@ -62,8 +56,6 @@ grid_type: * - ``tripolar_grid`` - ``nxbnds``, ``nybnds``, ``xbnds``, ``ybnds`` must be specified to define the grid bounds. - - angular_midpoint: """"""""""""""""" @@ -159,13 +151,11 @@ nlon: Number of model grid points(supergrid) for each zonal regions of varying resolution. - nxbnds: """"""" Specify number of zonal regions for varying resolution. - nybnds: """"""" @@ -239,4 +229,9 @@ Specify boundaries for defining zonal regions of varying resolution. When ``trip ybnds: """""" -Specify boundaries for defining meridional regions of varying resolution. \ No newline at end of file +Specify boundaries for defining meridional regions of varying resolution. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/mpas.rst b/docs/sections/user_guide/yaml/components/mpas.rst index 2431c1824..1614b94f5 100644 --- a/docs/sections/user_guide/yaml/components/mpas.rst +++ b/docs/sections/user_guide/yaml/components/mpas.rst @@ -5,6 +5,8 @@ mpas Structured YAML to run MPAS is validated by JSON Schema and requires the ``mpas:`` block, described below. If ``mpas`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``mpas:`` block, explained in detail below: .. highlight:: yaml @@ -18,14 +20,12 @@ UW YAML for the ``mpas:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ -Describes the boundary condition files needed for the forecast. These will be the output from the -``init_atmosphere`` executable, which may be run using the ``mpas_init`` UW driver. Please see its -documentation :ref:`here `. +Describes the boundary condition files needed for the forecast. These will be the output from the ``init_atmosphere`` executable, which may be run using the ``mpas_init`` UW driver. Please see its documentation :ref:`here `. interval_hours: """"""""""""""" @@ -76,8 +76,7 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``mpas`` will find its namelist, streams file, and necessary data -files and write its outputs. +The path to the run directory. streams: ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/mpas_init.rst b/docs/sections/user_guide/yaml/components/mpas_init.rst index b2f324f04..b6419cd4e 100644 --- a/docs/sections/user_guide/yaml/components/mpas_init.rst +++ b/docs/sections/user_guide/yaml/components/mpas_init.rst @@ -5,6 +5,8 @@ mpas_init Structured YAML to run MPAS Init is validated by JSON Schema and requires the ``mpas_init:`` block, described below. If ``mpas_init`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``mpas_init:`` block, explained in detail below: .. highlight:: yaml @@ -18,13 +20,12 @@ UW YAML for the ``mpas_init:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. boundary_conditions: ^^^^^^^^^^^^^^^^^^^^ -Describes the boundary condition files needed for the forecast. These will most likely be the output -of the ``ungrib`` tool. +Describes the boundary condition files needed for the forecast. These will most likely be the output of the ``ungrib`` tool. interval_hours: """"""""""""""" @@ -75,8 +76,7 @@ Identical to ``files_to_copy:`` except that symbolic links will be created in th run_dir: ^^^^^^^^ -The path to the directory where ``mpas_init`` will find its namelist, streams file, and necessary data -files and write its outputs. +The path to the run directory. streams: ^^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst index 373324b1f..6b8ab91d7 100644 --- a/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst +++ b/docs/sections/user_guide/yaml/components/sfc_climo_gen.rst @@ -16,7 +16,7 @@ UW YAML for the ``sfc_climo_gen:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. namelist: ^^^^^^^^^ @@ -26,4 +26,4 @@ Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_va run_dir: ^^^^^^^^ -The path to the directory where ``sfc_climo_gen`` will find its namelist and write its outputs. +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/shave.rst b/docs/sections/user_guide/yaml/components/shave.rst index 25f7aa385..0bb818083 100644 --- a/docs/sections/user_guide/yaml/components/shave.rst +++ b/docs/sections/user_guide/yaml/components/shave.rst @@ -16,7 +16,7 @@ UW YAML for the ``shave:`` Block execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. config: ^^^^^^^ @@ -33,14 +33,17 @@ nh4: The number of halo rows/columns. - nx: """ The i/x dimensions of the compute domain (not including halo). - ny: """ -The j/y dimensions of the compute domain (not including halo) \ No newline at end of file +The j/y dimensions of the compute domain (not including halo) + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/components/ungrib.rst b/docs/sections/user_guide/yaml/components/ungrib.rst index a2b7f3d88..7db1df55d 100644 --- a/docs/sections/user_guide/yaml/components/ungrib.rst +++ b/docs/sections/user_guide/yaml/components/ungrib.rst @@ -5,18 +5,20 @@ ungrib Structured YAML to run the WRF preprocessing component ``ungrib`` is validated by JSON Schema and requires the ``ungrib:`` block, described below. If ``ungrib`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. +.. include:: ../../../../shared/injected_cycle.rst + Here is a prototype UW YAML ``ungrib:`` block, explained in detail below: .. highlight:: yaml .. literalinclude:: ../../../../shared/ungrib.yaml UW YAML for the ``ungrib:`` Block ----------------------------------------- +--------------------------------- execution: ^^^^^^^^^^ -See :ref:`here ` for details. +See :ref:`this page ` for details. gfs_files: ^^^^^^^^^^ @@ -46,7 +48,7 @@ An absolute-path template to the GRIB-formatted files to be processed by ``ungri run_dir: ^^^^^^^^ -The path to the directory where ``ungrib`` will find its namelist and write its outputs. +The path to the run directory. vtable: ^^^^^^^ diff --git a/docs/sections/user_guide/yaml/components/upp.rst b/docs/sections/user_guide/yaml/components/upp.rst new file mode 100644 index 000000000..822d35b85 --- /dev/null +++ b/docs/sections/user_guide/yaml/components/upp.rst @@ -0,0 +1,57 @@ +.. _upp_yaml: + +upp +=== + +Structured YAML to run the UPP post-processor is validated by JSON Schema and requires the ``upp:`` block, described below. If UPP is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. + +.. include:: ../../../../shared/injected_cycle.rst +.. include:: ../../../../shared/injected_leadtime.rst + +Here is a prototype UW YAML ``upp:`` block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: ../../../../shared/upp.yaml + +UW YAML for the ``upp:`` Block +------------------------------ + +execution: +^^^^^^^^^^ + +See :ref:`this page ` for details. + +files_to_copy: +^^^^^^^^^^^^^^ + +See :ref:`this page ` for details. + +files_to_link: +"""""""""""""" + +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. + +namelist_file: +"""""""""""""" + +Supports ``base_file:`` and ``update_values:`` blocks (see the :ref:`updating_values` for details). + +The following namelists and variables can be customized: + +.. list-table:: + :widths: 10 95 + :header-rows: 1 + + * - Namelist + - Variables + * - ``model_inputs`` + - ``datestr``, ``filename``, ``filenameflat``, ``filenameflux``, ``grib``, ``ioform``, ``modelname`` + * - ``nampgb`` + - ``aqf_on``, ``d2d_chem``, ``d3d_on``, ``filenameaer``, ``gccpp_on``, ``gocart_on``, ``gtg_on``, ``hyb_sigp``, ``kpo``, ``kpv``, ``kth``, ``method_blsn``, ``nasa_on``, ``numx``, ``po``, ``popascal``, ``pv``, ``rdaod``, ``slrutah_on``, ``th``, ``vtimeunits``, ``write_ifi_debug_files`` + +Read more on the UPP namelists, including variable meanings and appropriate values, `here `_. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 1e77b6d1b..248a07eb2 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -37,7 +37,6 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input % uw config realize --input-file input.yaml --output-format yaml f2: 5.859 - ``!int`` ^^^^^^^^ @@ -56,7 +55,6 @@ Converts the tagged node to a Python ``int`` value. For example, given ``input.y f2: 11 f2: 140 - ``!include`` ^^^^^^^^^^^^ @@ -80,7 +78,6 @@ and ``supplemental.yaml``: e: 2.718 pi: 3.141 - ``!remove`` ^^^^^^^^^^^ diff --git a/docs/shared/injected_cycle.rst b/docs/shared/injected_cycle.rst new file mode 100644 index 000000000..b9ec27bc3 --- /dev/null +++ b/docs/shared/injected_cycle.rst @@ -0,0 +1,13 @@ +* This driver receives a ``cycle`` argument, which it makes available as a Python ``datetime`` object to Jinja2 when realizing its input config. This supports specification of cycle-specific values. For example, the key-value pair + + .. code-block:: yaml + + gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc: /some/path/{{ cycle.strftime('%Y%m%d%H') }}/gfs.t{{ cycle.strftime('%H') }}z.atmanl.nc + + would be rendered as + + .. code-block:: yaml + + gfs.t18z.atmanl.nc: /some/path/2024021218/gfs.t18z.atmanl.nc + + for cycle ``2024-02-12T18``. diff --git a/docs/shared/injected_leadtime.rst b/docs/shared/injected_leadtime.rst new file mode 100644 index 000000000..c0740f8f8 --- /dev/null +++ b/docs/shared/injected_leadtime.rst @@ -0,0 +1,15 @@ +* This driver receives a ``leadtime`` argument, which it makes available as a Python ``timedelta`` object to Jinja2 when realizing its input config. This supports specification of leadtime-specific values. For example, the key-value pairs + + .. code-block:: yaml + + datestr: "{{ (cycle + leadtime).strftime('%Y-%m-%d_%H:%M:%S') }}" + suffix: f{{ '%03d' % (leadtime.total_seconds() / 3600) }} + + would be rendered as + + .. code-block:: yaml + + datestr: 2024-05-09_06:00:00 + suffix: f018 + + for cycle ``2024-05-08T12`` and leadtime ``18``. diff --git a/docs/shared/shave.yaml b/docs/shared/shave.yaml index 70fc752c2..927706f3f 100644 --- a/docs/shared/shave.yaml +++ b/docs/shared/shave.yaml @@ -1,14 +1,14 @@ shave: - execution: - batchargs: - cores: 1 - walltime: "00:01:00" - executable: /path/to/shave config: input_grid_file: /path/to/input/grid/file nh4: 1 nx: 214 ny: 128 + execution: + batchargs: + cores: 1 + walltime: "00:01:00" + executable: /path/to/shave run_dir: /path/to/dir/run platform: account: me diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml new file mode 100644 index 000000000..5c8ea99d4 --- /dev/null +++ b/docs/shared/upp.yaml @@ -0,0 +1,40 @@ +upp: + execution: + batchargs: + export: NONE + nodes: 1 + walltime: "00:05:00" + envcmds: + - module use /path/to/modulefiles + - module load runtime-module + - source /etc/profile.d/slurm.sh + executable: /path/to/upp.x + mpiargs: + - "--ntasks $SLURM_CPUS_ON_NODE" + mpicmd: srun + files_to_copy: + postxconfig-NT.txt: /path/to/postxconfig-NT.txt + files_to_link: + eta_micro_lookup.dat: /path/to/nam_micro_lookup.dat + params_grib2_tbl_new: /path/to/params_grib2_tbl_new + namelist: + base_file: /path/to/base.nml + update_values: + model_inputs: + datestr: "{{ (cycle + leadtime).strftime('%Y-%m-%d_%H:%M:%S') }}" + filename: /path/to/dynf{{ '%03d' % (leadtime.total_seconds() / 3600) }}.nc + filenameflux: /path/to/phyf{{ '%03d' % (leadtime.total_seconds() / 3600) }}.nc + grib: grib2 + ioform: netcdf + modelname: FV3R + nampgb: + kpo: 3 + numx: 1 + po: + - 1000 + - 100 + - 1 + run_dir: /path/to/run +platform: + account: me + scheduler: slurm diff --git a/docs/static/custom.css b/docs/static/custom.css index 859fe191f..cb22e007e 100644 --- a/docs/static/custom.css +++ b/docs/static/custom.css @@ -15,3 +15,9 @@ html.writer-html5 .rst-content table.docutils td>p { .wy-table-responsive { overflow-x: hidden; } + +/* Let table content word-wrap. */ + +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: inherit; +} diff --git a/src/uwtools/api/esg_grid.py b/src/uwtools/api/esg_grid.py index 5ddded361..4485432f3 100644 --- a/src/uwtools/api/esg_grid.py +++ b/src/uwtools/api/esg_grid.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/global_equiv_resol.py b/src/uwtools/api/global_equiv_resol.py index 0c1b394ab..0095c4f6c 100644 --- a/src/uwtools/api/global_equiv_resol.py +++ b/src/uwtools/api/global_equiv_resol.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/make_hgrid.py b/src/uwtools/api/make_hgrid.py index 3fb5ebb16..e241632b1 100644 --- a/src/uwtools/api/make_hgrid.py +++ b/src/uwtools/api/make_hgrid.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/sfc_climo_gen.py b/src/uwtools/api/sfc_climo_gen.py index 6d643554c..628d5e739 100644 --- a/src/uwtools/api/sfc_climo_gen.py +++ b/src/uwtools/api/sfc_climo_gen.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/shave.py b/src/uwtools/api/shave.py index 6c82fb053..da1c3f2a1 100644 --- a/src/uwtools/api/shave.py +++ b/src/uwtools/api/shave.py @@ -7,6 +7,6 @@ from uwtools.utils.api import make_execute as _make_execute from uwtools.utils.api import make_tasks as _make_tasks -execute = _make_execute(_Driver, with_cycle=False) +execute = _make_execute(_Driver) tasks = _make_tasks(_Driver) __all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/api/upp.py b/src/uwtools/api/upp.py new file mode 100644 index 000000000..08b993f78 --- /dev/null +++ b/src/uwtools/api/upp.py @@ -0,0 +1,12 @@ +""" +API access to the ``uwtools`` ``upp`` driver. +""" + +from uwtools.drivers.support import graph +from uwtools.drivers.upp import UPP as _Driver +from uwtools.utils.api import make_execute as _make_execute +from uwtools.utils.api import make_tasks as _make_tasks + +execute = _make_execute(_Driver, with_cycle=True, with_leadtime=True) +tasks = _make_tasks(_Driver) +__all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 2c616e320..7d75a608c 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -12,7 +12,7 @@ from functools import partial from importlib import import_module from pathlib import Path -from typing import Any, Callable, Dict, List, NoReturn, Tuple +from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple import uwtools.api import uwtools.api.config @@ -27,6 +27,7 @@ from uwtools.utils.file import get_file_format, resource_path FORMATS = FORMAT.extensions() +LEADTIME_DESC = "HH[:MM[:SS]]" TITLE_REQ_ARG = "Required arguments" Args = Dict[str, Any] @@ -74,6 +75,7 @@ def main() -> None: STR.sfcclimogen, STR.shave, STR.ungrib, + STR.upp, ] } modes = {**tools, **drivers} @@ -548,9 +550,11 @@ def _add_arg_config_file(group: Group, required: bool = False) -> None: def _add_arg_cycle(group: Group) -> None: + offset = dt.timedelta(hours=(dt.datetime.now(dt.UTC).hour // 6) * 6) + cycle = dt.datetime.combine(dt.date.today(), dt.datetime.min.time()) + offset group.add_argument( _switch(STR.cycle), - help="The cycle in ISO8601 format", + help="The cycle in ISO8601 format (e.g. %s)" % cycle.strftime("%Y-%m-%dT%H"), required=True, type=dt.datetime.fromisoformat, ) @@ -642,6 +646,15 @@ def _add_arg_keys(group: Group) -> None: ) +def _add_arg_leadtime(group: Group) -> None: + group.add_argument( + _switch(STR.leadtime), + help=f"The leadtime as {LEADTIME_DESC}", + required=True, + type=_timedelta_from_str, + ) + + def _add_arg_output_block(group: Group): group.add_argument( _switch(STR.outblock), @@ -810,25 +823,35 @@ def _add_subparser(subparsers: Subparsers, name: str, helpmsg: str) -> Parser: return parser -def _add_subparser_for_driver(name: str, subparsers: Subparsers, with_cycle: bool) -> ModeChecks: +def _add_subparser_for_driver( + name: str, + subparsers: Subparsers, + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, +) -> ModeChecks: """ Subparser for a driver mode. :param name: Name of the driver whose subparser to configure. :param subparsers: Parent parser's subparsers, to add this subparser to. :param with_cycle: Does this driver require a cycle? + :param with_leadtime: Does this driver require a leadtime? """ parser = _add_subparser(subparsers, name, "Execute %s tasks" % name) _basic_setup(parser) subparsers = _add_subparsers(parser, STR.action, STR.task.upper()) return { - task: _add_subparser_for_driver_task(subparsers, task, helpmsg, with_cycle) + task: _add_subparser_for_driver_task(subparsers, task, helpmsg, with_cycle, with_leadtime) for task, helpmsg in import_module("uwtools.api.%s" % name).tasks().items() } def _add_subparser_for_driver_task( - subparsers: Subparsers, task: str, helpmsg: str, with_cycle: bool + subparsers: Subparsers, + task: str, + helpmsg: str, + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, ) -> ActionChecks: """ Subparser for a driver action. @@ -837,11 +860,14 @@ def _add_subparser_for_driver_task( :param task: The task to add a subparser for. :param helpmsg: Help message for task. :param with_cycle: Does this driver require a cycle? + :param with_leadtime: Does this driver require a leadtime? """ parser = _add_subparser(subparsers, task, helpmsg.rstrip(".")) required = parser.add_argument_group(TITLE_REQ_ARG) if with_cycle: _add_arg_cycle(required) + if with_leadtime: + _add_arg_leadtime(required) optional = _basic_setup(parser) _add_arg_config_file(group=optional) _add_arg_batch(optional) @@ -935,6 +961,8 @@ def _dispatch_to_driver(name: str, args: Args) -> bool: } if cycle := args.get(STR.cycle): kwargs[STR.cycle] = cycle + if leadtime := args.get(STR.leadtime): + kwargs[STR.leadtime] = leadtime return execute(**kwargs) @@ -965,22 +993,35 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: STR.template: partial(_add_subparser_template, subparsers), } drivers = { - x: partial(_add_subparser_for_driver, x, subparsers, with_cycle) - for x, with_cycle in [ - (STR.chgrescube, True), - (STR.esggrid, False), - (STR.fv3, True), - (STR.globalequivresol, False), - (STR.jedi, True), - (STR.makehgrid, False), - (STR.mpas, True), - (STR.mpasinit, True), - (STR.sfcclimogen, False), - (STR.shave, False), - (STR.ungrib, True), + component: partial(_add_subparser_for_driver, component, subparsers) + for component in [ + STR.esggrid, + STR.globalequivresol, + STR.makehgrid, + STR.sfcclimogen, + STR.shave, + ] + } + drivers_with_cycle = { + component: partial(_add_subparser_for_driver, component, subparsers, with_cycle=True) + for component in [ + STR.chgrescube, + STR.fv3, + STR.jedi, + STR.mpas, + STR.mpasinit, + STR.ungrib, ] } - modes = {**tools, **drivers} + drivers_with_cycle_and_leadtime = { + component: partial( + _add_subparser_for_driver, component, subparsers, with_cycle=True, with_leadtime=True + ) + for component in [ + STR.upp, + ] + } + modes = {**tools, **drivers, **drivers_with_cycle, **drivers_with_cycle_and_leadtime} checks = {k: modes[k]() for k in sorted(modes.keys())} return vars(parser.parse_args(raw_args)), checks @@ -995,6 +1036,22 @@ def _switch(arg: str) -> str: return "--%s" % arg.replace("_", "-") +def _timedelta_from_str(tds: str) -> dt.timedelta: + """ + Return a timedelta parsed from a leadtime string. + + :param tds: The timedelta string to parse. + """ + fmts = ("%H:%M:%S", "%H:%M", "%H") + for fmt in fmts: + try: + t = dt.datetime.strptime(tds, fmt) + return dt.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) + except ValueError: + pass + _abort(f"Specify leadtime as {LEADTIME_DESC}") + + def _version() -> str: """ Return version information. diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 3dde13ce0..2c56cd0ef 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -6,7 +6,7 @@ import re import stat from abc import ABC, abstractmethod -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from textwrap import dedent from typing import Any, Dict, List, Optional, Type @@ -16,7 +16,7 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.validator import validate_internal -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.scheduler import JobScheduler from uwtools.utils.processing import execute @@ -33,6 +33,7 @@ def __init__( dry_run: bool = False, batch: bool = False, cycle: Optional[datetime] = None, + leadtime: Optional[timedelta] = None, ) -> None: """ A component driver. @@ -41,13 +42,19 @@ def __init__( :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? :param cycle: The cycle. + :param leadtime: The leadtime. """ self._config = YAMLConfig(config=config) self._dry_run = dry_run self._batch = batch - self._config.dereference() + if leadtime and not cycle: + raise UWError("When leadtime is specified, cycle is required") self._config.dereference( - context={**({"cycle": cycle} if cycle else {}), **self._config.data} + context={ + **({"cycle": cycle} if cycle else {}), + **({"leadtime": leadtime} if leadtime else {}), + **self._config.data, + } ) self._validate() @@ -163,6 +170,7 @@ def _resources(self) -> Dict[str, Any]: "account": platform["account"], "rundir": self._rundir, "scheduler": platform["scheduler"], + "stdout": "%s.out" % self._runscript_path.name, # config may override **self._driver_config.get("execution", {}).get("batchargs", {}), } @@ -254,7 +262,7 @@ def _write_runscript(self, path: Path, envvars: Dict[str, str]) -> None: envvars=envvars, execution=[ "time %s" % self._runcmd, - "test $? -eq 0 && touch %s/done.%s" % (self._rundir, self._driver_name), + "test $? -eq 0 && touch %s.done" % self._runscript_path.name, ], scheduler=self._scheduler if self._batch else None, ) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py new file mode 100644 index 000000000..e4d9d9e93 --- /dev/null +++ b/src/uwtools/drivers/upp.py @@ -0,0 +1,162 @@ +""" +A driver for UPP. +""" + +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from iotaa import asset, dryrun, task, tasks + +from uwtools.config.formats.nml import NMLConfig +from uwtools.drivers.driver import Driver +from uwtools.strings import STR +from uwtools.utils.tasks import filecopy, symlink + + +class UPP(Driver): + """ + A driver for UPP. + """ + + def __init__( + self, + cycle: datetime, + leadtime: timedelta, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + ): + """ + The driver. + + :param cycle: The cycle. + :param leadtime: The leadtime. + :param config: Path to config file (read stdin if missing or None). + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + """ + super().__init__( + config=config, + dry_run=dry_run, + batch=batch, + cycle=cycle, + leadtime=leadtime, + ) + if self._dry_run: + dryrun() + self._cycle = cycle + self._leadtime = leadtime + + # Workflow tasks + + @tasks + def files_copied(self): + """ + Files copied for run. + """ + yield self._taskname("files copied") + yield [ + filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() + ] + + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self._taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] + + @task + def namelist_file(self): + """ + The namelist file. + """ + path = self._namelist_path + yield self._taskname(str(path)) + yield asset(path, path.is_file) + yield None + path.parent.mkdir(parents=True, exist_ok=True) + self._create_user_updated_config( + config_class=NMLConfig, + config_values=self._driver_config["namelist"], + path=path, + ) + + @tasks + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [ + self.files_copied(), + self.files_linked(), + self.namelist_file(), + self.runscript(), + ] + + @task + def runscript(self): + """ + The runscript. + """ + path = self._runscript_path + yield self._taskname(path.name) + yield asset(path, path.is_file) + yield None + self._write_runscript(path=path, envvars={}) + + # Private helper methods + + @property + def _driver_name(self) -> str: + """ + Returns the name of this driver. + """ + return STR.upp + + @property + def _namelist_path(self) -> Path: + """ + Path to the namelist file. + """ + return self._rundir / "itag" + + @property + def _runscript_path(self) -> Path: + """ + Path to the runscript. + """ + return self._rundir / f"runscript.{self._driver_name}" + + @property + def _runcmd(self) -> str: + """ + Returns the full command-line component invocation. + """ + execution = self._driver_config.get("execution", {}) + mpiargs = execution.get("mpiargs", []) + components = [ + execution.get("mpicmd"), + *[str(x) for x in mpiargs], + "%s < %s" % (execution["executable"], self._namelist_path.name), + ] + return " ".join(filter(None, components)) + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return "%s %s %s" % ( + (self._cycle + self._leadtime).strftime("%Y%m%d %H:%M:%S"), + self._driver_name, + suffix, + ) diff --git a/src/uwtools/resources/jsonschema/upp.jsonschema b/src/uwtools/resources/jsonschema/upp.jsonschema new file mode 100644 index 000000000..ad66caf83 --- /dev/null +++ b/src/uwtools/resources/jsonschema/upp.jsonschema @@ -0,0 +1,188 @@ +{ + "properties": { + "upp": { + "additionalProperties": false, + "properties": { + "execution": { + "$ref": "urn:uwtools:execution" + }, + "files_to_copy": { + "$ref": "urn:uwtools:files-to-stage" + }, + "files_to_link": { + "$ref": "urn:uwtools:files-to-stage" + }, + "namelist": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { + "base_file": { + "type": "string" + }, + "update_values": { + "additionalProperties": false, + "properties": { + "model_inputs": { + "additionalProperties": false, + "properties": { + "datestr": { + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}:[0-9]{2}:[0-9]{2}$", + "type": "string" + }, + "filename": { + "maxLength": 256, + "type": "string" + }, + "filenameflat": { + "maxLength": 256, + "type": "string" + }, + "filenameflux": { + "maxLength": 256, + "type": "string" + }, + "grib": { + "enum": [ + "grib2" + ], + "type": "string" + }, + "ioform": { + "enum": [ + "binarynemsio", + "netcdf" + ], + "type": "string" + }, + "modelname": { + "enum": [ + "FV3R", + "3DRTMA", + "GFS", + "RAPR", + "NMM" + ], + "type": "string" + } + }, + "type": "object" + }, + "nampgb": { + "additionalProperties": false, + "properties": { + "aqf_on": { + "type": "boolean" + }, + "d2d_chem": { + "type": "boolean" + }, + "d3d_on": { + "type": "boolean" + }, + "filenameaer": { + "maxLength": 256, + "type": "string" + }, + "gccpp_on": { + "type": "boolean" + }, + "gocart_on": { + "type": "boolean" + }, + "gtg_on": { + "type": "boolean" + }, + "hyb_sigp": { + "type": "boolean" + }, + "kpo": { + "type": "integer" + }, + "kpv": { + "type": "integer" + }, + "kth": { + "type": "integer" + }, + "method_blsn": { + "type": "boolean" + }, + "nasa_on": { + "type": "boolean" + }, + "numx": { + "type": "integer" + }, + "po": { + "items": { + "type": "number" + }, + "maxItems": 70, + "type": "array" + }, + "popascal": { + "type": "boolean" + }, + "pv": { + "items": { + "type": "number" + }, + "maxItems": 70, + "type": "array" + }, + "rdaod": { + "type": "boolean" + }, + "slrutah_on": { + "type": "boolean" + }, + "th": { + "items": { + "type": "number" + }, + "maxItems": 70, + "type": "array" + }, + "vtimeunits": { + "enum": [ + "FMIN" + ], + "type": "string" + }, + "write_ifi_debug_files": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "run_dir": { + "type": "string" + } + }, + "required": [ + "execution", + "namelist", + "run_dir" + ], + "type": "object" + } + }, + "type": "object" +} diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 56be93668..14b27974b 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -86,6 +86,7 @@ class STR: jedi: str = "jedi" keys: str = "keys" keyvalpairs: str = "key_eq_val_pairs" + leadtime: str = "leadtime" link: str = "link" makehgrid: str = "make_hgrid" mode: str = "mode" @@ -112,6 +113,7 @@ class STR: total: str = "total" translate: str = "translate" ungrib: str = "ungrib" + upp: str = "upp" validate: str = "validate" valsfile: str = "values_file" valsfmt: str = "values_format" diff --git a/src/uwtools/tests/api/test_drivers.py b/src/uwtools/tests/api/test_drivers.py index aaa6e32f9..d7652f39c 100644 --- a/src/uwtools/tests/api/test_drivers.py +++ b/src/uwtools/tests/api/test_drivers.py @@ -1,6 +1,6 @@ # pylint: disable=missing-function-docstring,protected-access -from datetime import datetime as dt +import datetime as dt from unittest.mock import patch import iotaa @@ -18,6 +18,7 @@ sfc_climo_gen, shave, ungrib, + upp, ) from uwtools.drivers import support from uwtools.utils import api @@ -26,6 +27,7 @@ chgres_cube, esg_grid, fv3, + global_equiv_resol, jedi, make_hgrid, mpas, @@ -33,8 +35,10 @@ sfc_climo_gen, shave, ungrib, + upp, ] -nocycle = [esg_grid, global_equiv_resol, make_hgrid, sfc_climo_gen, shave] +with_cycle = [chgres_cube, fv3, jedi, mpas, mpas_init, ungrib, upp] +with_leadtime = [upp] @pytest.mark.parametrize("module", modules) @@ -47,12 +51,17 @@ def test_api_execute(module): "stdin_ok": True, "task": "foo", } - kwargs = kwbase if module in nocycle else {"cycle": dt.now(), **kwbase} + kwargs = { + **kwbase, + **({"cycle": dt.datetime.now()} if module in with_cycle else {}), + **({"leadtime": dt.timedelta(hours=24)} if module in with_leadtime else {}), + } with patch.object(api, "_execute") as _execute: module.execute(**kwargs) _execute.assert_called_once_with( driver_class=module._Driver, - cycle=None if module in nocycle else kwargs["cycle"], + cycle=kwargs["cycle"] if module in with_cycle else None, + leadtime=kwargs["leadtime"] if module in with_leadtime else None, **kwbase ) diff --git a/src/uwtools/tests/drivers/test_chgres_cube.py b/src/uwtools/tests/drivers/test_chgres_cube.py index 4d01455f6..2add920fd 100644 --- a/src/uwtools/tests/drivers/test_chgres_cube.py +++ b/src/uwtools/tests/drivers/test_chgres_cube.py @@ -80,7 +80,7 @@ def driverobj(config_file, cycle): return chgres_cube.ChgresCube(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_ChgresCube(driverobj): diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index fa54f57f0..e0b9aaee1 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -2,9 +2,9 @@ """ Tests for uwtools.drivers.driver module. """ +import datetime as dt import json import logging -from datetime import datetime from pathlib import Path from textwrap import dedent from unittest.mock import Mock, PropertyMock, patch @@ -16,7 +16,7 @@ from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers import driver -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.tests.support import regex_logged @@ -57,34 +57,41 @@ def write(path, s): @fixture -def driverobj(tmp_path): - cf = write( - tmp_path / "good.yaml", - { - "concrete": { - "base_file": str(write(tmp_path / "base.yaml", {"a": 11, "b": 22})), - "execution": { - "batchargs": { - "export": "NONE", - "nodes": 1, - "stdout": "{{ concrete.run_dir }}/out", - "walltime": "00:05:00", - }, - "executable": str(tmp_path / "qux"), - "mpiargs": ["bar", "baz"], - "mpicmd": "foo", +def config(tmp_path): + return { + "concrete": { + "base_file": str(write(tmp_path / "base.yaml", {"a": 11, "b": 22})), + "execution": { + "batchargs": { + "export": "NONE", + "nodes": 1, + "stdout": "{{ concrete.run_dir }}/out", + "walltime": "00:05:00", }, - "run_dir": "{{ rootdir }}/{{ cycle.strftime('%Y%m%d%H') }}/run", - "update_values": {"a": 33}, + "executable": str(tmp_path / "qux"), + "mpiargs": ["bar", "baz"], + "mpicmd": "foo", }, - "platform": { - "account": "me", - "scheduler": "slurm", - }, - "rootdir": "/path/to", + "run_dir": "{{ rootdir }}/{{ cycle.strftime('%Y%m%d%H') }}/run", + "update_values": {"a": 33}, + }, + "platform": { + "account": "me", + "scheduler": "slurm", }, + "rootdir": "/path/to", + } + + +@fixture +def driverobj(config): + return ConcreteDriver( + config=config, + dry_run=True, + batch=True, + cycle=dt.datetime(2024, 3, 22, 18), + leadtime=dt.timedelta(hours=24), ) - return ConcreteDriver(config=cf, dry_run=True, batch=True, cycle=datetime(2024, 3, 22, 18)) # Tests @@ -96,6 +103,12 @@ def test_Driver(driverobj): assert driverobj._batch is True +def test_Driver_cycle_leadtime_error(config): + with raises(UWError) as e: + ConcreteDriver(config=config, leadtime=dt.timedelta(hours=24)) + assert "When leadtime is specified, cycle is required" in str(e) + + # Tests for workflow methods @@ -207,6 +220,7 @@ def test_Driver__resources_pass(driverobj): "account": account, "rundir": driverobj._rundir, "scheduler": scheduler, + "stdout": "runscript.concrete.out", "walltime": walltime, } @@ -301,7 +315,7 @@ def test_Driver__write_runscript(driverobj, tmp_path): export BAZ=qux time foo bar baz {executable} - test $? -eq 0 && touch /path/to/2024032218/run/done.concrete + test $? -eq 0 && touch runscript.concrete.done """ with open(path, "r", encoding="utf-8") as f: actual = f.read() diff --git a/src/uwtools/tests/drivers/test_esg_grid.py b/src/uwtools/tests/drivers/test_esg_grid.py index 2b9d8e905..017629ef1 100644 --- a/src/uwtools/tests/drivers/test_esg_grid.py +++ b/src/uwtools/tests/drivers/test_esg_grid.py @@ -12,7 +12,7 @@ from uwtools.drivers import esg_grid from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -63,7 +63,7 @@ def driverobj(config_file): return esg_grid.ESGGrid(config=config_file, batch=True) -# Driver tests +# Tests def test_ESGGrid(driverobj): diff --git a/src/uwtools/tests/drivers/test_fv3.py b/src/uwtools/tests/drivers/test_fv3.py index ee44be6d2..4cf361fb0 100644 --- a/src/uwtools/tests/drivers/test_fv3.py +++ b/src/uwtools/tests/drivers/test_fv3.py @@ -84,7 +84,7 @@ def true(): return true -# Driver tests +# Tests def test_FV3(driverobj): diff --git a/src/uwtools/tests/drivers/test_global_equiv_resol.py b/src/uwtools/tests/drivers/test_global_equiv_resol.py index 818174f0e..f6afef1b7 100644 --- a/src/uwtools/tests/drivers/test_global_equiv_resol.py +++ b/src/uwtools/tests/drivers/test_global_equiv_resol.py @@ -12,7 +12,7 @@ from uwtools.drivers import global_equiv_resol from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -49,7 +49,7 @@ def driverobj(config_file): return global_equiv_resol.GlobalEquivResol(config=config_file, batch=True) -# Driver tests +# Tests def test_GlobalEquivResol(driverobj): diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index f1ccb91c3..79d9718c4 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -76,12 +68,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return jedi.JEDI(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_JEDI(driverobj): diff --git a/src/uwtools/tests/drivers/test_make_hgrid.py b/src/uwtools/tests/drivers/test_make_hgrid.py index 1644d3e45..5b63bac25 100644 --- a/src/uwtools/tests/drivers/test_make_hgrid.py +++ b/src/uwtools/tests/drivers/test_make_hgrid.py @@ -11,7 +11,7 @@ from uwtools.drivers import make_hgrid from uwtools.scheduler import Slurm -# Driver fixtures +# Fixtures @fixture @@ -54,7 +54,7 @@ def driverobj(config_file): return make_hgrid.MakeHgrid(config=config_file, batch=True) -# Driver tests +# Tests def test_MakeHgrid(driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas.py b/src/uwtools/tests/drivers/test_mpas.py index 3e88ced0b..88dc08dde 100644 --- a/src/uwtools/tests/drivers/test_mpas.py +++ b/src/uwtools/tests/drivers/test_mpas.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 3, 22, 6) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -77,12 +69,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 3, 22, 6) + + @fixture def driverobj(config_file, cycle): return mpas.MPAS(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_MPAS(driverobj): diff --git a/src/uwtools/tests/drivers/test_mpas_init.py b/src/uwtools/tests/drivers/test_mpas_init.py index ac60028ec..863d1bb64 100644 --- a/src/uwtools/tests/drivers/test_mpas_init.py +++ b/src/uwtools/tests/drivers/test_mpas_init.py @@ -20,14 +20,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -88,12 +80,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return mpas_init.MPASInit(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_MPASInit(driverobj): diff --git a/src/uwtools/tests/drivers/test_sfc_climo_gen.py b/src/uwtools/tests/drivers/test_sfc_climo_gen.py index 3c98f5d5d..b6de46322 100644 --- a/src/uwtools/tests/drivers/test_sfc_climo_gen.py +++ b/src/uwtools/tests/drivers/test_sfc_climo_gen.py @@ -13,6 +13,8 @@ from uwtools.drivers import sfc_climo_gen from uwtools.scheduler import Slurm +# Fixtures + config: dict = { "sfc_climo_gen": { "execution": { @@ -71,7 +73,7 @@ def driverobj(config_file): return sfc_climo_gen.SfcClimoGen(config=config_file, batch=True) -# Driver tests +# Tests def test_SfcClimoGen(driverobj): diff --git a/src/uwtools/tests/drivers/test_ungrib.py b/src/uwtools/tests/drivers/test_ungrib.py index d96270120..9dca1ed1c 100644 --- a/src/uwtools/tests/drivers/test_ungrib.py +++ b/src/uwtools/tests/drivers/test_ungrib.py @@ -16,14 +16,6 @@ # Fixtures -@fixture -def cycle(): - return dt.datetime(2024, 2, 1, 18) - - -# Driver fixtures - - @fixture def config(tmp_path): return { @@ -45,7 +37,7 @@ def config(tmp_path): "vtable": str(tmp_path / "Vtable.GFS"), }, "platform": { - "account": "wrfruc", + "account": "me", "scheduler": "slurm", }, } @@ -59,12 +51,17 @@ def config_file(config, tmp_path): return path +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + @fixture def driverobj(config_file, cycle): return ungrib.Ungrib(config=config_file, cycle=cycle, batch=True) -# Driver tests +# Tests def test_Ungrib(driverobj): diff --git a/src/uwtools/tests/drivers/test_upp.py b/src/uwtools/tests/drivers/test_upp.py new file mode 100644 index 000000000..50487c703 --- /dev/null +++ b/src/uwtools/tests/drivers/test_upp.py @@ -0,0 +1,186 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +UPP driver tests. +""" +import datetime as dt +from pathlib import Path +from unittest.mock import DEFAULT as D +from unittest.mock import patch + +import f90nml # type: ignore +import yaml +from pytest import fixture + +from uwtools.drivers import upp +from uwtools.scheduler import Slurm + +# Fixtures + + +@fixture +def config(tmp_path): + return { + "upp": { + "execution": { + "batchargs": { + "cores": 1, + "walltime": "00:01:00", + }, + "executable": str(tmp_path / "upp.exe"), + }, + "files_to_copy": { + "foo": str(tmp_path / "foo"), + "bar": str(tmp_path / "bar"), + }, + "files_to_link": { + "baz": str(tmp_path / "baz"), + "qux": str(tmp_path / "qux"), + }, + "namelist": { + "base_file": str(tmp_path / "base.nml"), + "update_values": { + "model_inputs": { + "grib": "grib2", + }, + "nampgb": { + "kpo": 3, + }, + }, + }, + "run_dir": str(tmp_path / "run"), + }, + "platform": { + "account": "me", + "scheduler": "slurm", + }, + } + + +@fixture +def config_file(config, tmp_path): + path = tmp_path / "config.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.dump(config, f) + return path + + +@fixture +def cycle(): + return dt.datetime(2024, 5, 6, 12) + + +@fixture +def driverobj(config_file, cycle, leadtime): + return upp.UPP(config=config_file, cycle=cycle, leadtime=leadtime, batch=True) + + +@fixture +def leadtime(): + return dt.timedelta(hours=24) + + +# Tests + + +def test_UPP(driverobj): + assert isinstance(driverobj, upp.UPP) + + +def test_UPP_dry_run(config_file, cycle, leadtime): + with patch.object(upp, "dryrun") as dryrun: + driverobj = upp.UPP( + config=config_file, cycle=cycle, leadtime=leadtime, batch=True, dry_run=True + ) + assert driverobj._dry_run is True + dryrun.assert_called_once_with() + + +def test_UPP_files_copied(driverobj): + for _, src in driverobj._driver_config["files_to_copy"].items(): + Path(src).touch() + for dst, _ in driverobj._driver_config["files_to_copy"].items(): + assert not Path(driverobj._rundir / dst).is_file() + driverobj.files_copied() + for dst, _ in driverobj._driver_config["files_to_copy"].items(): + assert Path(driverobj._rundir / dst).is_file() + + +def test_UPP_files_linked(driverobj): + for _, src in driverobj._driver_config["files_to_link"].items(): + Path(src).touch() + for dst, _ in driverobj._driver_config["files_to_link"].items(): + assert not Path(driverobj._rundir / dst).is_file() + driverobj.files_linked() + for dst, _ in driverobj._driver_config["files_to_link"].items(): + assert Path(driverobj._rundir / dst).is_symlink() + + +def test_UPP_namelist_file(driverobj): + datestr = "2024-05-05_12:00:00" + with open(driverobj._driver_config["namelist"]["base_file"], "w", encoding="utf-8") as f: + print("&model_inputs datestr='%s' / &nampgb kpv=88 /" % datestr, file=f) + dst = driverobj._rundir / "itag" + assert not dst.is_file() + driverobj.namelist_file() + assert dst.is_file() + nml = f90nml.read(dst) + assert isinstance(nml, f90nml.Namelist) + assert nml["model_inputs"]["datestr"] == datestr + assert nml["model_inputs"]["grib"] == "grib2" + assert nml["nampgb"]["kpo"] == 3 + assert nml["nampgb"]["kpv"] == 88 + + +def test_UPP_provisioned_run_directory(driverobj): + with patch.multiple( + driverobj, + files_copied=D, + files_linked=D, + namelist_file=D, + runscript=D, + ) as mocks: + driverobj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_UPP_run_batch(driverobj): + with patch.object(driverobj, "_run_via_batch_submission") as func: + driverobj.run() + func.assert_called_once_with() + + +def test_UPP_run_local(driverobj): + driverobj._batch = False + with patch.object(driverobj, "_run_via_local_execution") as func: + driverobj.run() + func.assert_called_once_with() + + +def test_UPP_runscript(driverobj): + with patch.object(driverobj, "_runscript") as runscript: + driverobj.runscript() + runscript.assert_called_once() + args = ("envcmds", "envvars", "execution", "scheduler") + types = [list, dict, list, Slurm] + assert [type(runscript.call_args.kwargs[x]) for x in args] == types + + +def test_UPP__driver_config(driverobj): + assert driverobj._driver_config == driverobj._config["upp"] + + +def test_UPP__taskname(driverobj): + assert driverobj._taskname("foo") == "20240507 12:00:00 upp foo" + + +def test_UPP__validate(driverobj): + driverobj._validate() + + +def test_UPP__namelist_path(driverobj): + assert driverobj._namelist_path == driverobj._rundir / "itag" + + +def test_UPP__runscript_path(driverobj): + assert driverobj._runscript_path == driverobj._rundir / "runscript.upp" diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 77c0d3c62..aeb79bae6 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -558,11 +558,13 @@ def test__dispatch_template_translate_no_optional(): def test__dispatch_to_driver(): name = "adriver" cycle = dt.datetime.now() + leadtime = dt.timedelta(hours=24) args: dict = { "action": "foo", "batch": True, "config_file": "config.yaml", "cycle": cycle, + "leadtime": leadtime, "dry_run": False, "graph_file": None, "stdin_ok": True, @@ -574,6 +576,7 @@ def test__dispatch_to_driver(): batch=True, config="config.yaml", cycle=cycle, + leadtime=leadtime, dry_run=False, graph_file=None, task="foo", @@ -650,5 +653,14 @@ def test__switch(): assert cli._switch("foo_bar") == "--foo-bar" +def test__timedelta_from_str(capsys): + assert cli._timedelta_from_str("11:12:13") == dt.timedelta(hours=11, minutes=12, seconds=13) + assert cli._timedelta_from_str("11:12") == dt.timedelta(hours=11, minutes=12) + assert cli._timedelta_from_str("11") == dt.timedelta(hours=11) + with raises(SystemExit): + cli._timedelta_from_str("foo") + assert f"Specify leadtime as {cli.LEADTIME_DESC}" in capsys.readouterr().err + + def test__version(): assert re.match(r"version \d+\.\d+\.\d+ build \d+", cli._version()) diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 67d1f43ac..29c791212 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -78,6 +78,11 @@ def ungrib_prop(): return partial(schema_validator, "ungrib", "properties", "ungrib", "properties") +@fixture +def upp_prop(): + return partial(schema_validator, "upp", "properties", "upp", "properties") + + # chgres-cube @@ -859,3 +864,134 @@ def test_schema_ungrib_run_dir(ungrib_prop): # Must be a string: assert not errors("/some/path") assert "88 is not of type 'string'" in errors(88) + + +# upp + + +def test_schema_upp(): + config = { + "execution": { + "batchargs": { + "cores": 1, + "walltime": "00:01:00", + }, + "executable": "/path/to/upp.exe", + }, + "namelist": { + "base_file": "/path/to/base.nml", + "update_values": { + "model_inputs": { + "grib": "grib2", + }, + "nampgb": { + "kpo": 3, + }, + }, + }, + "run_dir": "/path/to/run", + } + errors = schema_validator("upp", "properties", "upp") + # Basic correctness: + assert not errors(config) + # Some top-level keys are required: + for key in ("execution", "namelist", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Other top-level keys are optional: + assert not errors({**config, "files_to_copy": {"dst": "src"}}) + assert not errors({**config, "files_to_link": {"dst": "src"}}) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) + + +def test_schema_upp_namelist(upp_prop): + maxpathlen = 256 + errors = upp_prop("namelist") + # At least one of base_file or update_values is required: + assert "is not valid" in errors({}) + # Just base_file is ok: + assert not errors({"base_file": "/path/to/base.nml"}) + # Just update_values is ok: + assert not errors({"update_values": {"model_inputs": {"grib": "grib2"}}}) + # Both base_file and update_values are ok: + assert not errors( + {"base_file": "/path/to/base.nml", "update_values": {"model_inputs": {"grib": "grib2"}}} + ) + # Only two specific namelists are allowed: + assert "Additional properties are not allowed" in errors( + {"udpate_values": {"another_namelist": {}}} + ) + # model_inputs: datestr requires a specific format: + assert not errors({"update_values": {"model_inputs": {"datestr": "2024-05-06_12:00:00"}}}) + assert "does not match" in errors( + {"update_values": {"model_inputs": {"datestr": "2024-05-06T12:00:00"}}} + ) + # model_inputs: String pathnames have a max length: + for key in ["filename", "filenameflat", "filenameflux"]: + assert not errors({"update_values": {"model_inputs": {key: "c" * maxpathlen}}}) + assert "too long" in errors( + {"update_values": {"model_inputs": {key: "c" * (maxpathlen + 1)}}} + ) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {key: 88}}}) + # model_inputs: Only one grib value is supported: + assert "not one of ['grib2']" in errors({"update_values": {"model_inputs": {"grib": "grib1"}}}) + assert "not of type 'string'" in errors({"update_values": {"model_inputs": {"grib": 88}}}) + # model_inputs: Only certain ioform values are supported: + assert "not one of ['binarynemsio', 'netcdf']" in errors( + {"update_values": {"model_inputs": {"ioform": "jpg"}}} + ) + # model_inputs: Only certain modelname values are supported: + assert "not one of ['FV3R', '3DRTMA', 'GFS', 'RAPR', 'NMM']" in errors( + {"update_values": {"model_inputs": {"modelname": "foo"}}} + ) + # model_inputs: No other keys are supported: + assert "Additional properties are not allowed" in errors( + {"update_values": {"model_inputs": {"something": "else"}}} + ) + # nampgb: Some boolean keys are supported: + for key in [ + "aqf_on", + "d2d_chem", + "gccpp_on", + "gocart_on", + "gtg_on", + "hyb_sigp", + "method_blsn", + "nasa_on", + "popascal", + "rdaod", + "slrutah_on", + "write_ifi_debug_files", + ]: + assert not errors({"update_values": {"nampgb": {key: True}}}) + assert "not of type 'boolean'" in errors({"update_values": {"nampgb": {key: 88}}}) + # nampgb: String pathnames have a max length: + for key in ["filenameaer"]: + assert not errors({"update_values": {"nampgb": {key: "c" * maxpathlen}}}) + assert "too long" in errors({"update_values": {"nampgb": {key: "c" * (maxpathlen + 1)}}}) + assert "not of type 'string'" in errors({"update_values": {"nampgb": {key: 88}}}) + # nampgb: Some integer keys are supported: + for key in ["kpo", "kpv", "kth", "numx"]: + assert not errors({"update_values": {"nampgb": {key: 88}}}) + assert "not of type 'integer'" in errors({"update_values": {"nampgb": {key: True}}}) + # nampgb: Some arrays of numbers are supported: + nitems = 70 + for key in ["po", "pv", "th"]: + assert not errors({"update_values": {"nampgb": {key: [3.14] * nitems}}}) + assert "too long" in errors({"update_values": {"nampgb": {key: [3.14] * (nitems + 1)}}}) + assert "not of type 'number'" in errors( + {"update_values": {"nampgb": {key: [True] * nitems}}} + ) + # nampgb: Only one vtimeunits value is supported: + assert "not one of ['FMIN']" in errors({"update_values": {"nampgb": {"vtimeunits": "FOO"}}}) + # nampgb: No other keys are supported: + assert "Additional properties are not allowed" in errors( + {"update_values": {"nampgb": {"something": "else"}}} + ) + + +def test_schema_upp_run_dir(upp_prop): + errors = upp_prop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index 0cbe3985b..d35360be6 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -1,6 +1,6 @@ # pylint: disable=missing-function-docstring,protected-access,redefined-outer-name -from datetime import datetime as dt +import datetime as dt from pathlib import Path from unittest.mock import patch @@ -56,22 +56,48 @@ def test_make_execute(execute_kwargs): assert ":param task:" in func.__doc__ with patch.object(api, "_execute", return_value=True) as _execute: assert func(**execute_kwargs) is True - _execute.assert_called_once_with(driver_class=ConcreteDriver, cycle=None, **execute_kwargs) + _execute.assert_called_once_with( + driver_class=ConcreteDriver, cycle=None, leadtime=None, **execute_kwargs + ) def test_make_execute_cycle(execute_kwargs): - execute_kwargs["cycle"] = dt.now() + execute_kwargs["cycle"] = dt.datetime.now() func = api.make_execute(driver_class=ConcreteDriver, with_cycle=True) assert func.__name__ == "execute" assert func.__doc__ is not None assert ":param cycle:" in func.__doc__ assert ":param driver_class:" not in func.__doc__ assert ":param task:" in func.__doc__ + with patch.object(api, "_execute", return_value=True) as _execute: + assert func(**execute_kwargs) is True + _execute.assert_called_once_with( + driver_class=ConcreteDriver, leadtime=None, **execute_kwargs + ) + + +def test_make_execute_cycle_leadtime(execute_kwargs): + execute_kwargs["cycle"] = dt.datetime.now() + execute_kwargs["leadtime"] = dt.timedelta(hours=24) + func = api.make_execute(driver_class=ConcreteDriver, with_cycle=True, with_leadtime=True) + assert func.__name__ == "execute" + assert func.__doc__ is not None + assert ":param cycle:" in func.__doc__ + assert ":param leadtime:" in func.__doc__ + assert ":param driver_class:" not in func.__doc__ + assert ":param task:" in func.__doc__ with patch.object(api, "_execute", return_value=True) as _execute: assert func(**execute_kwargs) is True _execute.assert_called_once_with(driver_class=ConcreteDriver, **execute_kwargs) +def test_make_execute_leadtime_no_cycle_error(execute_kwargs): + execute_kwargs["leadtime"] = dt.timedelta(hours=24) + with raises(UWError) as e: + api.make_execute(driver_class=ConcreteDriver, with_leadtime=True) + assert "When leadtime is specified, cycle is required" in str(e) + + def test_make_tasks(): func = api.make_tasks(driver_class=ConcreteDriver) assert func.__name__ == "tasks" @@ -100,7 +126,8 @@ def test__execute(execute_kwargs, tmp_path): **execute_kwargs, "driver_class": ConcreteDriver, "config": config, - "cycle": dt.now(), + "cycle": dt.datetime.now(), + "leadtime": dt.timedelta(hours=24), "graph_file": graph_file, } assert not graph_file.is_file() diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index 166f0d70f..a3abc0ea5 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -32,12 +32,17 @@ def ensure_data_source( return str2path(data_source) -def make_execute(driver_class: type[Driver], with_cycle: bool) -> Callable[..., bool]: +def make_execute( + driver_class: type[Driver], + with_cycle: Optional[bool] = False, + with_leadtime: Optional[bool] = False, +) -> Callable[..., bool]: """ Returns a function that executes tasks for the given driver. :param driver_class: The driver class whose tasks to execute. - :param with_cycle: Does the driver's constructor take the 'cycle' parameter? + :param with_cycle: Does the driver's constructor take a 'cycle' parameter? + :param with_leadtime: Does the driver's constructor take a 'leadtime' parameter? """ def execute( # pylint: disable=unused-argument @@ -52,6 +57,7 @@ def execute( # pylint: disable=unused-argument driver_class=driver_class, task=task, cycle=None, + leadtime=None, config=config, batch=batch, dry_run=dry_run, @@ -67,11 +73,34 @@ def execute_cycle( # pylint: disable=unused-argument dry_run: bool = False, graph_file: Optional[Union[Path, str]] = None, stdin_ok: bool = False, + ) -> bool: + return _execute( + driver_class=driver_class, + task=task, + leadtime=None, + cycle=cycle, + config=config, + batch=batch, + dry_run=dry_run, + graph_file=graph_file, + stdin_ok=stdin_ok, + ) + + def execute_cycle_leadtime( # pylint: disable=unused-argument + task: str, + cycle: dt.datetime, + leadtime: dt.timedelta, + config: Optional[Union[Path, str]] = None, + batch: bool = False, + dry_run: bool = False, + graph_file: Optional[Union[Path, str]] = None, + stdin_ok: bool = False, ) -> bool: return _execute( driver_class=driver_class, task=task, cycle=cycle, + leadtime=leadtime, config=config, batch=batch, dry_run=dry_run, @@ -79,12 +108,21 @@ def execute_cycle( # pylint: disable=unused-argument stdin_ok=stdin_ok, ) + execute_cycle_leadtime.__name__ = "execute" execute_cycle.__name__ = "execute" assert _execute.__doc__ is not None - execute_cycle.__doc__ = re.sub(r"\n *:param driver_class:.*\n", "\n", _execute.__doc__) + execute_cycle_leadtime.__doc__ = re.sub(r"\n *:param driver_class:.*\n", "\n", _execute.__doc__) + execute_cycle.__doc__ = re.sub( + r"\n *:param leadtime:.*\n", "\n", execute_cycle_leadtime.__doc__ + ) execute.__doc__ = re.sub(r"\n *:param cycle:.*\n", "\n", execute_cycle.__doc__) + if with_leadtime and not with_cycle: + raise UWError("When leadtime is specified, cycle is required") + if with_cycle: + if with_leadtime: + return execute_cycle_leadtime return execute_cycle return execute @@ -121,6 +159,7 @@ def _execute( driver_class: type[Driver], task: str, cycle: Optional[dt.datetime] = None, + leadtime: Optional[dt.timedelta] = None, config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, @@ -136,6 +175,7 @@ def _execute( :param driver_class: Class of driver object to instantiate. :param task: The task to execute. :param cycle: The cycle. + :param leadtime: The leadtime. :param config: Path to config file (read stdin if missing or None). :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. @@ -150,6 +190,8 @@ def _execute( ) if cycle: kwargs["cycle"] = cycle + if leadtime: + kwargs["leadtime"] = leadtime obj = driver_class(**kwargs) getattr(obj, task)() if graph_file: