From 4ca4d3d44511e7b2700a2af3e06b8b050b864bfe Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 22 Oct 2024 06:39:50 -0500 Subject: [PATCH 1/5] Fix build flags --- workflow/generate_workflows.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index ab40214655a..d7294399ac6 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -390,7 +390,7 @@ if [[ "${_build}" == "true" ]]; then printf "Building via build_all.sh %s\n\n" "${_build_flags}" # Let the output of build_all.sh go to stdout regardless of verbose options #shellcheck disable=SC2086,SC2248 - ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} + ${HOMEgfs}/sorc/build_all.sh ${_build_flags} ${_verbose_flag} fi # Link the workflow silently unless there's an error From fd4fb16152052db5f30c823ede615e7643d8cbe3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 22 Oct 2024 07:16:52 -0500 Subject: [PATCH 2/5] Address code review from #3004 --- workflow/generate_workflows.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index d7294399ac6..c4f89eef6e1 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -2,7 +2,7 @@ ### function _usage() { - cat <<-EOF + cat << EOF This script automates the experiment setup process for the global workflow. Options are also available to update submodules, build the workflow (with specific build flags), specicy which YAMLs and YAML directory to run, and @@ -204,8 +204,7 @@ else echo "The RUNTESTS directory ${_runtests} already exists." echo "Would you like to remove it?" _attempts=0 - while read -r _from_stdin - do + while read -r _from_stdin; do if [[ "${_from_stdin^^}" =~ Y ]]; then rm -rf "${_runtests}" mkdir -p "${_runtests}" From d3fc20a13c6e7972259065b2afbda94587974c4c Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 22 Oct 2024 08:27:12 -0500 Subject: [PATCH 3/5] Add documentation on generate_workflows.sh --- docs/source/development.rst | 25 ++++++++++++++++++++++++- docs/source/setup.rst | 2 ++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/source/development.rst b/docs/source/development.rst index 4739d2b602c..54d2d7c0201 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -12,6 +12,7 @@ Code managers * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov + * David Huber - @DavidHuber-NOAA / david.huber@noaa.gov .. _development: @@ -70,7 +71,29 @@ The following steps should be followed in order to make changes to the develop b Development Tools ================= -See the ``/test`` folder in global-workflow for available development and testing tools. +Two sets of testing are available for use by developers. The first is the capability to run continuous integration tests locally and the second are a set of comparison tools. + +--------------------------- +Continuous Integration (CI) +--------------------------- + +The global workflow comes fitted with a suite of system tests that run various types of workflow. These tests are commonly run for pull requests before they may be merged into the develop branch. At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform. + +The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :ref:`Experiment Setup<` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute + +:: + + cd workflow + ./generate_workflows.sh -A "your_hpc_account" -b -G -c /path/to/root/directory + +where: + + * ``-A`` is used to specify the HPC (slurm or PBS) account to use + * ``-b`` indicates that the workflow should be built fresh + * ``-G`` specifies that all of the GFS cases should be run (this also influences the build flags to use) + * ``-c`` tells the tool to append the rocotorun commands for each experiment to your crontab + +More details on how to use the tool are provided by running ``generate_workflows.sh -h``. ---------------- Comparison Tools diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 1715899927d..b2dc4d32459 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -1,3 +1,5 @@ +.. _experiment-setup: + ================ Experiment Setup ================ From ab6c44f2f792149827f92bf0e8ce6dec7d025550 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 22 Oct 2024 11:08:19 -0500 Subject: [PATCH 4/5] Change ref to doc to another webpage --- docs/source/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/development.rst b/docs/source/development.rst index 54d2d7c0201..fc07f3f55ee 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -79,7 +79,7 @@ Continuous Integration (CI) The global workflow comes fitted with a suite of system tests that run various types of workflow. These tests are commonly run for pull requests before they may be merged into the develop branch. At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform. -The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :ref:`Experiment Setup<` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute +The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :doc:`setup` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute :: From 81f34dacca162b5c0d128f8582042b70014fcd01 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 22 Oct 2024 11:21:14 -0500 Subject: [PATCH 5/5] Fix some documentation issues --- docs/source/clone.rst | 2 +- docs/source/hpc.rst | 4 ++-- docs/source/init.rst | 2 +- docs/source/setup.rst | 2 -- docs/source/wave.rst | 6 +++--- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/source/clone.rst b/docs/source/clone.rst index c365f0ed0a1..d3f81f2e475 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -9,7 +9,7 @@ Quick Instructions Quick clone/build/link instructions (more detailed instructions below). .. note:: - Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. + Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. Clone the `global-workflow` and `cd` into the `sorc` directory: diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 643cffdef0e..e83851b1a27 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -90,9 +90,9 @@ Optimizing the global workflow on S4 The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui (innocent.souopgui@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files. -======================================== +================================================== Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) -======================================== +================================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index aa71e4e2945..bd1bc1d2ce0 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -301,7 +301,7 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ -| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| +| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs\_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ diff --git a/docs/source/setup.rst b/docs/source/setup.rst index b2dc4d32459..b4d9d32f8c9 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -179,8 +179,6 @@ where: * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] * ``$ICSDIR`` is the path to the ICs for your run if generated separately. [default: None] -.. [#] More Coupled configurations in cycled mode are currently under development and not yet available - Example: :: diff --git a/docs/source/wave.rst b/docs/source/wave.rst index 7b4f7471b83..56aa34ce3b8 100644 --- a/docs/source/wave.rst +++ b/docs/source/wave.rst @@ -79,7 +79,7 @@ While the creation of these files are generally considered out of scope of this * Instructions for creating mesh.${WAVEGRID}.nc can be found at https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html#ww3 * The ww3_gint.WHTGRIDINT.bin.${WAVEGRID} can be created by running the ww3_gint routine as desired and then saved. -Once the new fix files have been created, :ref:`open an issue to have the master fix file directory updated`. This is a separate step than the process to update the workflow below. +Once the new fix files have been created, `open an issue to have the master fix file directory updated `. This is a separate step than the process to update the workflow below. ******************************** Updating Config and Script Files @@ -87,8 +87,8 @@ Updating Config and Script Files You will need to update the following files: -* parm/config/*/config.ufs -* parm/config/*/config.wave +* parm/config/\*/config.ufs +* parm/config/\*/config.wave * scripts/exgfs_wave_post_gridded_sbs.sh You will need to add the following files: