example: extending workflow

cylc/flow/etc/examples/extending-workflow/.validate

@@ -0,0 +1,128 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+test_simple () {
+    local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+
+    # lint
+    cylc lint ./simple
+
+    # copy into a temp directory
+    local SRC_DIR="$(mktemp -d)"
+    cp simple/flow.cylc "$SRC_DIR"
+
+    # speed things up with simulation mode
+    cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
+    [runtime]
+        [[root]]
+            [[[simulation]]]
+                default run length = PT0S
+__HERE__
+
+    # start the workflow
+    cylc vip \
+        --check-circular \
+        --no-run-name \
+        --no-detach \
+        --workflow-name "$ID" \
+        --mode=simulation \
+        "$SRC_DIR"
+
+    # it should have reached the 2002 cycle
+    grep '2002/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    ! grep '2003/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+
+    # edit the "stop after cycle point"
+    sed -i \
+        's/stop after cycle point.*/stop after cycle point = 2004/' \
+        "${SRC_DIR}/flow.cylc"
+
+    # continue the run
+    cylc vr \
+        --no-detach \
+        --mode=simulation \
+        --yes \
+        "$ID"
+
+    # it should have reached the 2004 cycle
+    grep '2004/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    ! grep '2005/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+
+    # clean up
+    cylc clean "$ID"
+
+    rm -r "${SRC_DIR}"
+}
+
+
+test_complex () {
+    local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+
+    # lint
+    cylc lint ./complex
+
+    # copy into a temp directory
+    local SRC_DIR="$(mktemp -d)"
+    cp complex/flow.cylc "$SRC_DIR"
+
+    # speed things up with simulation mode
+    cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
+    [runtime]
+        [[root]]
+            [[[simulation]]]
+                default run length = PT0S
+__HERE__
+
+    # start the workflow
+    cylc vip \
+        --check-circular \
+        --no-run-name \
+        --no-detach \
+        --workflow-name "$ID" \
+        --mode=simulation \
+        -s stop_cycle=2004 \
+        "$SRC_DIR"
+
+    # it should have reached the 2004 cycle
+    grep '2004/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    grep '2004/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    ! grep '2005/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+
+    # continue the run
+    cylc vr \
+        --no-detach \
+        --mode=simulation \
+        --yes \
+        -s stop_cycle=2006 \
+        "$ID"
+
+    # it should have reached the 2006 cycle
+    grep '2006/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    grep '2006/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    ! grep '2005/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+
+    # clean up
+    cylc clean "$ID"
+
+    rm -r "${SRC_DIR}"
+}
+
+
+# test_simple
+test_complex

cylc/flow/etc/examples/extending-workflow/complex/flow.cylc

@@ -0,0 +1,56 @@
+#!Jinja2
+
+{#
+    The default is "2010", this can be overridden on the command line
+    e.g. "cylc play -s stop_cycle=2020"
+#}
+{% set stop_cycle = stop_cycle | default('2010') %}
+
+[meta]
+    title = "Advanced extendable workflow"
+    description = """
+        Use the "stop_cycle" Jinja2 variable to control the workflow end point.
+
+        To extend the workflow further at a later date, restart it with a new
+        "stop_cycle".
+    """
+
+[scheduler]
+    # use the year for the cycle point
+    # (strip off the month, day, hour and minute)
+    cycle point format = CCYY
+    allow implicit tasks = True
+
+[scheduling]
+    initial cycle point = 2000
+    stop after cycle point = {{ stop_cycle }}
+    [[graph]]
+        # first cycle only
+        R1 = """
+            build => install => run_model
+        """
+
+        # first three cycles only
+        R3/2000/P1Y = """
+            initial_conditions => run_model
+        """
+
+        # every cycle
+        P1Y = """
+            run_model[-P1Y] => run_model
+        """
+
+        # last three cycles
+        R1/{{ stop_cycle }}-P2Y = """
+            # add the first process task
+            run_model => process
+        """
+        R2/P1Y/{{ stop_cycle }} = """
+            process[-P1Y] => process
+            run_model => process
+        """
+
+        # last cycle
+        R1/{{ stop_cycle }} = """
+            process => analyse => plot
+        """

cylc/flow/etc/examples/extending-workflow/index.rst

@@ -0,0 +1,182 @@
+Extending Workflow
+==================
+
+.. cylc-scope:: flow.cylc[scheduling]
+
+Sometimes we may run a workflow to completion, but subsequently wish to
+run it for a few more cycles.
+
+With Cylc 7 this was often done by changing the `final cycle point` and
+restarting the workflow. This approach worked, but was a little awkward.
+It's possible with Cylc 8, but we would recommend moving away from this
+pattern instead.
+
+The recommended approach to this problem (Cylc 6+) is to use the
+`stop after cycle point` rather than the `final cycle point`.
+
+The `stop after cycle point` tells Cylc to **stop** after the workflow passes
+the specified point, whereas the `final cycle point` tells Cylc that the
+workflow **finishes** at the specified point.
+
+When a workflow **finishes**, it is a little awkward to restart as you have to
+tell Cylc which tasks to continue on from. The `stop after cycle point`
+solution avoids this issue.
+
+
+Simple Example
+--------------
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/extending-workflow/simple
+
+This workflow will stop at the end of the ``2002`` cycle:
+
+.. literalinclude:: simple/flow.cylc
+   :language: cylc
+
+After it has run and shut down, change the `stop after cycle point` to
+the desired value and restart it. E.g:
+
+.. code-block:: bash
+
+   # install and run the workflow:
+   cylc vip
+
+   # then later edit "stop after cycle point" to "2004"
+
+   # then reinstall and restart the workflow:
+   cylc vr
+
+The workflow will continue from where it left off and run until the end of the
+``2004`` cycle. Because the workflow never hit the `final cycle point` it
+never "finished" so no special steps are required to restart the workflow.
+
+You can also set the `stop after cycle point` when you start the workflow:
+
+.. code-block:: bash
+
+   cylc play --stop-cycle-point=2020 myworkflow
+
+Or change it at any point whilst the workflow is running:
+
+.. code-block:: bash
+
+   cylc stop myworkflow//2030  # change the stop after cycle point to 2030
+
+.. note::
+
+   If you set the `stop after cycle point` on the command line, this value will
+   take precedence over the one in the workflow configuration. Use
+   ``cylc play --stop-cycle-point=reload`` to restart the workflow using the
+   `stop after cycle point` configured in the workflow configuration.
+
+
+Complex Example
+---------------
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/extending-workflow/complex
+
+Sometimes we may want to run some tasks at the `final cycle point` or in the
+cycles leading up to the `final cycle point`. For example you might have some
+analysis or housekeeping tasks to run at the end of the workflow.
+
+The following recurrence formats come in handy for such cases:
+
+.. code-block:: sub
+
+   # run once in the "2000" cycle
+   R1/2000  # 2000
+
+   # run every year three times ending in 2000
+   R3/P1Y/2000  # 1998, 1999, 2000
+
+For more information see :ref:`user_guide.cycling_format_4`.
+
+This example uses Jinja2 to template the cycling expressions above to schedule
+tasks to run at, and in the run up to, the `stop after cycle point`:
+
+.. literalinclude:: complex/flow.cylc
+   :language: cylc
+
+.. digraph:: Example
+   :align: center
+
+   size = "10, 10"
+
+   subgraph cluster_1 {
+      label = "Spin Up (first 3 cycles)"
+      style = "dashed"
+
+      build_2000 [label="build\n2000"]
+      install_2000 [label="install\n2000"]
+      initial_conditions_2000 [label="initial_conditions\n2000"]
+      initial_conditions_2001 [label="initial_conditions\n2001"]
+      initial_conditions_2002 [label="initial_conditions\n2002"]
+      run_model_2000 [label="run_model\n2000"]
+      run_model_2001 [label="run_model\n2001"]
+      run_model_2002 [label="run_model\n2002"]
+
+      build_2000 -> install_2000 -> run_model_2000
+      run_model_2000 -> run_model_2001 -> run_model_2002
+      initial_conditions_2000 -> run_model_2000
+      initial_conditions_2001 -> run_model_2001
+      initial_conditions_2002 -> run_model_2002
+   }
+
+   subgraph cluster_2 {
+      label = "Main section"
+      style = "dashed"
+
+      run_model_2003 [label="run_model\n2003"]
+      run_model_2004 [label="run_model\n2004"]
+      run_model_2005 [label="run_model\n2005"]
+      run_model_2006 [label="run_model\n2006"]
+      run_model_2007 [label="run_model\n2007"]
+
+      run_model_2002 -> run_model_2003 -> run_model_2004 -> run_model_2005
+      run_model_2005 -> run_model_2006 -> run_model_2007
+   }
+
+   subgraph cluster_3 {
+      label = "Spin down (last 3 cycles)"
+      style = "dashed"
+
+      run_model_2008 [label="run_model\n2008"]
+      run_model_2009 [label="run_model\n2009"]
+      run_model_2010 [label="run_model\n2010"]
+      process_2008 [label="process\n2008"]
+      process_2009 [label="process\n2009"]
+      process_2010 [label="process\n2010"]
+      analyse_2010 [label="analyse\n2010"]
+      plot_2010 [label="plot\n2010"]
+
+      run_model_2007 -> run_model_2008 -> run_model_2009 -> run_model_2010
+      run_model_2008 -> process_2008
+      run_model_2009 -> process_2009
+      run_model_2010 -> process_2010
+      process_2008 -> process_2009
+      process_2009 -> process_2010
+      process_2010 -> analyse_2010 -> plot_2010
+   }
+
+To run the workflow:
+
+.. code-block:: bash
+
+   # install and run the workflow:
+   cylc vip
+
+   # then reinstall and restart the workflow extending the stop cycle:
+   cylc vr -s stop_cycle=2020
+
+
+.. cylc-scope::

cylc/flow/etc/examples/extending-workflow/simple/flow.cylc

@@ -0,0 +1,24 @@
+[meta]
+    title = "Basic extendable workflow"
+    description = """
+        Use the "stop after cycle point" rather than the "final cycle point"
+        to allow this workflow to be easily extended at a later date.
+    """
+
+[scheduler]
+    # use the year for the cycle point
+    # (strip off the month, day, hour and minute)
+    cycle point format = CCYY
+
+[scheduling]
+    initial cycle point = 2000
+    stop after cycle point = 2002  # stop after two years of simulated time
+    [[graph]]
+        P1Y = """
+            z[-P1Y] => a
+            a => z
+        """
+
+[runtime]
+    [[a]]
+    [[z]]