ecFlow Suite Generator Application

ecf/defs/GFSApp.def

@@ -0,0 +1,66 @@
+#### totality only
+extern /task_limit:TOTALITY
+
+suite fcstonly
+  inlimit /task_limit:TOTALITY
+  edit PSLOT 'ecffo_c192'
+  edit PDY '20200906'
+  edit EDATE '20200907'
+  edit ECF_INCLUDE '/scratch1/NCEPDEV/global/Lin.Gan/git/global-workflow-develop/ecf/include'
+  edit ENVIR 'para'
+  edit MACHINE_SITE 'development'
+  edit CYC '00'
+  edit QUEUE 'batch'
+  edit QUEUE_SERVICE 'service'
+  edit PROJENVIR 'DEV'
+  edit HERA_ACCOUNT 'fv3-cpu'
+  edit EMC_USER 'Lin.Gan'
+  edit gfs_ver 'v16.2.0'
+  edit PACKAGEHOME '/scratch1/NCEPDEV/global/%EMC_USER%/para/packages/gfs.%gfs_ver%'
+  edit ECF_FILES '%PACKAGEHOME%/ecf/scripts/gfs'
+  edit PROJ 'GFS'
+  family GFSApp
+    edit NET 'gfs'
+    family gfs
+      edit RUN 'gfs'
+      family atmos
+        task jgfs_getic
+        task jgfs_forecast
+          trigger ./atmos/jgfs_getic == complete 
+        family post
+          task jgfs_atmos_post_manager
+            trigger ../../jgfs_forecast == active
+            event 1 release_post000
+            event 2 release_post001
+            event 3 release_post002
+            event 4 release_post003
+          task jgfs_atmos_post_f000
+            trigger ./jgfs_atmos_post_manager:release_post000
+            edit FHRGRP '001'
+            edit FHRLST 'f000'
+            edit FHR 'f000'
+            edit HR '000'
+          task jgfs_atmos_post_f001
+            trigger ./jgfs_atmos_post_manager:release_post001
+            edit FHRGRP '002'
+            edit FHRLST 'f001'
+            edit FHR 'f001'
+            edit HR '001'
+          task jgfs_atmos_post_f002
+            trigger ./jgfs_atmos_post_manager:release_post002
+            edit FHRGRP '003'
+            edit FHRLST 'f002'
+            edit FHR 'f002'
+            edit HR '002'
+          task jgfs_atmos_post_f003
+            trigger ./jgfs_atmos_post_manager:release_post003
+            edit FHRGRP '004'
+            edit FHRLST 'f003'
+            edit FHR 'f003'
+            edit HR '003'
+        endfamily # post
+      endfamily # atmos
+    endfamily # gfs
+  endfamily # GFSApp
+endsuite
+

workflow_generator/README.md

@@ -0,0 +1,232 @@
+# Workflow Setup Utility:
+
+## Introduction
+This utility is designed to be an automated ecFlow and Rocoto generation application,
+used to create the folder structures and scripts needed to execute the workflows
+for either application. As of April 2022, this application only works for ecFlow.
+
+### How ecFlow Setup Works
+For ecFlow creation, the application takes a YAML file as input, pulls in any
+environment variables that are specified in the YAML, then using the ecFlow
+API, a definition file is created. Additionally, since ecFlow definition files
+are dependent on folder structures, the application also identifies the scripts
+associated with tasks and creates the folders for them, checks the script repository
+folder and puts the scripts in their appropriate location.
+
+Please refer to the [setup the YAML](#configuring-the-yaml-file) section for instructions
+on how to setup the YAML file for what you want.
+
+## Setup for using the utility with ecFlow
+This utility uses Python3.6 and later. It will not work with Python anything before
+Python3.6.
+
+### Pre-Requisites
+In order to run the application the following Python3 modules need to be available:
+* ecflow
+* numpy
+* PyYAML
+These modules should be available on Hera and Orion.
+
+### Experiment Setup
+This application requires the use of a config.base file. The location of the file
+can be specified with the `--expdir` parameter. The file will be read in and
+the ush/rocoto/workflow_utils.py script will be used to populate any environment
+variables that are needed.
+
+### Required Environment Variables
+If not setup within the script, the following environmnt variables are required:
+* Account
+* Queue
+* machine
+* RUN_ENVIR
+These parameters are populated as 'edits' within the ecFlow definition file for
+any of the suites that are created.
+
+An additional environment variable that is needed is:
+* ECFgfs
+This will be used as the base location for storing the suite scripts and also
+used as the base location to look for the script repository. The application
+assumes the default that the script repo is ECFgfs/scripts. Suggested edit is to
+add the following to the config.base file:
+* export ECFgfs=$HOMEgfs/ecf
+
+## Configuring the YAML file
+The utility works primarily off of the yaml file used to define the suites,
+families, and tasks.
+
+### Script Repository
+The scriptrepo can be specified either by an environment variable or by setting
+a `scriptrepo: /path/to/scripts` variable in the YAML file at the top level.
+
+The script repository is the location the application will search for the ecf scripts to 
+deploy the to the correct location. 
+
+### Setting up externs
+To add externs, add the `externs:` label to the base level of the yaml file,
+then add each extern as a yaml list object
+* Example:
+
+```YAML
+externs:
+- "/prod18/enkfgdas/post"
+- "/prod18/enkfgfs/post"
+```
+
+### Setting up a suite
+To add a suite add the `suites:` label to the base level and then add the suite
+names you want as dictionary objects:
+* Example:
+
+```YAML
+suites:
+  fcstonly:
+  fcstplus:
+```
+
+### Setting up families
+Once a suite has been setup, add families as dictionary objects under the families.
+Families can be dictionary objects under other families. In the example below the
+fcstonly suite has the family gfs and the post family under that. The fcstplus
+suite has the family gdas and the family atmos under that.
+* Example
+
+```YAML
+suites:
+  fcstonly:
+    gfs:
+      post:
+  fcstplus:
+    gdas:
+      atmos:
+```
+
+### Adding edits
+Edits can be added to either families, tasks or suites by putting an `edit:`
+dictionary tag and then listing the edits below. In the example below, the `RUN`
+edit will be assigned the parameter `00` for the `gfs` family.
+* Example
+
+```YAML
+suites:
+  fcstonly:
+    gfs:
+      edits:
+        RUN: '00'
+```
+
+### Setting up the tasks
+After the families are defined, tasks are defined by placing a `tasks:` dictionary
+object under a family. Tasks are then put into dictionary objects. In the example below, the `gfs` family has an `atmos` family underneath it
+with the `jgfs_forecast` task  
+* Example
+
+```YAML
+gfs:
+  atmos:
+    tasks:
+      jgfs_forecast:
+```
+
+#### Task Options
+Additional options are `triggers`, `events`, `edits`, and `defstatus`.
+
+##### Triggers
+To add a trigger to a task, add a `triggers:` heading underneath the task or family. The triggers need to be a list
+item with the identifier for what you want the trigger to look for. So for a task, it would be `- task: task_name` or
+for a family it would be `- family: family_name`
+
+Trigger list items can also have events, states, or suites as part of their configuration. Valid events must be listed
+in the definition file previously. Valid states are complete, active, or queued. The suite must also be a defined suite
+somewhere in the YAML file. 
+
+* Example
+```YAML
+gfs:
+  edits:
+    RUN: 'gfs'
+    NET: 'gfs'
+  tasks:
+    jgfs_forecast:
+      triggers:
+      - task: jgfs_atmos_analysis
+        event: release_fcst
+      - task: jgfs_wave_prep
+```
+
+##### Events
+To add events to a task, add an `events:` heading underneath the task. The items in the events is a list of strings.
+* Example: This has two triggers and one event associated with it. 
+```YAML
+analysis:
+  tasks:
+    jgfs_atmos_analysis:
+      triggers:
+      - task: jobsproc_gfs_atmos_prep
+      - task: jgfs_atmos_emcsfc_sfc_prep
+      events:
+      - "release_fcst"
+```
+
+##### Edits
+To add edits to a task or family or suite, add an `edits:` heading underneath the node. The items go in dictionary
+format such that `edit: edit_value`
+* Example
+```YAML
+gfs:
+  edits:
+    RUN: 'gfs'
+    NET: 'gfs'
+```
+
+##### Defstatus
+To add defstatus to a task or family, add a `defstatus:` parameter underneath the node. It will have a value associated
+with it such that `defstatus: value`
+* Example
+```YAML
+obsproc:
+  defstatus: complete
+```
+### Ranges and Lists
+Families or tasks, or even triggers and events can have ranges or lists associated with them to shorten the creation
+of the definition YAML. The range notation uses the `( )` bracket syntax to specify and follows the same pattern as 
+Python ranges. If one value is specified, it assumes it is the max value, starts at 1, increments by 1 up to the value
+specified. If two values are presented, then it uses the first as the initial starting value, increments by 1 up to the
+max value specified. If all three, it uses them with the initial, increment, and max range values. If no value or 
+no max value is specified it uses what ever values are used in the parent counter. 
+* Example
+```YAML
+jgfs_atmos_post_f( 384 ):
+    template: jgfs_atmos_post_master
+    triggers:
+    - task: jgfs_atmos_post_manager
+      event: release_post( )
+    edits:
+      FHRGRP: '( 1, )'
+      FHRLST: 'f( )'
+      FHR: 'f( )'
+      HR: '( )'
+```
+
+Lists are similar to the ranges but use the `[ ]` bracket syntax. Items in the list can be of any type and will run 
+the same way as ranges do
+* Example
+```YAML
+post:
+  tasks:
+    jgfs_atmos_post_manager[ 1,2 ]:
+```
+## Run the utility
+1. Change into the workflow directory:
+` cd global-workflow/workflow_generator`
+1. Run the utility
+```
+python3 setup_workflow.py --expdir ../parm/config
+```
+
+### Command Line Options
+* --ecflow-config
+  * Path to the YAML configuration file to use to generate the definition and folder/scripts. 
+* --expdir
+  * Path to the experiment directory. Must contain config.base.
+* --savedir
+  * Path to where the definition file will be saved. Defaults to current directory.

workflow_generator/ecflow_build.yml

@@ -0,0 +1,68 @@
+---
+# scriptrepo: /path/to/scripts
+externs:
+- "/prod18/enkfgdas/post"
+suites:
+  fcstonly:
+    edits:
+      CYC: env.ARCH_CYC
+      EDATE: env.EDATE
+    nodes:
+      GFSApp:
+        edits:
+          NET: 'gfs'
+        gfs:
+          repeat: "2022032400 to 2022042400 by 18:0"
+          edits:
+            RUN: 'gfs'
+          atmos:
+            tasks:
+              jgfs_forecast:
+                triggers:
+                - task: jgfs_getic
+                - task: jgfs_atmos_post_f( 2,1 )
+                - task: jgfs_forecast
+                  suite: fcstplus
+              jgfs_getic:
+                events:
+                - test_event
+                edits:
+                  RUN: 'gfs'
+            post:
+              tasks:
+                jgfs_atmos_post_manager[ 1,2 ]:
+                  template: jgfs_atmos_post_manager
+                  triggers:
+                  - task: jgfs_forecast
+                    state: active
+                  events:
+                  - "release_post(4)f"
+                jgfs_atmos_post_f(2,env.FHMAX_GFS):
+                  template: jgfs_atmos_post_master
+                  triggers:
+                  - task: jgfs_atmos_post_manager[1,2]
+                    event: "release_post(1,2)f"
+                  events:
+                  - "test_event"
+                  edits:
+                    FHRGRP: "( 1,,6 )"
+                    FHRGRP2: [ a,b,c,d ]
+                    FHRLST: "f[ a,b,3,4 ] -testing"
+                    FHR: "f(1,6,)"
+                    HR: "( )"
+                    TEST: 'Test Edit'
+          wave:
+            init:
+              tasks:
+                jgfs_wave_init:
+                jgfs_forecast:
+  fcstplus:
+    edits:
+      CYC: '06'
+    nodes:
+      nonGFSApp:
+        tasks:
+          jgfs_forecast:
+            triggers:
+            - task: jgfs_getic
+              suite: fcstonly