From a3dda6b03db5a60f54862d1a3d017323418de714 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Jun 2022 10:15:49 -0400 Subject: [PATCH 01/41] eomg and eobs share the same resources, make them so --- parm/config/config.resources | 32 +++++++++++++++++++------------- ush/rocoto/configuration.py | 0 ush/rocoto/hosts.py | 0 ush/rocoto/workflow_tasks.py | 0 ush/rocoto/workflow_xml.py | 0 5 files changed, 19 insertions(+), 13 deletions(-) create mode 100644 ush/rocoto/configuration.py create mode 100644 ush/rocoto/hosts.py create mode 100644 ush/rocoto/workflow_tasks.py create mode 100644 ush/rocoto/workflow_xml.py diff --git a/parm/config/config.resources b/parm/config/config.resources index 35e0c6a76dd..262a1ba9385 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -177,12 +177,12 @@ elif [ $step = "analdiag" ]; then elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" - export npe_gldas=96 - export nth_gldas=1 + export npe_gldas=96 + export nth_gldas=1 export npe_node_gldas=$npe_node_max - export npe_gaussian=96 - export nth_gaussian=1 - export npe_node_gaussian=24 + export npe_gaussian=96 + export nth_gaussian=1 + export npe_node_gaussian=24 if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_gldas=112 ; fi if [[ "$machine" == "WCOSS_C" ]]; then export memory_gldas="3072M"; fi @@ -351,13 +351,13 @@ elif [ $step = "vrfy" ]; then export npe_vrfy_gfs=1 export npe_node_vrfy_gfs=1 if [[ "$machine" == "WCOSS_C" ]]; then - export memory_vrfy="3072M" + export memory_vrfy="3072M" elif [[ "$machine" == "HERA" ]]; then - export memory_vrfy="16384M" + export memory_vrfy="16384M" fi elif [ $step = "metp" ]; then - + export nth_metp=1 export wtime_metp="03:00:00" export npe_metp=4 @@ -366,9 +366,9 @@ elif [ $step = "metp" ]; then export npe_metp_gfs=4 export npe_node_metp_gfs=4 if [[ "$machine" == "WCOSS_C" ]]; then - export memory_metp="3072M" + export memory_metp="3072M" elif [[ "$machine" == "THEIA" ]]; then - export memory_metp="16384M" + export memory_metp="16384M" fi elif [ $step = "echgres" ]; then @@ -385,9 +385,9 @@ elif [ $step = "init" ]; then export nth_init=1 export npe_node_init=6 if [ $machine = "WCOSS_DELL_P3" ]; then - export memory_init="10G" + export memory_init="10G" else - export memory_init="70G" + export memory_init="70G" fi elif [ $step = "init_chem" ]; then @@ -430,10 +430,16 @@ elif [ $step = "eobs" -o $step = "eomg" ]; then elif [ $CASE = "C96" -o $CASE = "C48" ]; then export npe_eobs=20 fi + export npe_eomg=$npe_eobs export nth_eobs=2 if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export nth_eobs=7; fi + export nth_eomg=$nth_eobs export npe_node_eobs=$(echo "$npe_node_max / $nth_eobs" | bc) - if [[ "$machine" == "WCOSS_C" ]]; then export memory_eobs="3072M"; fi + export npe_node_eomg=$npe_node_eobs + if [[ "$machine" == "WCOSS_C" ]]; then + export memory_eobs="3072M" + export memory_eomg=$memory_eobs + fi elif [ $step = "ediag" ]; then diff --git a/ush/rocoto/configuration.py b/ush/rocoto/configuration.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py new file mode 100644 index 00000000000..e69de29bb2d From 8e6edf8f9f48d1cde5ed8dacec764fe092fc476b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Jun 2022 10:16:52 -0400 Subject: [PATCH 02/41] wip on setup_workflow refactor --- .gitignore | 1 + ush/rocoto/configuration.py | 153 ++++++ ush/rocoto/hosts.py | 234 ++++++++ ush/rocoto/rocoto.py | 141 +++-- ush/rocoto/setup_workflow.py | 2 +- ush/rocoto/setup_workflow_fcstonly.py | 192 ++----- ush/rocoto/workflow_tasks.py | 740 ++++++++++++++++++++++++++ ush/rocoto/workflow_utils.py | 564 +++++--------------- ush/rocoto/workflow_xml.py | 82 +++ 9 files changed, 1459 insertions(+), 650 deletions(-) diff --git a/.gitignore b/.gitignore index e8638c013f8..9147274742f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__ *.[aox] *.mod *.sw[a-p] +.idea/ # Ignore folders #------------------- diff --git a/ush/rocoto/configuration.py b/ush/rocoto/configuration.py index e69de29bb2d..5313c99414a 100644 --- a/ush/rocoto/configuration.py +++ b/ush/rocoto/configuration.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +import os +import random +import glob +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Union + + +__all__ = ['Configuration'] + + +class ShellScriptException(Exception): + def __init__(self, scripts, errors): + self.scripts = scripts + self.errors = errors + super(ShellScriptException, self).__init__( + str(errors) + + ': error processing' + + (' '.join(scripts))) + + +class UnknownConfigError(Exception): + pass + + +class Configuration: + """ + Configuration parser for the global-workflow + (or generally for sourcing a shell script into a python dictionary) + """ + + DATE_ENV_VARS = ['CDATE', 'SDATE', 'EDATE'] + TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.'] + BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS + BOOLS = [x.upper() for x in BOOLS] + BOOLS + + def __init__(self, config_dir: Union[str, Path]): + """ + Given a directory containing config files (config.XYZ), + return a list of configs minus the ones ending with ".default" + """ + + self.config_dir = config_dir + self.configs = self._get_configs + + @property + def _get_configs(self) -> list: + """ + Given a directory containing config files (config.XYZ), + return a list of configs minus the ones ending with ".default" + """ + result = list() + for config in glob.glob(f'{self.config_dir}/config.*'): + if not config.endswith('.default'): + result.append(config) + + return result + + def find_config(self, config_name: str) -> str: + """ + Given a config file name, find the full path of the config file + """ + + for config in self.configs: + if config_name == os.path.basename(config): + return config + + raise UnknownConfigError( + f'{config_name} does not exist (known: {repr(config_name)}), ABORT!') + + def parse_config(self, files: Union[str, bytes, list]) -> dict: + """ + Given the name of config file(s), key-value pair of all variables in the config file(s) + are returned as a dictionary + :param files: config file or list of config files + :type files: list or str or unicode + :return: Key value pairs representing the environment variables defined + in the script. + :rtype: dict + """ + if isinstance(files, (str, bytes)): + files = [files] + files = [self.find_config(file) for file in files] + varbles = dict() + for key, value in self._get_script_env(files).items(): + if key in self.DATE_ENV_VARS: # likely a date, convert to datetime + varbles[key] = datetime.strptime(value, '%Y%m%d%H') + elif value in self.BOOLS: # Likely a boolean, convert to True/False + varbles[key] = self._bool_or_not(value) + elif '.' in value: # Likely a number and that too a float + varbles[key] = self._cast_or_not(float, value) + else: # Still could be a number, may be an integer + varbles[key] = self._cast_or_not(int, value) + + return varbles + + def print_config(self, files: Union[str, bytes, list]): + """ + Given the name of config file(s), key-value pair of all variables in the config file(s) are printed + Same signature as parse_config + :param files: config file or list of config files + :type files: list or str or unicode + :return: None + """ + config = self.parse_config(files) + line_separator = '\n' + print(f'{line_separator.join(f"{key}: {config[key]}" for key in config.keys())}') + + @classmethod + def _get_script_env(cls, scripts): + default_env = cls._get_shell_env([]) + and_script_env = cls._get_shell_env(scripts) + vars_just_in_script = set(and_script_env) - set(default_env) + union_env = dict(default_env) + union_env.update(and_script_env) + return dict([(v, union_env[v]) for v in vars_just_in_script]) + + @staticmethod + def _get_shell_env(scripts): + varbls = dict() + runme = ''.join([f'source {s} ; ' for s in scripts]) + magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' + runme += f'/bin/echo -n "{magic}" ; /usr/bin/env -0' + with open('/dev/null', 'w') as null: + env = subprocess.Popen(runme, shell=True, stdin=null.fileno(), + stdout=subprocess.PIPE) + (out, err) = env.communicate() + out = out.decode() + begin = out.find(magic) + if begin < 0: + raise ShellScriptException(scripts, 'Cannot find magic string; ' + 'at least one script failed: '+repr(out)) + for entry in out[begin+len(magic):].split('\x00'): + iequal = entry.find('=') + varbls[entry[0:iequal]] = entry[iequal+1:] + return varbls + + @staticmethod + def _cast_or_not(type, value): + try: + return type(value) + except ValueError: + return value + + @staticmethod + def _bool_or_not(value): + try: + return value.lower() in Configuration.TRUTHS + except AttributeError: + return value diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index e69de29bb2d..296913623d2 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 + +import os + +__all__ = ['Host'] + + +class Host: + """ + Gather Host specific information. + Someday the info will be pushed out of the code and into a yaml or something. + """ + + # SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', + # 'WCOSS_C', 'WCOSS_DELL_P3', 'WCOSS2'] + SUPPORTED_HOSTS = ['HERA', 'ORION', + 'WCOSS_DELL_P3'] # TODO - Remove this and uncomment above line + + def __init__(self): + + self.machine = self.detect + self.scheduler = self.get_scheduler + self.info = self._get_info + + @classmethod + @property + def detect(cls): + + # machine = 'NOTFOUND' # TODO - Uncomment this line + machine = 'ORION' # TODO - Remove this line + + if os.path.exists('/scratch1/NCEPDEV'): + machine = 'HERA' + elif os.path.exists('/work/noaa'): + machine = 'ORION' + elif os.path.exists('/lfs4/HFIP'): + machine = 'JET' + elif os.path.exists('/gpfs') and os.path.exists('/etc/SuSE-release'): + machine = 'WCOSS_C' + elif os.path.exists('/gpfs/dell2'): + machine = 'WCOSS_DELL_P3' + elif os.path.exists('/lfs/f1'): + machine = 'WCOSS2' + + if machine not in Host.SUPPORTED_HOSTS: + raise NotImplementedError(f'This machine is not a supported host.\n' + + 'Currently supported hosts are:\n' + + f'{" | ".join(Host.SUPPORTED_HOSTS)}') + + return machine + + @classmethod + @property + def get_scheduler(cls) -> str: + + SCHEDULER_MAP = {'HERA': 'slurm', + 'JET': 'slurm', + 'ORION': 'slurm', + 'WCOSS_C': 'lsfcray', + 'WCOSS_DELL_P3': 'lsf', + 'WCOSS_DELL_P3p5': 'lsf', + 'WCOSS2': 'pbspro'} + + machine = Host.detect + try: + return SCHEDULER_MAP[machine.upper()] + except KeyError: + raise NotImplementedError(f'{machine} is not a supported host.\n' + + 'Currently supported hosts are:\n' + + f'{" | ".join(Host.SUPPORTED_HOSTS)}') + + @property + def _get_info(self) -> dict: + + HOST_INFO_MAP = { + 'HERA': self._hera(), + 'ORION': self._orion(), + 'JET': self._jet(), + 'WCOSS_C': self._wcoss_c(), + 'WCOSS_DELL_P3': self._wcoss_dell_p3(), + 'WCOSS_DELL_P3p5': self._wcoss_dell_p3p5(), + 'WCOSS2': self._wcoss2()} + + try: + info = HOST_INFO_MAP[self.machine.upper()] + except KeyError as exc: + raise NotImplementedError(f'{self.machine} is not a supported host.\n' + + 'Currently supported hosts are:\n' + + f'{" | ".join(Host.SUPPORTED_HOSTS)}') + + return info + + @staticmethod + def _hera() -> dict: + + info = { + 'base_git': '/scratch1/NCEPDEV/global/glopara/git', + 'base_svn': '/scratch1/NCEPDEV/global/glopara/svn', + 'dmpdir': '/scratch1/NCEPDEV/global/glopara/dump', + 'nwprod': '/scratch1/NCEPDEV/global/glopara/nwpara', + 'comroot': '/scratch1/NCEPDEV/global/glopara/com', + 'homedir': '/scratch1/NCEPDEV/global/$USER', + 'stmp': '/scratch1/NCEPDEV/stmp2/$USER', + 'ptmp': '/scratch1/NCEPDEV/stmp4/$USER', + 'noscrub': '$HOMEDIR', + 'account': 'fv3-cpu', + 'queue': 'batch', + 'queue_service': 'service', + 'partition_batch': 'hera', + 'chgrp_rstprod': 'YES', + 'chgrp_cmd': 'chgrp rstprod', + 'hpssarch': 'YES', + 'localarch': 'NO', + 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', + } + + return info + + @staticmethod + def _orion() -> dict: + + info = { + 'base_git': '/work/noaa/global/glopara/git', + 'base_svn': '/work/noaa/global/glopara/svn', + 'dmpdir': '/work/noaa/rstprod/dump', + 'nwprod': '/work/noaa/global/glopara/nwpara', + 'comroot': '/work/noaa/global/glopara/com', + 'homedir': '/work/noaa/global/$USER', + 'stmp': '/work/noaa/stmp/$USER', + 'ptmp': '/work/noaa/stmp/$USER', + 'noscrub': '$HOMEDIR', + 'account': 'fv3-cpu', + 'queue': 'batch', + 'queue_service': 'service', + 'partition_batch': 'orion', + 'chgrp_rstprod': 'YES', + 'chgrp_cmd': 'chgrp rstprod', + 'hpssarch': 'NO', + 'localarch': 'NO', + 'atardir': '$NOSCRUB/archive_rotdir/$PSLOT', + } + + return info + + @staticmethod + def _jet() -> dict: + + info = {} + + return info + + @staticmethod + def _wcoss_c() -> dict: + + info = { + 'base_git': '/gpfs/hps3/emc/global/noscrub/emc.glopara/git', + 'base_svn': '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn', + 'dmpdir': '/gpfs/dell3/emc/global/dump', + 'nwprod': '${NWROOT:-"/gpfs/hps/nco/ops/nwprod"}', + 'comroot': '${COMROOT:-"/gpfs/hps/nco/ops/com"}', + 'homedir': '/gpfs/hps3/emc/global/noscrub/$USER', + 'stmp': '/gpfs/hps2/stmp/$USER', + 'ptmp': '/gpfs/hps2/ptmp/$USER', + 'noscrub': '/gpfs/hps3/emc/global/noscrub/$USER', + 'account': 'GFS-DEV', + 'queue': 'dev', + 'queue_service': 'dev_transfer', + 'chgrp_rstprod': 'YES', + 'chgrp_cmd': 'chgrp rstprod', + 'hpssarch': 'YES', + 'localarch': 'NO', + 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', + } + + return info + + @staticmethod + def _wcoss_dell_p3() -> dict: + + info = { + 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', + 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', + 'dmpdir': '/gpfs/dell3/emc/global/dump', + 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', + 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', + 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', + 'stmp': '/gpfs/dell3/stmp/$USER', + 'ptmp': '/gpfs/dell3/ptmp/$USER', + 'noscrub': '$HOMEDIR', + 'account': 'GFS-DEV', + 'queue': 'dev', + 'queue_service': 'dev_transfer', + 'partition_batch': None, + 'chgrp_rstprod': 'YES', + 'chgrp_cmd': 'chgrp rstprod', + 'hpssarch': 'YES', + 'localarch': 'NO', + 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', + } + + return info + + @staticmethod + def _wcoss_dell_p3p5() -> dict: + + info = { + 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', + 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', + 'dmpdir': '/gpfs/dell3/emc/global/dump', + 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', + 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', + 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', + 'stmp': '/gpfs/dell3/stmp/$USER', + 'ptmp': '/gpfs/dell3/ptmp/$USER', + 'noscrub': '$HOMEDIR', + 'account': 'GFS-DEV', + 'queue': 'dev2', + 'queue_service': 'dev2_transfer', + 'partition_batch': None, + 'chgrp_rstprod': 'YES', + 'chgrp_cmd': 'chgrp rstprod', + 'hpssarch': 'YES', + 'localarch': 'NO', + 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', + } + + return info + + @staticmethod + def _wcoss2() -> dict: + + info = {} + + return info diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 2a027fdc4fa..1301955e88d 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -1,15 +1,22 @@ #!/usr/bin/env python3 +from typing import Union, List, Dict, Any + ''' MODULE: rocoto.py ABOUT: - Helper module to create tasks, metatasks, and dependencies + Helper module to create tasks, metatasks, and dependencies for Rocoto ''' -def create_metatask(task_dict, metatask_dict): - ''' +__all__ = ['create_task', 'create_metatask', + 'add_dependency', 'create_dependency', + 'create_envar', 'create_entity', 'create_cycledef'] + + +def create_metatask(task_dict: Dict[str, Any], metatask_dict: Dict[str, Any]) -> List[str]: + """ create a Rocoto metatask given a dictionary containing task and metatask information :param metatask_dict: metatask key-value parameters :type metatask_dict: dict @@ -17,7 +24,7 @@ def create_metatask(task_dict, metatask_dict): :type task_dict: dict :return: Rocoto metatask :rtype: list - ''' + """ # Grab metatask info from the metatask_dict metataskname = metatask_dict.get('metataskname', 'demometatask') @@ -44,14 +51,14 @@ def create_metatask(task_dict, metatask_dict): return strings -def create_task(task_dict): - ''' +def create_task(task_dict: Dict[str, Any]) -> List[str]: + """ create a Rocoto task given a dictionary containing task information :param task_dict: task key-value parameters :type task_dict: dict :return: Rocoto task :rtype: list - ''' + """ # Grab task info from the task_dict taskname = task_dict.get('taskname', 'demotask') @@ -68,7 +75,7 @@ def create_task(task_dict): native = task_dict.get('native', None) memory = task_dict.get('memory', None) resources = task_dict.get('resources', None) - envar = task_dict.get('envar', None) + envar = task_dict.get('envars', None) dependency = task_dict.get('dependency', None) str_maxtries = str(maxtries) @@ -114,38 +121,31 @@ def create_task(task_dict): return strings -def add_dependency(dep_dict): - ''' +def add_dependency(dep_dict: Dict[str, Any]) -> str: + """ create a simple Rocoto dependency given a dictionary with dependency information :param dep_dict: dependency key-value parameters :type dep_dict: dict :return: Rocoto simple dependency :rtype: str - ''' + """ + + tag_map = {'task': _add_task_tag, + 'metatask': _add_task_tag, + 'data': _add_data_tag, + 'cycleexist': _add_cycle_tag, + 'streq': _add_streq_tag, + 'strneq': _add_streq_tag} dep_condition = dep_dict.get('condition', None) dep_type = dep_dict.get('type', None) - if dep_type in ['task', 'metatask']: - - string = add_task_tag(dep_dict) - - elif dep_type in ['data']: - - string = add_data_tag(dep_dict) - - elif dep_type in ['cycleexist']: - - string = add_cycle_tag(dep_dict) - - elif dep_type in ['streq', 'strneq']: - - string = add_streq_tag(dep_dict) - - else: - - msg = f'Unknown dependency type {dep_dict["type"]}' - raise KeyError(msg) + try: + string = tag_map[dep_type](dep_dict) + except KeyError: + raise KeyError(f'{dep_type} is an unknown dependency type.\n' + + 'Currently supported dependency types are:\n' + + f'{" | ".join(tag_map.keys())}') if dep_condition is not None: string = f'<{dep_condition}>{string}' @@ -153,14 +153,14 @@ def add_dependency(dep_dict): return string -def add_task_tag(dep_dict): - ''' +def _add_task_tag(dep_dict: Dict[str, Any]) -> str: + """ create a simple task or metatask tag :param dep_dict: dependency key-value parameters :type dep_dict: dict :return: Rocoto simple task or metatask dependency :rtype: str - ''' + """ dep_type = dep_dict.get('type', None) dep_name = dep_dict.get('name', None) @@ -178,14 +178,15 @@ def add_task_tag(dep_dict): return string -def add_data_tag(dep_dict): - ''' + +def _add_data_tag(dep_dict: Dict[str, Any]) -> str: + """ create a simple data tag :param dep_dict: dependency key-value parameters :type dep_dict: dict :return: Rocoto simple task or metatask dependency :rtype: str - ''' + """ dep_type = dep_dict.get('type', None) dep_data = dep_dict.get('data', None) @@ -212,14 +213,15 @@ def add_data_tag(dep_dict): return string -def add_cycle_tag(dep_dict): - ''' + +def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: + """ create a simple cycle exist tag :param dep_dict: dependency key-value parameters :type dep_dict: dict :return: Rocoto simple task or metatask dependency :rtype: str - ''' + """ dep_type = dep_dict.get('type', None) dep_offset = dep_dict.get('offset', None) @@ -232,14 +234,15 @@ def add_cycle_tag(dep_dict): return string -def add_streq_tag(dep_dict): - ''' + +def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: + """ create a simple string comparison tag :param dep_dict: dependency key-value parameters :type dep_dict: dict :return: Rocoto simple task or metatask dependency :rtype: str - ''' + """ dep_type = dep_dict.get('type', None) dep_left = dep_dict.get('left', None) @@ -264,8 +267,8 @@ def add_streq_tag(dep_dict): def _traverse(o, tree_types=(list, tuple)): - ''' - Traverse through a list of lists or tuples and yeild the value + """ + Traverse through a list of lists or tuples and yield the value Objective is to flatten a list of lists or tuples :param o: list of lists or not :type o: list, tuple, scalar @@ -273,7 +276,7 @@ def _traverse(o, tree_types=(list, tuple)): :type tree_types: tuple :return: value in the list or tuple :rtype: scalar - ''' + """ if isinstance(o, tree_types): for value in o: @@ -283,9 +286,9 @@ def _traverse(o, tree_types=(list, tuple)): yield o -def create_dependency(dep_condition=None, dep=None): - ''' - create a compound dependency given a list of dependendies, and compounding condition +def create_dependency(dep_condition=None, dep=None) -> List[str]: + """ + create a compound dependency given a list of dependencies, and compounding condition the list of dependencies are created using add_dependency :param dep_condition: dependency condition :type dep_condition: boolean e.g. and, or, true, false @@ -293,7 +296,7 @@ def create_dependency(dep_condition=None, dep=None): :type dep: str or list :return: Rocoto compound dependency :rtype: list - ''' + """ dep = dep if isinstance(dep, list) else [dep] @@ -316,9 +319,9 @@ def create_dependency(dep_condition=None, dep=None): return strings -def create_envar(name=None,value=None): - ''' - create an Rocoto environment variable given name and value +def create_envar(name: str, value: Union[str, float, int]) -> str: + """ + create a Rocoto environment variable given name and value returns the environment variable as a string :param name: name of the environment variable :type name: str @@ -326,20 +329,14 @@ def create_envar(name=None,value=None): :type value: str or float or int or unicode :return: Rocoto environment variable key-value pair :rtype: str - ''' + """ - string = '' - string += '' - string += f'{name}' - string += f'{str(value)}' - string += '' - - return string + return f'{name}{str(value)}' def create_cycledef(group=None, start=None, stop=None, step=None): - ''' - create an Rocoto cycle definition + """ + create a Rocoto cycle definition returns the environment variable as a string :param group: cycle definition group name :type group: str @@ -353,18 +350,13 @@ def create_cycledef(group=None, start=None, stop=None, step=None): :type value: str or float or int or unicode :return: Rocoto cycledef variable string :rtype: str - ''' + """ - string = '' - string += f'' - string += f'{start} {stop} {step}' - string += '' + return f'{start} {stop} {step}' - return string - -def create_entity(name=None, value=None): - ''' +def create_entity(name: str, value: Union[str, float, int]) -> str: + """ create an XML ENTITY variable given name and value returns the variable as a string :param name: name of the variable @@ -373,7 +365,6 @@ def create_entity(name=None, value=None): :type value: str or float or int or unicode :return: XML entity variable key-value pair :rtype: str - ''' - - return f'' + """ + return f'' diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 43ee7ab8caa..4fd4d4dd6c0 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -672,7 +672,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} # deps.append(rocoto.add_dependency(dep_dict)) # dependencies = rocoto.create_dependency(dep=deps) - # task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies) + # task = wfu.create_wf_task('wavestat', cdump=cdump, envars=envars, dependency=dependencies) # dict_tasks['%swavestat' % cdump] = task # vrfy diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index e0fd8021231..70e3e9f0894 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -21,32 +21,43 @@ import os import sys import re +from typing import List, Dict, Any + import numpy as np from datetime import datetime from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter import rocoto +from hosts import Host import workflow_utils as wfu +from configuration import Configuration + +taskplan = ['getic', 'init', 'coupled_ic', 'aerosol_init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', + 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', + 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', + 'awips', 'vrfy', 'metp', 'arch', 'ocnpost'] -taskplan = ['getic', 'init', 'coupled_ic', 'aerosol_init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch', 'ocnpost'] def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir',help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - parser.add_argument('--cdump',help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) + parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', + formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, + required=False, default=os.environ['PWD']) + parser.add_argument('--self', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', + required=False) args = parser.parse_args() - configs = wfu.get_configs(args.expdir) + cfg = Configuration(args.expdir) - _base = wfu.config_parser([wfu.find_config('config.base', configs)]) + _base = cfg.parse_config('config.base') - if not os.path.samefile(args.expdir,_base['EXPDIR']): + if not os.path.samefile(args.expdir, _base['EXPDIR']): print('MISMATCH in experiment directories!') print(f'''config.base: EXPDIR = {repr(_base['EXPDIR'])}''') print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - dict_configs = wfu.source_configs(configs, taskplan) + dict_configs = wfu.source_configs(cfg, taskplan) dict_configs['base']['CDUMP'] = args.cdump @@ -59,115 +70,20 @@ def main(): return -def get_preamble(): +def get_resources(dict_configs, task_plan, self='gdas'): ''' - Generate preamble for XML + Create resource entities ''' - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' + resources = wfu.get_taskplan_resources(dict_configs, task_plan, cdump=cdump) - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - hpssarch = base.get('HPSSARCH', 'NO').upper() + scheduler = Host.get_scheduler strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if base['INTERVAL'] is None: - print('cycle INTERVAL cannot be None') - sys.exit(1) - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if scheduler in ['slurm']: - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_resources(dict_configs, cdump='gdas'): - ''' - Create resource entities - ''' - - strings = [] - strings.append('\t\n') strings.append('\n') - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - reservation = base.get('RESERVATION', 'NONE').upper() - scheduler = wfu.get_scheduler(machine) - - do_wave = base.get('DO_WAVE', 'NO').upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_metp = base.get('DO_METP', 'NO').upper() - - for task in taskplan: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) + for task in task_plan: taskstr = f'{task.upper()}_{cdump.upper()}' @@ -222,7 +138,7 @@ def get_postgroups(post, cdump='gdas'): return fhrgrp, fhrdep, fhrlst -def get_workflow(dict_configs, cdump='gdas'): +def get_workflow(dict_configs, self='gdas'): ''' Create tasks for forecast only workflow ''' @@ -263,8 +179,9 @@ def get_workflow(dict_configs, cdump='gdas'): base_cplic = dict_configs['coupled_ic']['BASE_CPLIC'] # ATM ICs - for file in ['gfs_ctrl.nc'] + [f'{datatype}_data.tile{tile_index}.nc' for datatype in ['gfs', 'sfc'] for tile_index in range(1, n_tiles + 1)]: - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ATMIC']}/@Y@m@d@H/&CDUMP;/{base.get('CASE','C384')}/INPUT/{file}" + for file in ['gfs_ctrl.nc'] + [f'{datatype}_data.tile{tile_index}.nc' for datatype in ['gfs', 'sfc'] for + tile_index in range(1, n_tiles + 1)]: + data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ATMIC']}/@Y@m@d@H/&CDUMP;/{base.get('CASE', 'C384')}/INPUT/{file}" dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -279,7 +196,7 @@ def get_workflow(dict_configs, cdump='gdas'): # Ice ICs if do_ice in ["YES"]: ice_res = base.get('ICERES', '025') - ice_res_dec = f'{float(ice_res)/100:.2f}' + ice_res_dec = f'{float(ice_res) / 100:.2f}' data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ICEIC']}/@Y@m@d@H/ice/{ice_res:03d}/cice5_model_{ice_res_dec}.res_@Y@m@d@H.nc" dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -409,10 +326,10 @@ def get_workflow(dict_configs, cdump='gdas'): # fcst deps = [] data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} + dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} + dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -522,9 +439,9 @@ def get_workflow(dict_configs, cdump='gdas'): # waveawipsbulls if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) @@ -534,7 +451,7 @@ def get_workflow(dict_configs, cdump='gdas'): # waveawipsgridded if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) @@ -732,7 +649,7 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') - #postsnd + # postsnd if do_bufrsnd in ['Y', 'YES']: deps = [] dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} @@ -773,7 +690,7 @@ def get_workflow(dict_configs, cdump='gdas'): # vrfy if do_vrfy in ['Y', 'YES']: deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) @@ -783,7 +700,7 @@ def get_workflow(dict_configs, cdump='gdas'): # metp if do_metp in ['Y', 'YES']: deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;') @@ -798,27 +715,27 @@ def get_workflow(dict_configs, cdump='gdas'): # arch deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) if do_vrfy in ['Y', 'YES']: - dep_dict = {'type':'task', 'name':f'{cdump}vrfy'} + dep_dict = {'type': 'task', 'name': f'{cdump}vrfy'} deps.append(rocoto.add_dependency(dep_dict)) if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} + dep_dict = {'type': 'metatask', 'name': f'{cdump}metp'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':f'{hpssarch}'} + dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': f'{hpssarch}'} deps.append(rocoto.add_dependency(dep_dict)) if do_wave in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave_bnd in ['YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) if do_ocean in ['Y', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}ocnpost'} - deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{cdump}ocnpost'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) tasks.append(task) @@ -827,7 +744,7 @@ def get_workflow(dict_configs, cdump='gdas'): return ''.join(tasks) -def get_workflow_body(dict_configs, cdump='gdas'): +def get_workflow_body(dict_configs, self='gdas'): ''' Create the workflow body ''' @@ -837,7 +754,8 @@ def get_workflow_body(dict_configs, cdump='gdas'): strings.append('\n') strings.append(']>\n') strings.append('\n') - strings.append('\n') + strings.append( + '\n') strings.append('\n') strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') strings.append('\n') @@ -857,7 +775,6 @@ def create_xml(dict_configs): XML directory containing XML templates, create the workflow XML ''' - dict_configs['base']['INTERVAL'] = wfu.get_gfs_interval(dict_configs['base']['gfs_cyc']) base = dict_configs['base'] @@ -876,7 +793,7 @@ def create_xml(dict_configs): if 'MEMORY' not in each_line: temp_workflow += each_line else: - if any( substring in each_line for substring in memory_dict): + if any(substring in each_line for substring in memory_dict): temp_workflow += each_line workflow = temp_workflow @@ -892,6 +809,7 @@ def create_xml(dict_configs): return + if __name__ == '__main__': main() sys.exit(0) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index e69de29bb2d..f16e704c09a 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -0,0 +1,740 @@ +#!/usr/bin/env python + +import numpy as np +from typing import Dict, Any +import rocoto +import workflow_utils as wfu + + +class Tasks: + + def __init__(self, dict_configs: Dict[str, Dict[str, Any]], cdump: str) -> None: + + self.cdump = cdump + + # Save dict_configs and base in the internal state (never know where it may be needed) + self._dict_configs = dict_configs + self._base = dict_configs['base'] + + self.do_hybvar = self._base.get('DOHYBVAR', False) + self.do_wave = self._base.get('DO_WAVE', False) + self.do_ocean = self._base.get('DO_OCEAN', False) + self.do_ice = self._base.get('DO_ICE', False) + self.do_aero = self._base.get('DO_AERO', False) + self.do_gldas = self._base.get('DO_GLDAS', False) + self.do_bufrsnd = self._base.get('DO_BUFRSND', False) + self.do_gempak = self._base.get('DO_GEMPAK', False) + self.do_awips = self._base.get('DO_AWIPS', False) + self.do_wafs = self._base.get('DO_WAFS', False) + self.do_vrfy = self._base.get('DO_VRFY', True) + self.do_metp = self._base.get('DO_METP', False) + + if self.do_wave: + self.do_wave_bnd = self._dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', True) + wave_cdump = self._base.get('WAVE_CDUMP', 'BOTH').lower() + if wave_cdump in ['both']: + self.wave_cdumps = ['gfs', 'gdas'] + elif wave_cdump in ['gfs', 'gdas']: + self.wave_cdumps = [wave_cdump] + + envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'), + 'HOMEgfs': self._base.get('HOMEgfs'), + 'EXPDIR': self._base.get('EXPDIR'), + 'CDUMP': self.cdump, + 'CDATE': '@Y@m@d@H', + 'PDY': '@Y@m@d', + 'cyc': '@H'} + self.envars = self._set_envars(envar_dict) + + if self.do_hybvar: + self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) + eupd_cdump = self._base.get('EUPD_CYC', 'gdas').lower() + if eupd_cdump in ['both']: + self.eupd_cdumps = ['gfs', 'gdas'] + elif eupd_cdump in ['gfs', 'gdas']: + self.eupd_cdumps = [eupd_cdump] + + self.hyb_cycledef = 'enkf' # TODO - sort this out a bit more + + @staticmethod + def _set_envars(envar_dict) -> list: + + envars = [] + for key, value in envar_dict.items(): + envars.append(rocoto.create_envar(name=key, value=str(value))) + + return envars + + @staticmethod + def _get_hybgroups(nens: int, nmem_per_group: int): + ngrps = nens / nmem_per_group + groups = ' '.join([f'{x:02d}' for x in range(1, int(ngrps) + 1)]) + return groups + + def waveinit(self): + task = wfu.create_wf_task('waveinit', cdump=self.cdump, envar=self.envars) + return task + + def waveprep(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self}waveinit'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('waveprep', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task + + def aerosol_init(self): # TODO + raise NotImplementedError('aerosol_init has not been implemented yet!') + + def anal(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_hybvar: + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('anal', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task + + def analcalc(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_hybvar and self.cdump in ['gdas']: + dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('analcalc', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task + + def analdiag(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('analdiag', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task + + def gldas(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gldas', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task + + def fcst(self, fcst_type): + + fcst_map = {'free': self._fcst_free, + 'cycled': self._fcst_cycled} + + try: + task = fcst_map[fcst_type] + except KeyError: + raise NotImplementedError(f'{fcst_type} is not a valid type.\n' + + 'Currently supported forecast types are:\n' + + f'{" | ".join(fcst_map.keys())}') + + return task + + @property + def _fcst_free(self): + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + if self.do_wave and self.cdump in self.wave_cdumps: + dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.do_aero: # TODO - check dependency on cycleexist of previous cycle? + dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + + task = wfu.create_wf_task('fcst', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + @property + def _fcst_cycled(self): + + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + dep = rocoto.add_dependency(dep_dict) + dependencies = rocoto.create_dependency(dep=dep) + + if self.do_gldas and self.cdump in ['gdas']: + dep_dict = {'type': 'task', 'name': f'{self.cdump}gldas'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.do_wave and self.cdump in self.wave_cdumps: + dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + + task = wfu.create_wf_task('fcst', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def post(self): + return self._post_task('post', add_anl_to_post=True) + + def ocnpost(self): + return self._post_task('ocnpost', add_anl_to_post=False) + + def _post_task(self, task_name, add_anl_to_post=False): + if task_name not in ['post', 'ocnpost']: + raise KeyError(f'Invalid post-processing task: {task_name}') + + if task_name in 'ocnpost': + add_anl_to_post = False + + def _get_postgroups(cdump, config, add_anl=False): + + fhmin = config['FHMIN'] + fhmax = config['FHMAX'] + fhout = config['FHOUT'] + + # Get a list of all forecast hours + fhrs = [] + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax + fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max( + [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) + fhout = config['FHOUT_GFS'] + fhmax_hf = config['FHMAX_HF_GFS'] + fhout_hf = config['FHOUT_HF_GFS'] + fhrs_hf = range(fhmin, fhmax_hf + fhout_hf, fhout_hf) + fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + + npostgrp = config['NPOSTGRP'] + ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) + + fhrs = [f'f{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [fhr.tolist() for fhr in fhrs] + + anl = ['anl'] if add_anl else [] + + grp = ' '.join(anl + [f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) + dep = ' '.join(anl + [fhr[-1] for fhr in fhrs]) + lst = ' '.join(anl + ['_'.join(fhr) for fhr in fhrs]) + + return grp, dep, lst + + deps = [] + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.log#dep#.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + postenvars = self.envars + postenvar_dict = {'FHRGRP': '#grp#', + 'FHRLST': '#lst#', + 'ROTDIR': self._base.get('ROTDIR')} + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = _get_postgroups(self.cdump, self._dict_configs[task_name], add_anl=add_anl_to_post) + vardict = {varname2: varval2, varname3: varval3} + + task = wfu.create_wf_task(task_name, cdump=self.cdump, envar=postenvars, dependency=dependencies, + metatask=task_name, varname=varname1, varval=varval1, vardict=vardict) + + return task + + def wavepostsbs(self): + deps = [] + for wave_grid in self._dict_configs['wavepostsbs']['waveGRD'].split(): + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/wave/rundata/{self.cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('wavepostsbs', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wavepostbndpnt(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('wavepostbndpnt', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wavepostbndpntbll(self): + deps = [] + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.logf180.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('wavepostbndpntbll', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wavepostpnt(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_wave_bnd: + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpntbll'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('wavepostpnt', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wavegempak(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('wavegempak', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def waveawipsbulls(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('waveawipsbulls', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def waveawipsgridded(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('waveawipsgridded', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wafs(self): + return self._wafs_task('wafs') + + def wafsgcip(self): + return self._wafs_task('wafsgcip') + + def wafsgrib2(self): + return self._wafs_task('wafsgrib2') + + def wafsgrib20p25(self): + return self._wafs_task('wafsgrib20p25') + + def _wafs_task(self, task_name): + if task_name not in ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25']: + raise KeyError(f'Invalid WAFS task: {task_name}') + + deps = [] + fhrlst = [6] + [*range(12, 36 + 3, 3)] + for fhr in fhrlst: + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.wafs.grb2if{fhr:03d}' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task(task_name, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wafsblending(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}wafsgrib2'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def wafsblending0p25(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}wafsgrib20p25'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending0p25', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def postsnd(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('postsnd', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def awips(self): + + def _get_awipsgroups(cdump, config): + + fhmin = config['FHMIN'] + fhmax = config['FHMAX'] + fhout = config['FHOUT'] + + # Get a list of all forecast hours + fhrs = [] + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax + fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max( + [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) + fhout = config['FHOUT_GFS'] + fhmax_hf = config['FHMAX_HF_GFS'] + fhout_hf = config['FHOUT_HF_GFS'] + if fhmax > 240: + fhmax = 240 + if fhmax_hf > 240: + fhmax_hf = 240 + fhrs_hf = range(fhmin, fhmax_hf + fhout_hf, fhout_hf) + fhrs = fhrs_hf + range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout) + + nawipsgrp = config['NAWIPSGRP'] + ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) + + fhrs = [f'f{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [fhr.tolist() for fhr in fhrs] + + grp = ' '.join([f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) + dep = ' '.join([fhr[-1] for fhr in fhrs]) + lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) + + return grp, dep, lst + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + awipsenvars = self.envars + awipsenvar_dict = {'FHRGRP': '#grp#', + 'FHRLST': '#lst#', + 'ROTDIR': self._base.get('ROTDIR')} + for key, value in awipsenvar_dict.items(): + awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._dict_configs['awips']) + vardict = {varname2: varval2, varname3: varval3} + + task = wfu.create_wf_task('awips', cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + + return task + + def gempak(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gempak', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def vrfy(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('vrfy', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def metp(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + metpenvars = self.envars + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS'), # TODO - in Forecast-only, this is `SDATE` on the RHS + 'METPCASE': '#metpcase#'} + for key, value in metpenvar_dict.items(): + metpenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1 = 'metpcase' + varval1 = 'g2g1 g2o1 pcp1' + + task = wfu.create_wf_task('metp', cdump=self.cdump, envar=metpenvars, dependency=dependencies, + metatask='metp', varname=varname1, varval=varval1) + + return task + + def arch(self): + deps = [] + if self.do_vrfy: + dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_metp and self.cdump in ['gfs']: + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_wave: + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_wave_bnd and self.cdump in ['gfs']: + dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('arch', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + # Start of ensemble tasks + def eobs(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('eobs', cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) + + return task + + def eomg(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + eomgenvars = self.envars + eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) + + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['eobs']['NMEM_EOMGGRP']) + + task = wfu.create_wf_task('eomg', cdump=self.cdump, envar=eomgenvars, dependency=dependencies, + metatask='eomn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) + + return task + + def ediag(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('ediag', cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) + + return task + + def eupd(self): + deps = [] + if self.lobsdiag_forenkf: + dep_dict = {'type': 'task', 'name': f'{self.cdump}ediag'} + else: + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}eomn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + task = wfu.create_wf_task('eupd', cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) + + return task + + @staticmethod + def is_this_a_gdas_task(cdump, task_name): + if cdump != 'gdas': + raise TypeError(f'{task_name} must be part of the "gdas" cycle and not {cdump}') + + def ecen(self): + + self.is_this_a_gdas_task(self.cdump, 'ecen') + + def _get_ecengroups(): + + if self._base.get('DOIAU_ENKF', False): + fhrs = list(self._base.get('IAUFHRS', '6').split(',')) + + necengrp = self._dict_configs['ecen']['NECENGRP'] + ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) + + fhrs = [f'{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [fhr.tolist() for fhr in fhrs] + + grp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) + dep = ' '.join([f[-1] for f in fhrs]) + lst = ' '.join(['_'.join(f) for f in fhrs]) + + else: + grp = '000' + dep = 'f006' + lst = 'f006' + + return grp, dep, lst + + eupd_cdump = 'gdas' if 'gdas' in self.eupd_cdumps else 'gfs' + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + ecenenvars = self.envars + ecenenvar_dict = {'FHRGRP': '#grp#', + 'FHRLST': '#lst#'} + for key, value in ecenenvar_dict.items(): + ecenenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = _get_ecengroups() + vardict = {varname2: varval2, varname3: varval3} + + task = wfu.create_wf_task('ecen', cdump=self.cdump, envar=ecenenvars, dependency=dependencies, + metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict, + cycledef=self.hyb_cycledef) + return task + + def esfc(self): + + self.is_this_a_gdas_task(self.cdump, 'esfc') + + eupd_cdump = 'gdas' if 'gdas' in self.eupd_cdumps else 'gfs' + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('esfc', cdump='gdas', envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) + + return task + + def efcs(self): + + self.is_this_a_gdas_task(self.cdump, 'efcs') + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}ecmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + efcsenvars = self.envars + efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) + + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['efcs']['NMEM_EFCSGRP']) + + task = wfu.create_wf_task('efcs', cdump=self.cdump, envar=efcsenvars, dependency=dependencies, + metatask='efmn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) + + return task + + def echgres(self): + + self.is_this_a_gdas_task(self.cdump, 'echgres') + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}efmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('echgres', cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) + + return task + + def epos(self): + + self.is_this_a_gdas_task(self.cdump, 'epos') + + def _get_eposgroups(epos): + fhmin = epos['FHMIN_ENKF'] + fhmax = epos['FHMAX_ENKF'] + fhout = epos['FHOUT_ENKF'] + fhrs = range(fhmin, fhmax + fhout, fhout) + + neposgrp = epos['NEPOSGRP'] + ngrps = neposgrp if len(fhrs) > neposgrp else len(fhrs) + + fhrs = [f'f{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [f.tolist() for f in fhrs] + + grp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) + dep = ' '.join([f[-1] for f in fhrs]) + lst = ' '.join(['_'.join(f) for f in fhrs]) + + return grp, dep, lst + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}efmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + eposenvars = self.envars + eposenvar_dict = {'FHRGRP': '#grp#', + 'FHRLST': '#lst#'} + for key, value in eposenvar_dict.items(): + eposenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = _get_eposgroups(self._dict_configs['epos']) + vardict = {varname2: varval2, varname3: varval3} + + task = wfu.create_wf_task('epos', cdump=self.cdump, envar=eposenvars, dependency=dependencies, + metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, + cycledef=self.hyb_cycledef) + + return task + + def earc(self): + + self.is_this_a_gdas_task(self.cdump, 'earc') + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}epmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + earcenvars = self.envars + earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) + + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['efcs']['NMEM_EARCGRP']) + + task = wfu.create_wf_task('earc', cdump=self.cdump, envar=earcenvars, dependency=dependencies, + metatask='eamn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) + + return task diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index d981a17c257..17c94899022 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -1,468 +1,192 @@ #!/usr/bin/env python -''' +""" Module containing functions all workflow setups require -''' -import random -import re -import os, sys, stat -import socket -import glob -import subprocess +""" +import os import numpy as np from distutils.spawn import find_executable -from datetime import datetime, timedelta -import rocoto - -DATE_ENV_VARS=['CDATE','SDATE','EDATE'] -SCHEDULER_MAP={'HERA':'slurm', - 'JET':'slurm', - 'ORION':'slurm', - 'WCOSS':'lsf', - 'WCOSS_DELL_P3':'lsf', - 'WCOSS_C':'lsfcray'} - -class UnknownMachineError(Exception): pass -class UnknownConfigError(Exception): pass -class ShellScriptException(Exception): - def __init__(self,scripts,errors): - self.scripts = scripts - self.errors = errors - super(ShellScriptException,self).__init__( - str(errors)+ - ': error processing'+ - (' '.join(scripts))) - -class HostInfo: - ''' - Gather Host specific information. - Someday the content will be pushed out of the code and into a yaml or something. - ''' - - def __init__(self, machine, **inputs): - - supported_hosts = { - 'WCOSS_C': self.wcoss_c, - 'WCOSS_DELL_P3': self.wcoss_dell_p3, - 'WCOSS_DELL_P3p5': self.wcoss_dell_p3p5, - 'HERA': self.hera, - 'ORION': self.orion} - - try: - self.info = supported_hosts[machine.upper()] - except KeyError: - raise NotImplementedError(f'{machine} is not a supported host.\n' + - 'Currently supported hosts are:\n' + - f'{" | ".join(supported_hosts.keys())}') - - self.machine = machine +from typing import List, Dict, Any +from rocoto import create_task, create_metatask +from hosts import Host +from configuration import Configuration - return +SERVICE_TASKS = ['arch', 'earc', 'getic'] - @property - def wcoss_c(self): - - info = { - 'base_git': '/gpfs/hps3/emc/global/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/hps/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/hps/nco/ops/com"}', - 'homedir': '/gpfs/hps3/emc/global/noscrub/$USER', - 'stmp': '/gpfs/hps2/stmp/$USER', - 'ptmp': '/gpfs/hps2/ptmp/$USER', - 'noscrub': '/gpfs/hps3/emc/global/noscrub/$USER', - 'account': 'GFS-DEV', - 'queue': 'dev', - 'queue_service': 'dev_transfer', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - - @property - def wcoss_dell_p3(self): - - info = { - 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', - 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', - 'stmp': '/gpfs/dell3/stmp/$USER', - 'ptmp': '/gpfs/dell3/ptmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'GFS-DEV', - 'queue': 'dev', - 'queue_service': 'dev_transfer', - 'partition_batch': None, - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @property - def wcoss_dell_p3p5(self): - - info = { - 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', - 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', - 'stmp': '/gpfs/dell3/stmp/$USER', - 'ptmp': '/gpfs/dell3/ptmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'GFS-DEV', - 'queue': 'dev2', - 'queue_service': 'dev2_transfer', - 'partition_batch': None, - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @property - def hera(self): - - info = { - 'base_git': '/scratch1/NCEPDEV/global/glopara/git', - 'base_svn': '/scratch1/NCEPDEV/global/glopara/svn', - 'dmpdir': '/scratch1/NCEPDEV/global/glopara/dump', - 'nwprod': '/scratch1/NCEPDEV/global/glopara/nwpara', - 'comroot': '/scratch1/NCEPDEV/global/glopara/com', - 'homedir': '/scratch1/NCEPDEV/global/$USER', - 'stmp': '/scratch1/NCEPDEV/stmp2/$USER', - 'ptmp': '/scratch1/NCEPDEV/stmp4/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'fv3-cpu', - 'queue': 'batch', - 'queue_service': 'service', - 'partition_batch': 'hera', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @property - def orion(self): - - info = { - 'base_git': '/work/noaa/global/glopara/git', - 'base_svn': '/work/noaa/global/glopara/svn', - 'dmpdir': '/work/noaa/rstprod/dump', - 'nwprod': '/work/noaa/global/glopara/nwpara', - 'comroot': '/work/noaa/global/glopara/com', - 'homedir': '/work/noaa/global/$USER', - 'stmp': '/work/noaa/stmp/$USER', - 'ptmp': '/work/noaa/stmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'fv3-cpu', - 'queue': 'batch', - 'queue_service': 'service', - 'partition_batch': 'orion', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'NO', - 'localarch': 'NO', - 'atardir': '$NOSCRUB/archive_rotdir/$PSLOT', - } - - return info - -def get_shell_env(scripts): - vars=dict() - runme=''.join([ f'source {s} ; ' for s in scripts ]) - magic=f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' - runme+=f'/bin/echo -n "{magic}" ; /usr/bin/env -0' - with open('/dev/null','w') as null: - env=subprocess.Popen(runme,shell=True,stdin=null.fileno(), - stdout=subprocess.PIPE) - (out,err)=env.communicate() - out = out.decode() - begin=out.find(magic) - if begin<0: - raise ShellScriptException(scripts,'Cannot find magic string; ' - 'at least one script failed: '+repr(out)) - for entry in out[begin+len(magic):].split('\x00'): - iequal=entry.find('=') - vars[entry[0:iequal]] = entry[iequal+1:] - return vars - -def get_script_env(scripts): - default_env=get_shell_env([]) - and_script_env=get_shell_env(scripts) - vars_just_in_script=set(and_script_env)-set(default_env) - union_env=dict(default_env) - union_env.update(and_script_env) - return dict([ (v,union_env[v]) for v in vars_just_in_script ]) - -def cast_or_not(type,value): - try: - return type(value) - except ValueError: - return value -def get_configs(expdir): +def source_configs(config: Configuration, tasks) -> dict: """ - Given an experiment directory containing config files, - return a list of configs minus the ones ending with ".default" + Given the configuration object and tasks, + return a dictionary for each task + Every task depends on "config.base" """ - result=list() - for config in glob.glob(f'{expdir}/config.*'): - if not config.endswith('.default'): - result.append(config) - return result - -def find_config(config_name, configs): - - for config in configs: - if config_name == os.path.basename(config): - return config - - raise UnknownConfigError(f'{config_name} does not exist (known: {repr(config_name)}), ABORT!') - -def source_configs(configs, tasks): - ''' - Given list of config files, source them - and return a dictionary for each task - Every task depends on config.base - ''' - dict_configs = {} + dict_configs = dict() # Return config.base as well - dict_configs['base'] = config_parser([find_config('config.base', configs)]) + dict_configs['base'] = config.parse_config('config.base') # Source the list of input tasks for task in tasks: - files = [] - - files.append(find_config('config.base', configs)) + # All tasks must source config.base first + files = ['config.base'] if task in ['eobs', 'eomg']: - files.append(find_config('config.anal', configs)) - files.append(find_config('config.eobs', configs)) + files.append('config.anal') + files.append('config.eobs') elif task in ['eupd']: - files.append(find_config('config.anal', configs)) - files.append(find_config('config.eupd', configs)) + files.append('config.anal') + files.append('config.eupd') elif task in ['efcs']: - files.append(find_config('config.fcst', configs)) - files.append(find_config('config.efcs', configs)) + files.append('config.fcst') + files.append('config.efcs') elif 'wave' in task: - files.append(find_config(f'config.wave', configs)) - files.append(find_config(f'config.{task}', configs)) + files.append('config.wave') + files.append(f'config.{task}') else: - files.append(find_config(f'config.{task}', configs)) + files.append(f'config.{task}') print(f'sourcing config.{task}') - dict_configs[task] = config_parser(files) + dict_configs[task] = config.parse_config(files) return dict_configs -def config_parser(files): - """ - Given the name of config file, key-value pair of all variables in the config file is returned as a dictionary - :param files: config file or list of config files - :type files: list or str or unicode - :return: Key value pairs representing the environment variables defined - in the script. - :rtype: dict - """ - if isinstance(files,(str, bytes)): - files=[files] - varbles=dict() - for key,value in get_script_env(files).items(): - if key in DATE_ENV_VARS: # likely a date, convert to datetime - varbles[key] = datetime.strptime(value,'%Y%m%d%H') - elif '.' in value: # Likely a number and that too a float - varbles[key] = cast_or_not(float,value) - else: # Still could be a number, may be an integer - varbles[key] = cast_or_not(int,value) - - return varbles - -def detectMachine(): - - machines = ['HERA', 'ORION', 'WCOSS_C', 'WCOSS_DELL_P3', 'JET'] - - if os.path.exists('/scratch1/NCEPDEV'): - return 'HERA' - elif os.path.exists('/work/noaa'): - return 'ORION' - elif os.path.exists('/gpfs') and os.path.exists('/etc/SuSE-release'): - return 'WCOSS_C' - elif os.path.exists('/gpfs/dell2'): - return 'WCOSS_DELL_P3' - elif os.path.exists('/lfs4/HFIP'): - return 'JET' - else: - print(f'workflow is currently only supported on: {machines}') - raise NotImplementedError('Cannot auto-detect platform, ABORT!') - -def get_scheduler(machine): - try: - return SCHEDULER_MAP[machine] - except KeyError: - raise UnknownMachineError(f'Unknown machine: {machine}, ABORT!') - -def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, \ - metatask=None, varname=None, varval=None, vardict=None, \ +def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, + metatask=None, varname=None, varval=None, vardict=None, final=False): - if metatask is None: taskstr = f'{task}' else: taskstr = f'{task}#{varname}#' metataskstr = f'{cdump}{metatask}' - metatask_dict = {'metataskname': metataskstr, \ - 'varname': f'{varname}', \ - 'varval': f'{varval}', \ + metatask_dict = {'metataskname': metataskstr, + 'varname': f'{varname}', + 'varval': f'{varval}', 'vardict': vardict} taskstr = f'{cdump}{taskstr}' cycledefstr = cdump if cycledef is None else cycledef - task_dict = {'taskname': f'{taskstr}', \ - 'cycledef': f'{cycledefstr}', \ - 'maxtries': '&MAXTRIES;', \ - 'command': f'&JOBS_DIR;/{task}.sh', \ - 'jobname': f'&PSLOT;_{taskstr}_@H', \ - 'account': '&ACCOUNT;', \ - 'queue': f'&QUEUE_{task.upper()}_{cdump.upper()};', \ - 'walltime': f'&WALLTIME_{task.upper()}_{cdump.upper()};', \ - 'native': f'&NATIVE_{task.upper()}_{cdump.upper()};', \ - 'memory': f'&MEMORY_{task.upper()}_{cdump.upper()};', \ - 'resources': f'&RESOURCES_{task.upper()}_{cdump.upper()};', \ - 'log': f'&ROTDIR;/logs/@Y@m@d@H/{taskstr}.log', \ - 'envar': envar, \ - 'dependency': dependency, \ + task_dict = {'taskname': f'{taskstr}', + 'cycledef': f'{cycledefstr}', + 'maxtries': '&MAXTRIES;', + 'command': f'&JOBS_DIR;/{task}.sh', + 'jobname': f'&PSLOT;_{taskstr}_@H', + 'account': '&ACCOUNT;', + 'queue': f'&QUEUE_{task.upper()}_{cdump.upper()};', + 'walltime': f'&WALLTIME_{task.upper()}_{cdump.upper()};', + 'native': f'&NATIVE_{task.upper()}_{cdump.upper()};', + 'memory': f'&MEMORY_{task.upper()}_{cdump.upper()};', + 'resources': f'&RESOURCES_{task.upper()}_{cdump.upper()};', + 'log': f'&ROTDIR;/logs/@Y@m@d@H/{taskstr}.log', + 'envars': envar, + 'dependency': dependency, 'final': final} # Add partition for machines using slurm - if get_scheduler(detectMachine()) in ['slurm']: + if Host.get_scheduler in ['slurm']: task_dict['partition'] = f'&PARTITION_{task.upper()}_{cdump.upper()};' if metatask is None: - task = rocoto.create_task(task_dict) + task = create_task(task_dict) else: - task = rocoto.create_metatask(task_dict, metatask_dict) + task = create_metatask(task_dict, metatask_dict) task = ''.join(task) return task -def get_gfs_interval(gfs_cyc): - ''' - return interval in hours based on gfs_cyc - ''' - - # Get interval from cyc_input - if gfs_cyc == 0: - interval = None - if gfs_cyc == 1: - interval = '24:00:00' - elif gfs_cyc == 2: - interval = '12:00:00' - elif gfs_cyc == 4: - interval = '06:00:00' - - return interval - +def get_gfs_interval(gfs_cyc: int) -> str: + """ + return interval in hours based on gfs_cyc + """ -def get_resources(machine, cfg, task, reservation, cdump='gdas'): + gfs_internal_map = {'0': None, '1': '24:00:00', '2': '12:00:00', '4': '06:00:00'} - scheduler = get_scheduler(machine) + try: + return gfs_internal_map[str(gfs_cyc)] + except KeyError: + raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - if cdump in ['gfs'] and f'wtime_{task}_gfs' in cfg.keys(): - wtimestr = cfg[f'wtime_{task}_gfs'] - else: - wtimestr = cfg[f'wtime_{task}'] - ltask = 'eobs' if task in ['eomg'] else task +def get_resource(task_dict: dict, task_name: str, cdump: str = 'gdas') -> dict: + """ + Given a task name (task_name) and its configuration (task_dict), + return a dictionary of resources (task_resource) used by the task. + Task resource dictionary includes: + account, walltime, cores, nodes, ppn, threads, memory, queue, partition, native + """ - memory = cfg.get(f'memory_{ltask}', None) + scheduler = Host.get_scheduler - if cdump in ['gfs'] and f'npe_{task}_gfs' in cfg.keys(): - tasks = cfg[f'npe_{ltask}_gfs'] - else: - tasks = cfg[f'npe_{ltask}'] + account = task_dict['ACCOUNT'] - if cdump in ['gfs'] and f'npe_node_{task}_gfs' in cfg.keys(): - ppn = cfg[f'npe_node_{ltask}_gfs'] - else: - ppn = cfg[f'npe_node_{ltask}'] + walltime = task_dict[f'wtime_{task_name}'] + if cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_dict.keys(): + walltime = task_dict[f'wtime_{task_name}_gfs'] - if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION', 'JET' ]: - if cdump in ['gfs'] and f'nth_{task}_gfs' in cfg.keys(): - threads = cfg[f'nth_{ltask}_gfs'] - else: - threads = cfg[f'nth_{ltask}'] + cores = task_dict[f'npe_{task_name}'] + if cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_dict.keys(): + cores = task_dict[f'npe_{task_name}_gfs'] - nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn))) + ppn = task_dict[f'npe_node_{task_name}'] + if cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_dict.keys(): + ppn = task_dict[f'npe_node_{task_name}_gfs'] - memstr = '' if memory is None else str(memory) - natstr = '' + nodes = np.int(np.ceil(np.float(cores) / np.float(ppn))) - if scheduler in ['slurm']: - natstr = '--export=NONE' + threads = task_dict[f'nth_{task_name}'] + if cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_dict.keys(): + threads = task_dict[f'nth_{task_name}_gfs'] - if machine in ['HERA', 'JET', 'ORION', 'WCOSS_C', 'WCOSS_DELL_P3']: + compute = f'{nodes}:ppn={ppn}:tpp={threads}' # TODO - remove dependence on compute - if machine in ['HERA', 'JET', 'ORION']: - resstr = f'{nodes}:ppn={ppn}:tpp={threads}' - else: - resstr = f'{nodes}:ppn={ppn}' + memory = task_dict.get(f'memory_{task_name}', None) - if machine in ['WCOSS_C'] and task in ['arch', 'earc', 'getic']: - resstr += '' + native = '--export=NONE' if scheduler in ['slurm'] else None - if machine in ['WCOSS_DELL_P3']: - if not reservation in ['NONE']: - natstr = f"-U {reservation} -R 'affinity[core({threads})]'" - else: - natstr = f"-R 'affinity[core({threads})]'" + queue = task_dict['QUEUE'] + if task_name in SERVICE_TASKS and scheduler not in ['slurm']: + queue = task_dict['QUEUE_SERVICE'] - if task in ['arch', 'earc', 'getic']: - natstr = "-R 'affinity[core(1)]'" + partition = None + if scheduler in ['slurm']: + partition = task_dict['PARTITION_SERVICE'] if task_name in SERVICE_TASKS else task_dict['PARTITION_BATCH'] + task_resource = dict() + task_resource['account'] = account + task_resource['walltime'] = walltime + task_resource['nodes'] = nodes + task_resource['cores'] = cores + task_resource['ppn'] = ppn + task_resource['threads'] = threads + task_resource['compute'] = compute # TODO - remove in the future + task_resource['memory'] = memory + task_resource['native'] = native + task_resource['queue'] = queue + task_resource['partition'] = partition - elif machine in ['WCOSS']: - resstr = f'{tasks}' + return task_resource - if task in ['arch', 'earc', 'getic']: - queuestr = '&QUEUE;' if scheduler in ['slurm'] else '&QUEUE_SERVICE;' - else: - queuestr = '&QUEUE;' - return wtimestr, resstr, queuestr, memstr, natstr +def get_taskplan_resources(dict_configs: List[Dict[str, Any]], task_plan: List[str], cdump='gdas') -> Dict[str, Any]: + """ + Given a list of dictionary containing tasks, and a task plan, + return the resource dictionary for the tasks in the task plan + """ + resources = dict() + for task in task_plan: + task_name = cdump + task + resources[task_name] = get_resource(dict_configs[task], task, cdump=cdump) + return resources def create_crontab(base, cronint=5): - ''' - Create crontab to execute rocotorun every cronint (5) minutes - ''' + """ + Create crontab to execute rocotorun every cronint (5) minutes + """ # No point creating a crontab if rocotorun is not available. rocotoruncmd = find_executable('rocotorun') @@ -470,56 +194,22 @@ def create_crontab(base, cronint=5): print('Failed to find rocotorun, crontab will not be created') return -# Leaving the code for a wrapper around crontab file if needed again later -# if check_slurm(): -# -# cronintstr = '*/%d * * * *' % cronint -# rocotorunstr = '%s -d %s/%s.db -w %s/%s.xml' % (rocotoruncmd, base['EXPDIR'], base['PSLOT'], base['EXPDIR'], base['PSLOT']) -# -# wrapper_strings = [] -# wrapper_strings.append('#!/bin/env tcsh\n') -# wrapper_strings.append('\n') -# wrapper_strings.append('module load slurm\n') -# wrapper_strings.append('module load rocoto/1.3.0-RC4\n') -# wrapper_strings.append('\n') -# wrapper_strings.append(rocotorunstr) -# -# hostname = 'tfe02' -# script_file = os.path.join(base['EXPDIR'], '%s.sh' % base['PSLOT']) -# -# fh = open(script_file, 'w') -# fh.write(''.join(wrapper_strings)) -# os.chmod(script_file,stat.S_IRWXU|stat.S_IRWXG|stat.S_IRWXO) -# fh.close() -# -# rocotorunstr = 'ssh %s %s/%s.sh' % (socket.gethostname(), base['EXPDIR'], base['PSLOT']) -# -# else: - rocotorunstr = f'''{rocotoruncmd} -d {base['EXPDIR']}/{base['PSLOT']}.db -w {base['EXPDIR']}/{base['PSLOT']}.xml''' cronintstr = f'*/{cronint} * * * *' - # On WCOSS, rocoto module needs to be loaded everytime cron runs - if base['machine'] in ['WCOSS']: - rocotoloadstr = '. /usrx/local/Modules/default/init/sh; module use -a /usrx/local/emc_rocoto/modulefiles; module load rocoto/1.3.0rc2)' - rocotorunstr = f'({rocotoloadstr} {rocotorunstr})' - try: - REPLYTO = os.environ['REPLYTO'] - except: - REPLYTO = '' - - strings = [] - - strings.append('\n') - strings.append(f'''#################### {base['PSLOT']} ####################\n''') - strings.append(f'MAILTO="{REPLYTO}"\n') - strings.append(f'{cronintstr} {rocotorunstr}\n') - strings.append('#################################################################\n') - strings.append('\n') - - fh = open(os.path.join(base['EXPDIR'], f'''{base['PSLOT']}.crontab'''), 'w') - fh.write(''.join(strings)) - fh.close() + replyto = os.environ['REPLYTO'] + except KeyError: + replyto = '' + + strings = ['', + f'#################### {base["PSLOT"]} ####################', + f'MAILTO="{replyto}"', + f'{cronintstr} {rocotorunstr}', + '#################################################################', + ''] + + with open(os.path.join(base['EXPDIR'], f'{base["PSLOT"]}.crontab'), 'w') as fh: + fh.write('\n'.join(strings)) return diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index e69de29bb2d..6b989486ed5 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +from datetime import datetime +from collections import OrderedDict +from hosts import Host +import rocoto + + +def get_preamble(): + """ + Generate preamble for XML + """ + + strings = [] + + strings.append('') + strings.append('') + + return '\n'.join(strings) + + +def get_definitions(base: dict, mode: str) -> str: + """ + Create entities related to the experiment + """ + + host = Host() + + scheduler = host.scheduler + + entity = OrderedDict() + + entity['PSLOT'] = base['PSLOT'] + entity['SDATE'] = base['SDATE'].strftime('%Y%m%d%H%M') + entity['EDATE'] = base['EDATE'].strftime('%Y%m%d%H%M') + + if mode in ['cycled']: + entity['INTERVAL'] = '06:00:00' + if base['gfs_cyc'] in [1, 2, 4]: + entity['SDATE_GFS'] = base['SDATE_GFS'].strftime('%Y%m%d%H%M') + entity['EDATE_GFS'] = base['EDATE_GFS'].strftime('%Y%m%d%H%M') + entity['INTERVAL_GFS'] = base['INTERVAL_GFS'] + entity['DMPDIR'] = base['DMPDIR'] + + elif mode in ['forecast-only']: + entity['INTERVAL'] = base.get('INTERVAL_GFS', '24:00:00') + entity['CDUMP'] = base['CDUMP'] + entity['CASE'] = base['CASE'] # TODO - is this really used in the XML? + entity['ICSDIR'] = base['ICSDIR'] + + entity['EXPDIR'] = base['EXPDIR'] + entity['ROTDIR'] = base['ROTDIR'] + entity['RUN_ENVIR'] = base.get('RUN_ENVIR', 'emc') + entity['HOMEgfs'] = base['HOMEgfs'] + entity['JOBS_DIR'] = base['BASE_JOB'] + entity['ACCOUNT'] = base['ACCOUNT'] + entity['QUEUE'] = base['QUEUE'] + entity['QUEUE_SERVICE'] = base['QUEUE_SERVICE'] + entity['SCHEDULER'] = scheduler + if scheduler in ['slurm']: + entity['PARTITION_BATCH'] = base['PARTITION_BATCH'] + entity['PARTITION_SERVICE'] = base['PARTITION_SERVICE'] + entity['ARCHIVE_TO_HPSS'] = base.get('HPSSARCH', 'NO') + entity['CYCLETHROTTLE'] = base.get('ROCOTO_CYCLETHROTTLE', 3) + entity['TASKTHROTTLE'] = base.get('ROCOTO_TASKTHROTTLE', 25) + entity['MAXTRIES'] = base.get('ROCOTO_MAXTRIES', 2) + entity['VERBOSITY'] = base.get('ROCOTO_VERBOSITY', 10) + + # Put them all in an XML key-value syntax + strings = [] + for key, value in entity.items(): + strings.append('\t' + rocoto.create_entity(key, value)) + + return '\n'.join(strings) From c7e41bd97010267ed2e7f1989c796ab0ea14d834 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 10 Jun 2022 08:53:35 -0500 Subject: [PATCH 03/41] commit before flight --- ush/rocoto/applications.py | 323 ++++++++ ush/rocoto/rocoto.py | 56 +- ush/rocoto/setup_workflow.py | 1034 +------------------------ ush/rocoto/setup_workflow_fcstonly.py | 701 +---------------- ush/rocoto/workflow_tasks.py | 254 ++++-- ush/rocoto/workflow_utils.py | 160 ++-- ush/rocoto/workflow_xml.py | 81 +- 7 files changed, 742 insertions(+), 1867 deletions(-) create mode 100644 ush/rocoto/applications.py diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py new file mode 100644 index 00000000000..cb9945ba78e --- /dev/null +++ b/ush/rocoto/applications.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python + +from typing import Dict, Any +from configuration import Configuration +import workflow_utils as wfu + + +class Application: + VALID_MODES = ['cycled', 'forecast-only'] + SERVICE_TASKS = ['arch', 'earc', 'getic'] + + def __init__(self, mode: str, configuration: Configuration) -> None: + + self.task_resources = None + self.task_names = None + if mode not in self.VALID_MODES: + raise NotImplementedError(f'{mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + + self.mode = mode + self.cfg = configuration + + _base = self.cfg.parse_config('config.base') + + self.model_app = _base.get('APP', 'ATM') + self.do_hybvar = _base.get('DOHYBVAR', False) + self.do_wave = _base.get('DO_WAVE', False) + self.do_ocean = _base.get('DO_OCEAN', False) + self.do_ice = _base.get('DO_ICE', False) + self.do_aero = _base.get('DO_AERO', False) + self.do_gldas = _base.get('DO_GLDAS', False) + self.do_bufrsnd = _base.get('DO_BUFRSND', False) + self.do_gempak = _base.get('DO_GEMPAK', False) + self.do_awips = _base.get('DO_AWIPS', False) + self.do_wafs = _base.get('DO_WAFS', False) + self.do_vrfy = _base.get('DO_VRFY', True) + self.do_metp = _base.get('DO_METP', False) + + self.do_hpssarch = _base.get('HPSSARCH', False) + + # Get a list of all possible jobs that would be part of the application + jobs_map = {'cycled': self._cycled_jobs, + 'forecast-only': self._forecast_only_jobs} + try: + self.job_names = jobs_map[self.mode] + except KeyError: + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + + # Source the configs for all the jobs in the application + self.job_configs = self.source_configs + + # Update the base config dictionary based on application + upd_base_map = {'cycled': self._cycled_upd_base, + 'forecast-only': self._forecast_only_upd_base} + try: + self.job_configs['base'] = upd_base_map[self.mode](self.job_configs['base']) + except KeyError: + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + + self._base = self.job_configs['base'] + + if self.do_hybvar: + self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) + eupd_cdump = self._base.get('EUPD_CYC', 'gdas').lower() + if eupd_cdump in ['both']: + self.eupd_cdumps = ['gfs', 'gdas'] + elif eupd_cdump in ['gfs', 'gdas']: + self.eupd_cdumps = [eupd_cdump] + + if self.do_wave: + wave_cdump = self._base.get('WAVE_CDUMP', 'BOTH').lower() + if wave_cdump in ['both']: + self.wave_cdumps = ['gfs', 'gdas'] + elif wave_cdump in ['gfs', 'gdas']: + self.wave_cdumps = [wave_cdump] + self.do_wave_bnd = self.job_configs['wavepostsbs'].get('DOBNDPNT_WAVE', False) + + @property + def _cycled_jobs(self): + """ + Returns the jobs that are involved in the cycled app + """ + + jobs = ['prep', + 'anal', 'analdiag', 'analcalc', 'gldas', + 'fcst', 'post', 'vrfy', 'arch'] + + if self.do_hybvar: + jobs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + + if self.do_metp: + jobs += ['metp'] + + if self.do_gempak: + jobs += ['gempak'] + + if self.do_awips: + jobs += ['awips'] + + if self.do_wave: + jobs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + if self.do_gempak: + jobs += ['wavegempak'] + if self.do_awips: + jobs += ['waveawipsbulls', 'waveawipsgridded'] + + if self.do_wafs: + jobs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] + + return jobs + + @property + def _forecast_only_jobs(self): + """ + Returns the jobs that are involved in the forecast-only app + """ + + jobs = ['fcst', 'post', 'vrfy', 'arch'] + + if self.model_app in ['S2S', 'S2SW']: + jobs += ['coupled_ic'] + else: + jobs += ['init'] + if self.do_hpssarch: + jobs += ['getic'] + + if self.do_aero: + jobs += ['aerosol_init'] + + if self.do_ocean or self.do_ice: + jobs += ['ocnpost'] + + if self.do_metp: + jobs += ['metp'] + + if self.do_gempak: + jobs += ['gempak'] + + if self.do_awips: + jobs += ['awips'] + + if self.do_wave: + jobs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + if self.do_gempak: + jobs += ['wavegempak'] + if self.do_awips: + jobs += ['waveawipsbulls', 'waveawipsgridded'] + + if self.do_wafs: + jobs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] + + return jobs + + @staticmethod + def _cycled_upd_base(base_in): + + return wfu.get_gfs_cyc_dates(base_in) + + @staticmethod + def _forecast_only_upd_base(base_in): + + base_out = base_in.copy() + base_out['INTERVAL'] = wfu.get_gfs_interval(base_in['gfs_cyc']) + + return base_out + + @property + def source_configs(self) -> Dict[str, Any]: + """ + Given the configuration object and jobs, + source the configurations for each job and return a dictionary + Every job depends on "config.base" + """ + + dict_jobs = dict() + + # Return config.base as well + dict_jobs['base'] = self.cfg.parse_config('config.base') + + # Source the list of input tasks + for job in self.job_names: + + # All tasks must source config.base first + files = ['config.base'] + + if job in ['eobs', 'eomg']: + files += ['config.anal', 'config.eobs'] + elif job in ['eupd']: + files += ['config.anal', 'config.eupd'] + elif job in ['efcs']: + files += ['config.fcst', 'config.efcs'] + elif 'wave' in job: + files += ['config.wave', f'config.{job}'] + else: + files += [f'config.{job}'] + + print(f'sourcing config.{job}') + dict_jobs[job] = self.cfg.parse_config(files) + + return dict_jobs + + def get_task_names(self): + + # Get a list of all possible tasks that would be part of the application + tasks_map = {'cycled': self._get_cycled_task_names, + 'forecast-only': self._get_forecast_only_task_names} + try: + self.task_names = tasks_map[self.mode]() + except KeyError: + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + + def _get_cycled_task_names(self): + + tasks = ['prep', 'anal', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + + gdas_only_tasks = ['analdiag'] + + gldas_tasks = ['gldas'] + wave_tasks = ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + + hybrid_tasks = ['eobs', 'eupd', 'echgres'] + hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] + + # First collect all gdas tasks + gdas_tasks = tasks + gdas_only_tasks + + if self.do_gldas: + gdas_tasks += gldas_tasks + + if self.do_wave: + gdas_tasks += wave_tasks + + if self.do_hybvar: + if 'gdas' in self.eupd_cdumps: + gdas_tasks += hybrid_tasks + gdas_tasks += hybrid_gdas_tasks + + # Now collect gfs tasks + gfs_tasks = tasks + + if self.do_hybvar: + if 'gfs' in self.eupd_cdumps: + gfs_tasks += hybrid_tasks + + if self.do_wave and 'gfs' in self.wave_cdumps: + gfs_tasks += wave_tasks + if self.do_gempak: + gfs_tasks += ['wavegempak'] + if self.do_awips: + gfs_tasks += ['waveawipsbulls', 'waveawipsgridded'] + + if self.do_bufrsnd: + gfs_tasks += ['postsnd'] + + if self.do_gempak: + gfs_tasks += ['gempak'] + + if self.do_awips: + gfs_tasks += ['awips'] + + if self.do_wafs: + gfs_tasks += ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25', 'wafsblending', 'wafsblending0p25'] + + if self.do_metp: + gfs_tasks += ['metp'] + + tasks = {'gdas': gdas_tasks, 'gfs': gfs_tasks} + + return tasks + + def _get_forecast_only_task_names(self): + + tasks = ['fcst', 'post', 'vrfy', 'arch'] + + if 'S2S' in self.model_app: + tasks += ['coupled_ic', 'ocnpost'] + else: + if self.do_hpssarch: + tasks += ['getic'] + tasks += ['init'] + + if 'AERO' in self.model_app: + tasks += ['aerosol_init'] + + if self.do_wave: + tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + if self.do_gempak: + tasks += ['wavegempak'] + if self.do_awips: + tasks += ['waveawipsbulls', 'waveawipsgridded'] + + if self.do_bufrsnd: + tasks += ['postsnd'] + + if self.do_gempak: + tasks += ['gempak'] + + if self.do_awips: + tasks += ['awips'] + + if self.do_wafs: + tasks += ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25', 'wafsblending', 'wafsblending0p25'] + + if self.do_metp: + tasks += ['metp'] + + return {'cdump': tasks} + + def get_resources(self): + + self.task_resources = dict() + for cdump, cdump_tasks in self.task_names.items(): + self.task_resources[cdump] = dict() + for task in cdump_tasks: + self.task_resources[cdump][task] = wfu.get_resource(self.job_configs[task], task, cdump=cdump) diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 1301955e88d..45cc0d69998 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -32,11 +32,10 @@ def create_metatask(task_dict: Dict[str, Any], metatask_dict: Dict[str, Any]) -> varval = metatask_dict.get('varval', 1) vardict = metatask_dict.get('vardict', None) - strings = [] + strings = [f'\n', + '\n', + f'\t{str(varval)}\n'] - strings.append(f'\n') - strings.append('\n') - strings.append(f'\t{str(varval)}\n') if vardict is not None: for key in vardict.keys(): value = str(vardict[key]) @@ -60,7 +59,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: :rtype: list """ - # Grab task info from the task_dict + # Grab task info from the task_names taskname = task_dict.get('taskname', 'demotask') cycledef = task_dict.get('cycledef', 'democycle') maxtries = task_dict.get('maxtries', 3) @@ -82,15 +81,14 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: str_final = ' final="true"' if final else '' envar = envar if isinstance(envar, list) else [envar] - strings = [] + strings = [f'\n', + '\n', + f'\t{command}\n', + '\n', + f'\t{jobname}\n', + f'\t{account}\n', + f'\t{queue}\n'] - strings.append(f'\n') - strings.append('\n') - strings.append(f'\t{command}\n') - strings.append('\n') - strings.append(f'\t{jobname}\n') - strings.append(f'\t{account}\n') - strings.append(f'\t{queue}\n') if partition is not None: strings.append(f'\t{partition}\n') if resources is not None: @@ -196,22 +194,29 @@ def _add_data_tag(dep_dict: Dict[str, Any]) -> str: msg = f'a data value is necessary for {dep_type} dependency' raise KeyError(msg) - if dep_offset is None: - if '@' in dep_data: - offset_string_b = '' + if not isinstance(dep_data, list): + dep_data = [dep_data] + + if not isinstance(dep_offset, list): + dep_offset = [dep_offset] + + assert len(dep_data) == len(dep_offset) + + strings = [''] + for data, offset in zip(dep_data, dep_offset): + if '@' in data: + offset_str = '' if offset in [None, ''] else f' offset="{offset}"' + offset_string_b = f'' offset_string_e = '' else: offset_string_b = '' offset_string_e = '' - else: - offset_string_b = f'' - offset_string_e = '' - string = '' - string += f'{offset_string_b}{dep_data}{offset_string_e}' - string += '' + strings.append(f'{offset_string_b}{data}{offset_string_e}') - return string + strings.append('') + + return ''.join(strings) def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: @@ -342,12 +347,9 @@ def create_cycledef(group=None, start=None, stop=None, step=None): :type group: str :param start: cycle start datetime :type start: str - :param end: cycle end datetime + :param step: cycle interval (timedelta) :type stop: str :param step: cycle interval (timedelta) - :type interval: str - :param value: value of the environment variable - :type value: str or float or int or unicode :return: Rocoto cycledef variable string :rtype: str """ diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 4fd4d4dd6c0..4c1971ed12f 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -68,8 +68,7 @@ def main(): dict_configs = wfu.source_configs(configs, steps) # Check and set gfs_cyc specific variables - if dict_configs['base']['gfs_cyc'] != 0: - dict_configs['base'] = get_gfs_cyc_dates(dict_configs['base']) + dict_configs['base'] = wfu.get_gfs_cyc_dates(dict_configs['base']) # First create workflow XML create_xml(dict_configs) @@ -80,151 +79,6 @@ def main(): return -def get_gfs_cyc_dates(base): - ''' - Generate GFS dates from experiment dates and gfs_cyc choice - ''' - - base_out = base.copy() - - gfs_cyc = base['gfs_cyc'] - sdate = base['SDATE'] - edate = base['EDATE'] - - interval_gfs = wfu.get_gfs_interval(gfs_cyc) - - # Set GFS cycling dates - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - fhmax_gfs = {} - for hh in ['00', '06', '12', '18']: - fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', 'FHMAX_GFS_00') - base_out['FHMAX_GFS'] = fhmax_gfs - - return base_out - - -def get_preamble(): - ''' - Generate preamble for XML - ''' - - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' - - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - hpssarch = base.get('HPSSARCH', 'NO').upper() - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - - if base['gfs_cyc'] != 0: - strings.append(get_gfs_dates(base)) - strings.append('\n') - - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if scheduler in ['slurm']: - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_gfs_dates(base): - ''' - Generate GFS dates entities - ''' - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - - return ''.join(strings) - - def get_gdasgfs_resources(dict_configs, cdump='gdas'): ''' Create GDAS or GFS resource entities @@ -381,766 +235,6 @@ def get_hyb_resources(dict_configs): return dict_resources - -def get_gdasgfs_tasks(dict_configs, cdump='gdas'): - ''' - Create GDAS or GFS tasks - ''' - - envars = [] - if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: - envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - base = dict_configs['base'] - gfs_cyc = base.get('gfs_cyc', 0) - gldas_cyc = base.get('gldas_cyc', 0) - dohybvar = base.get('DOHYBVAR', 'NO').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_wafs = base.get('WAFSF', 'NO').upper() - do_metp = base.get('DO_METP', 'NO').upper() - do_gldas = base.get('DO_GLDAS', 'NO').upper() - do_wave = base.get('DO_WAVE', 'NO').upper() - if do_wave in ['YES']: - do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() - do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - dumpsuffix = base.get('DUMP_SUFFIX', '') - gridsuffix = base.get('SUFFIX', '') - - dict_tasks = OrderedDict() - - # prep - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{gridsuffix}' - dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&DMPDIR;/{cdump}{dumpsuffix}.@Y@m@d/@H/{cdump}.t@Hz.updated.status.tm00.bufr_d' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False - - if gfs_enkf and cdump in ['gfs']: - if gfs_cyc == 4: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') - - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}prep'] = task - - # wave tasks in gdas or gfs or both - if do_wave_cdump in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif do_wave_cdump in ['GFS']: - cdumps = ['gfs'] - elif do_wave_cdump in ['GDAS']: - cdumps = ['gdas'] - - # waveinit - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks['{cdump}waveinit'] = task - - # waveprep - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}waveinit'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks['{cdump}waveprep'] = task - - # anal - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - else: - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}anal'] = task - - # analcalc - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': - dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - else: - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}analcalc'] = task - - # analdiag - if cdump in ['gdas']: - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - task = wfu.create_wf_task('analdiag', cdump=cdump, envar=envars, dependency=dependencies2) - - dict_tasks[f'{cdump}analdiag'] = task - - # gldas - if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - task = wfu.create_wf_task('gldas', cdump=cdump, envar=envars, dependency=dependencies2) - dict_tasks[f'{cdump}gldas'] = task - - # fcst - deps1 = [] - #data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - #dep_dict = {'type': 'data', 'data': data} - #deps1.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps1.append(rocoto.add_dependency(dep_dict)) - if do_gldas in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}gldas'} - deps1.append(rocoto.add_dependency(dep_dict)) - else: - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - elif cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies2) - else: - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies1) - - dict_tasks[f'{cdump}fcst'] = task - - # post - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}post'] = task - - # wavepostsbs - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostsbs'] = task - - # wavepostbndpnt - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostbndpnt'] = task - - # wavepostbndpntbll - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostbndpntbll'] = task - - # wavepostpnt - if do_wave in ['Y', 'YES'] and cdump in ['gdas', 'gfs']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES'] and cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostpnt'] = task - - # wavegempak - if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavegempak'] = task - - # waveawipsgridded - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}waveawipsgridded'] = task - - # waveawipsbulls - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}waveawipsbulls'] = task - - # wavestat - #if do_wave in ['Y', 'YES'] and cdump in cdumps: - # deps = [] - # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} - # deps.append(rocoto.add_dependency(dep_dict)) - # dependencies = rocoto.create_dependency(dep=deps) - # task = wfu.create_wf_task('wavestat', cdump=cdump, envars=envars, dependency=dependencies) - # dict_tasks['%swavestat' % cdump] = task - - # vrfy - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}vrfy'] = task - - # metp - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE_GFS;') - metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') - metpenvars = envars + [sdate_gfs] + [metpcase] - varname1 = 'metpcase' - varval1 = 'g2g1 g2o1 pcp1' - task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) - dict_tasks[f'{cdump}metp'] = task - - #postsnd - if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}postsnd'] = task - - # awips - if cdump in ['gfs'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}awips'] = task - - # gempak - if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}gempak'] = task - - # wafs - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafs'] = task - - # wafsgcip - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgcip'] = task - - # wafsgrib2 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgrib2'] = task - - # wafsgrib20p25 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgrib20p25'] = task - - # wafsblending - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsblending'] = task - - # wafsblending0p25 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsblending0p25'] = task - - # arch - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES'] and cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}arch'] = task - - return dict_tasks - - -def get_hyb_tasks(dict_configs, cycledef='enkf'): - ''' - Create Hybrid tasks - ''' - - # Determine groups based on ensemble size and grouping - base = dict_configs['base'] - nens = base['NMEM_ENKF'] - lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - - eobs = dict_configs['eobs'] - nens_eomg = eobs['NMEM_EOMGGRP'] - neomg_grps = nens / nens_eomg - EOMGGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(neomg_grps) + 1)]) - - efcs = dict_configs['efcs'] - nens_efcs = efcs['NMEM_EFCSGRP'] - nefcs_grps = nens / nens_efcs - EFCSGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(nefcs_grps) + 1)]) - - earc = dict_configs['earc'] - nens_earc = earc['NMEM_EARCGRP'] - nearc_grps = nens / nens_earc - EARCGROUPS = ' '.join([f'{x:02d}' for x in range(0, int(nearc_grps) + 1)]) - - envars = [] - if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: - envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - #envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#') - - dict_tasks = OrderedDict() - - if eupd_cyc in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif eupd_cyc in ['GFS']: - cdumps = ['gfs'] - elif eupd_cyc in ['GDAS']: - cdumps = ['gdas'] - - for cdump in cdumps: - - envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') - envars1 = envars + [envar_cdump] - - # eobs - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}eobs'] = task - - # eomn, eomg - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - eomgenvars= envars1 + [ensgrp] - task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}eomn'] = task - - # ediag - else: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('ediag', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}ediag'] = task - - # eupd - deps = [] - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}eomn'} - else: - dep_dict = {'type': 'task', 'name': f'{cdump}ediag'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}eupd'] = task - - # All hybrid tasks beyond this point are always executed in the GDAS cycle - cdump = 'gdas' - envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') - envars1 = envars + [envar_cdump] - cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas' - - # ecmn, ecen - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ecenenvars = envars1 + [fhrgrp] + [fhrlst] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_ecengroups(dict_configs, dict_configs['ecen'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('ecen', cdump=cdump, envar=ecenenvars, dependency=dependencies2, - metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}ecmn'] = task - - # esfc - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - task = wfu.create_wf_task('esfc', cdump=cdump, envar=envars1, dependency=dependencies2, cycledef=cycledef) - - dict_tasks[f'{cdump}esfc'] = task - - # efmn, efcs - deps1 = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}ecmn'} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}esfc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='or', dep=deps2) - - efcsenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies2, - metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}efmn'] = task - - # echgres - deps1 = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - task = wfu.create_wf_task('echgres', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) - - dict_tasks[f'{cdump}echgres'] = task - - # epmn, epos - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - eposenvars = envars1 + [fhrgrp] + [fhrlst] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}epmn'] = task - - # eamn, earc - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}epmn'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - earcenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}eamn'] = task - - return dict_tasks - - def get_workflow_header(base): ''' Create the workflow header block @@ -1167,132 +261,6 @@ def get_workflow_header(base): return ''.join(strings) -def get_workflow_footer(): - ''' - Generate workflow footer - ''' - - strings = [] - strings.append('\n\n') - - return ''.join(strings) - - -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'],post['FHMAX_GFS_06'],post['FHMAX_GFS_12'],post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout)) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['anl'] + [f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join(['anl'] + [f[-1] for f in fhrs]) - fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_awipsgroups(awips, cdump='gdas'): - - fhmin = awips['FHMIN'] - fhmax = awips['FHMAX'] - fhout = awips['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([awips['FHMAX_GFS_00'],awips['FHMAX_GFS_06'],awips['FHMAX_GFS_12'],awips['FHMAX_GFS_18']]) - fhout = awips['FHOUT_GFS'] - fhmax_hf = awips['FHMAX_HF_GFS'] - fhout_hf = awips['FHOUT_HF_GFS'] - if fhmax > 240: - fhmax = 240 - if fhmax_hf > 240: - fhmax_hf = 240 - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) - - nawipsgrp = awips['NAWIPSGRP'] - ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_ecengroups(dict_configs, ecen, cdump='gdas'): - - base = dict_configs['base'] - - if base.get('DOIAU_ENKF', 'NO') == 'YES' : - fhrs = list(base.get('IAUFHRS','6').split(',')) - ifhrs = [f'f00{ff}' for ff in fhrs] - ifhrs0 = ifhrs[0] - nfhrs = len(fhrs) - - ifhrs = [f'f00{ff}' for ff in fhrs] - ifhrs0 = ifhrs[0] - nfhrs = len(fhrs) - - necengrp = ecen['NECENGRP'] - ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) - - ifhrs = np.array_split(ifhrs, ngrps) - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in ifhrs]) - fhrlst = ' '.join(['_'.join(f) for f in ifhrs]) - - else: - fhrgrp='000' - fhrdep='f006' - fhrlst='f006' - - return fhrgrp, fhrdep, fhrlst - -def get_eposgroups(epos, cdump='gdas'): - - fhmin = epos['FHMIN_ENKF'] - fhmax = epos['FHMAX_ENKF'] - fhout = epos['FHOUT_ENKF'] - fhrs = range(fhmin, fhmax+fhout, fhout) - - neposgrp = epos['NEPOSGRP'] - ngrps = neposgrp if len(fhrs) > neposgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - def dict_to_strings(dict_in): strings = [] diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 70e3e9f0894..e2671348be1 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -1,12 +1,8 @@ #!/usr/bin/env python3 -''' +""" PROGRAM: - Create the ROCOTO workflow for a forecast only experiment given the configuration of the GFS parallel - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov + Create the ROCOTO workflow for any experiment given the configuration of the GFS parallel FILE DEPENDENCIES: 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. @@ -16,38 +12,62 @@ 1. PSLOT.xml: XML workflow 2. PSLOT.crontab: crontab for ROCOTO run command -''' +""" import os import sys import re -from typing import List, Dict, Any -import numpy as np -from datetime import datetime from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import rocoto from hosts import Host import workflow_utils as wfu from configuration import Configuration -taskplan = ['getic', 'init', 'coupled_ic', 'aerosol_init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', - 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', - 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', - 'awips', 'vrfy', 'metp', 'arch', 'ocnpost'] +@property +def input_args(): + """ + Method to collect user arguments for `setup_workflow.py` + """ -def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', + here = os.path.dirname(__file__) + top = os.path.abspath(os.path.join( + os.path.abspath(here), '../..')) + + description = """ + Sources configuration files based on application and\n + creates "PSLOT.xml" for use with Rocoto.\n + """ + + parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - parser.add_argument('--self', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', - required=False) + + # Set up sub-parsers for various modes of experimentation + subparser = parser.add_subparsers(dest='mode') + cycled = subparser.add_parser( + 'cycled', help='arguments for cycled mode') + forecasts = subparser.add_parser( + 'forecast-only', help='arguments for forecast-only mode') + + forecasts.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', + required=False) args = parser.parse_args() - cfg = Configuration(args.expdir) + return args + + +def main(): + + user_inputs = input_args + + cfg = Configuration(user_inputs.expdir) + + + sys.exit(0) _base = cfg.parse_config('config.base') @@ -70,7 +90,7 @@ def main(): return -def get_resources(dict_configs, task_plan, self='gdas'): +def get_resources(dict_configs, task_plan, cdump='gdas'): ''' Create resource entities ''' @@ -107,644 +127,7 @@ def get_resources(dict_configs, task_plan, self='gdas'): return ''.join(strings) -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = list(range(fhmin, fhmax + fhout, fhout)) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'], post['FHMAX_GFS_06'], post['FHMAX_GFS_12'], post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) - fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - -def get_workflow(dict_configs, self='gdas'): - ''' - Create tasks for forecast only workflow - ''' - - envars = [] - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - hpssarch = base.get('HPSSARCH', 'NO').upper() - app = base.get('APP', "ATM").upper() - do_wave = base.get('DO_WAVE', 'NO').upper() - do_ocean = base.get('DO_OCN', 'NO').upper() - do_ice = base.get('DO_ICE', 'NO').upper() - do_aero = base.get('DO_AERO', 'NO').upper() - do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - if do_wave in ['YES']: - do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_wafs = base.get('WAFSF', 'NO').upper() - do_vrfy = base.get('DO_VRFY', 'YES').upper() - do_metp = base.get('DO_METP', 'NO').upper() - n_tiles = 6 - - tasks = [] - - if app in ['S2S', 'S2SW']: - # Copy prototype ICs - deps = [] - base_cplic = dict_configs['coupled_ic']['BASE_CPLIC'] - - # ATM ICs - for file in ['gfs_ctrl.nc'] + [f'{datatype}_data.tile{tile_index}.nc' for datatype in ['gfs', 'sfc'] for - tile_index in range(1, n_tiles + 1)]: - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ATMIC']}/@Y@m@d@H/&CDUMP;/{base.get('CASE', 'C384')}/INPUT/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if do_ocean in ["YES"]: - ocn_res = base.get('OCNRES', '025') - for res in ['res'] + [f'res_{res_index}' for res_index in range(1, 5)]: - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_OCNIC']}/@Y@m@d@H/ocn/{ocn_res:03d}/MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if do_ice in ["YES"]: - ice_res = base.get('ICERES', '025') - ice_res_dec = f'{float(ice_res) / 100:.2f}' - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ICEIC']}/@Y@m@d@H/ice/{ice_res:03d}/cice5_model_{ice_res_dec}.res_@Y@m@d@H.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if do_wave in ["YES"]: - for wave_grid in dict_configs['waveinit']['waveGRD'].split(): - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_WAVIC']}/@Y@m@d@H/wav/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('coupled_ic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - else: - if hpssarch in ['YES']: - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) - - task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # init - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - if hpssarch in ['YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}getic'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep=deps) - - deps = [] - deps.append(dependencies) - if hpssarch in ['YES']: - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveinit - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars) - tasks.append(task) - tasks.append('\n') - - # waveprep - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}waveinit'} - deps.append(rocoto.add_dependency(dep_dict)) - if app not in ['S2S', 'S2SW']: - dep_dict = {'type': 'task', 'name': f'{cdump}init'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # aerosol_init - if do_aero in ['Y', 'YES']: - deps = [] - if app in ['S2S', 'S2SW']: - dep_dict = {'type': 'task', 'name': 'coupled_ic'} - else: - dep_dict = {'type': 'task', 'name': f'{cdump}init'} - - deps.append(rocoto.add_dependency(dep_dict)) - - # Files from current cycle - files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] - for file in files: - data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/{file}' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Files from previous cycle - dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} - deps.append(rocoto.add_dependency(dep_dict)) - - files = [f'@Y@m@d.@H0000.fv_core.res.nc'] + \ - [f'@Y@m@d.@H0000.fv_core.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] + \ - [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] - - data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RERUN_RESTART/' - dep_dict = {'type': 'data', 'data': data, 'offset': f'-{base["INTERVAL"]}'} - # Hack off the trailing tag because we are going to concatenate with the rest - dependency1 = rocoto.add_dependency(dep_dict)[:-10] - for file in files: - dep_dict = {'type': 'data', 'data': file} - # Hack off the leading tag to join with the earlier one - dependency2 = rocoto.add_dependency(dep_dict)[9:] - # Combine the two into a dependency with two different cyclestr tags - deps.append(dependency1 + dependency2) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('aerosol_init', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # fcst - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps) - - if do_aero in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}aerosol_init'} - deps.append(rocoto.add_dependency(dep_dict)) - deps2 = [] - dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} - deps2.append(rocoto.add_dependency(dep_dict)) - deps.append(rocoto.create_dependency(dep_condition='not', dep=deps2)) - dependencies3 = rocoto.create_dependency(dep_condition='or', dep=deps) - - deps = [] - deps.append(dependencies) - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps.append(dependencies2) - if do_aero in ['Y', 'YES']: - deps.append(dependencies3) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # post - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # wavepostsbs - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostbndpnt - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostbndpntbll - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostpnt - if do_wave in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavegempak - if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveawipsbulls - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveawipsgridded - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # ocnpost - if do_ocean in ['YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['ocnpost'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('ocnpost', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='ocnpost', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # wafs - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgcip - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgrib2 - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgrib20p25 - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsblending - if do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsblending0p25 - if do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # postsnd - if do_bufrsnd in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # awips - if do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # gempak - if do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # vrfy - if do_vrfy in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # metp - if do_metp in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;') - metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') - metpenvars = envars + [sdate_gfs] + [metpcase] - varname1 = 'metpcase' - varval1 = 'g2g1 g2o1 pcp1' - task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) - tasks.append(task) - tasks.append('\n') - - # arch - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_vrfy in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}metp'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': f'{hpssarch}'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_ocean in ['Y', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}ocnpost'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) - tasks.append(task) - tasks.append('\n') - - return ''.join(tasks) - - -def get_workflow_body(dict_configs, self='gdas'): +def get_workflow_body(dict_configs, cdump='gdas'): ''' Create the workflow body ''' diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index f16e704c09a..c726ca0a8e1 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -1,41 +1,26 @@ #!/usr/bin/env python import numpy as np -from typing import Dict, Any import rocoto import workflow_utils as wfu +from applications import Application class Tasks: - def __init__(self, dict_configs: Dict[str, Dict[str, Any]], cdump: str) -> None: + def __init__(self, app: Application, cdump: str) -> None: self.cdump = cdump + self.app = app # Save dict_configs and base in the internal state (never know where it may be needed) - self._dict_configs = dict_configs - self._base = dict_configs['base'] - - self.do_hybvar = self._base.get('DOHYBVAR', False) - self.do_wave = self._base.get('DO_WAVE', False) - self.do_ocean = self._base.get('DO_OCEAN', False) - self.do_ice = self._base.get('DO_ICE', False) - self.do_aero = self._base.get('DO_AERO', False) - self.do_gldas = self._base.get('DO_GLDAS', False) - self.do_bufrsnd = self._base.get('DO_BUFRSND', False) - self.do_gempak = self._base.get('DO_GEMPAK', False) - self.do_awips = self._base.get('DO_AWIPS', False) - self.do_wafs = self._base.get('DO_WAFS', False) - self.do_vrfy = self._base.get('DO_VRFY', True) - self.do_metp = self._base.get('DO_METP', False) - - if self.do_wave: - self.do_wave_bnd = self._dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', True) - wave_cdump = self._base.get('WAVE_CDUMP', 'BOTH').lower() - if wave_cdump in ['both']: - self.wave_cdumps = ['gfs', 'gdas'] - elif wave_cdump in ['gfs', 'gdas']: - self.wave_cdumps = [wave_cdump] + self._job_configs = self.app.job_configs + self._base = self._job_configs['base'] + + self.n_tiles = 6 # TODO - this needs to be elsewhere + + if self.app.do_hybvar: + self.hyb_cycledef = 'enkf' # TODO - sort this out a bit more envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'), 'HOMEgfs': self._base.get('HOMEgfs'), @@ -46,16 +31,6 @@ def __init__(self, dict_configs: Dict[str, Dict[str, Any]], cdump: str) -> None: 'cyc': '@H'} self.envars = self._set_envars(envar_dict) - if self.do_hybvar: - self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) - eupd_cdump = self._base.get('EUPD_CYC', 'gdas').lower() - if eupd_cdump in ['both']: - self.eupd_cdumps = ['gfs', 'gdas'] - elif eupd_cdump in ['gfs', 'gdas']: - self.eupd_cdumps = [eupd_cdump] - - self.hyb_cycledef = 'enkf' # TODO - sort this out a bit more - @staticmethod def _set_envars(envar_dict) -> list: @@ -70,27 +45,174 @@ def _get_hybgroups(nens: int, nmem_per_group: int): ngrps = nens / nmem_per_group groups = ' '.join([f'{x:02d}' for x in range(1, int(ngrps) + 1)]) return groups - + + def coupled_ic(self): + + cpl_ic = self._job_configs['coupled_ic'] + + deps = [] + + # Atm ICs + atm_res = self._base.get('CASE', 'C384') + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/&CDUMP;" + for file in ['gfs_ctrl.nc'] + \ + [f'{datatype}_data.tile{tile}.nc' + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: + data = f"{prefix}/{atm_res}/INPUT/{file}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Ocean ICs + if self.app.do_ocean: + ocn_res = f"{self._base.get('OCNRES', '025'):03d}" + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/ocn" + for res in ['res'] + [f'res_{res_index}' for res_index in range(1, 5)]: + data = f"{prefix}/{ocn_res}/MOM.{res}.nc" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Ice ICs + if self.app.do_ice: + ice_res = f"{self._base.get('ICERES', '025'):03d}" + ice_res_dec = f'{float(ice_res) / 100:.2f}' + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/ice" + data = f"{prefix}/{ice_res}/cice5_model_{ice_res_dec}.res_@Y@m@d@H.nc" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Wave ICs + if self.app.do_wave: + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/wav" + for wave_grid in self._job_configs['waveinit']['waveGRD'].split(): + data = f"{prefix}/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('coupled_ic', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def getic(self): + + files = ['INPUT/sfc_data.tile6.nc', + 'RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'] + + deps = [] + for file in files: + dep_dict = {'type': 'data', 'data': f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/{file}'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) + + task = wfu.create_wf_task('getic', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def init(self): + + files = ['gfs.t@Hz.sanl', + 'gfs.t@Hz.atmanl.nemsio', + 'gfs.t@Hz.atmanl.nc', + 'atmos/gfs.t@Hz.atmanl.nc', + 'atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'] + + deps = [] + for file in files: + dep_dict = {'type': 'data', 'data': f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/{file}'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + if self.app.do_hpssarch: + dep_dict = {'type': 'task', 'name': f'{self.cdump}getic'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + + task = wfu.create_wf_task('init', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def prep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + gfs_cyc = self._base["gfs_cyc"] + gfs_enkf = True if self.app.do_hybvar and 'gfs' in self.app.eupd_cdumps else False + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&DMPDIR;/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + cycledef = self.cdump + if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + + task = wfu.create_wf_task('prep', cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + + return task + def waveinit(self): + task = wfu.create_wf_task('waveinit', cdump=self.cdump, envar=self.envars) + return task def waveprep(self): + deps = [] dep_dict = {'type': 'task', 'name': f'{self}waveinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('waveprep', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task - def aerosol_init(self): # TODO - raise NotImplementedError('aerosol_init has not been implemented yet!') + def aerosol_init(self): + + deps = [] + # Files from current cycle + files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + for file in files: + data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/{file}' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Previous cycle + dep_dict = {'type': 'cycleexist', 'offset': f'-{self._base["INTERVAL"]}'} + deps.append(rocoto.add_dependency(dep_dict)) + + # Files from previous cycle + files = [f'@Y@m@d.@H0000.fv_core.res.nc'] + \ + [f'@Y@m@d.@H0000.fv_core.res.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + \ + [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + + for file in files: + data = ['&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RERUN_RESTART/', file] + offset = [f'-{self._base["INTERVAL"]}', None] + dep_dict = {'type': 'data', 'data': data, 'offset': offset} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('aerosol_init', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task def anal(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_hybvar: + if self.app.do_hybvar: dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -100,30 +222,39 @@ def anal(self): return task def analcalc(self): + deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_hybvar and self.cdump in ['gdas']: + if self.app.do_hybvar and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('analcalc', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task def analdiag(self): + deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('analdiag', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task def gldas(self): + deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gldas', cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task def fcst(self, fcst_type): @@ -151,11 +282,11 @@ def _fcst_free(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - if self.do_wave and self.cdump in self.wave_cdumps: + if self.app.do_wave and self.cdump in self.app.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.do_aero: # TODO - check dependency on cycleexist of previous cycle? + if self.app.do_aero: # TODO - check dependency on cycleexist of previous cycle? dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -172,11 +303,11 @@ def _fcst_cycled(self): dep = rocoto.add_dependency(dep_dict) dependencies = rocoto.create_dependency(dep=dep) - if self.do_gldas and self.cdump in ['gdas']: + if self.app.do_gldas and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.cdump}gldas'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.do_wave and self.cdump in self.wave_cdumps: + if self.app.do_wave and self.cdump in self.app.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -247,7 +378,7 @@ def _get_postgroups(cdump, config, add_anl=False): postenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_postgroups(self.cdump, self._dict_configs[task_name], add_anl=add_anl_to_post) + varval1, varval2, varval3 = _get_postgroups(self.cdump, self._job_configs[task_name], add_anl=add_anl_to_post) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task(task_name, cdump=self.cdump, envar=postenvars, dependency=dependencies, @@ -257,7 +388,7 @@ def _get_postgroups(cdump, config, add_anl=False): def wavepostsbs(self): deps = [] - for wave_grid in self._dict_configs['wavepostsbs']['waveGRD'].split(): + for wave_grid in self._job_configs['wavepostsbs']['waveGRD'].split(): data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/wave/rundata/{self.cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -292,7 +423,7 @@ def wavepostpnt(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_wave_bnd: + if self.app.do_wave_bnd: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpntbll'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -439,7 +570,7 @@ def _get_awipsgroups(cdump, config): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._dict_configs['awips']) + varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._job_configs['awips']) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('awips', cdump=self.cdump, envar=awipsenvars, dependency=dependencies, @@ -473,7 +604,8 @@ def metp(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) metpenvars = self.envars - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS'), # TODO - in Forecast-only, this is `SDATE` on the RHS + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS'), + # TODO - in Forecast-only, this is `SDATE` on the RHS 'METPCASE': '#metpcase#'} for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -488,18 +620,18 @@ def metp(self): def arch(self): deps = [] - if self.do_vrfy: + if self.app.do_vrfy: dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_metp and self.cdump in ['gfs']: + if self.app.do_metp and self.cdump in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_wave: + if self.app.do_wave: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_wave_bnd and self.cdump in ['gfs']: + if self.app.do_wave_bnd and self.cdump in ['gfs']: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -530,7 +662,7 @@ def eomg(self): eomgenvars = self.envars eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['eobs']['NMEM_EOMGGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['eobs']['NMEM_EOMGGRP']) task = wfu.create_wf_task('eomg', cdump=self.cdump, envar=eomgenvars, dependency=dependencies, metatask='eomn', varname='grp', varval=groups, @@ -551,7 +683,7 @@ def ediag(self): def eupd(self): deps = [] - if self.lobsdiag_forenkf: + if self.app.lobsdiag_forenkf: dep_dict = {'type': 'task', 'name': f'{self.cdump}ediag'} else: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}eomn'} @@ -577,7 +709,7 @@ def _get_ecengroups(): if self._base.get('DOIAU_ENKF', False): fhrs = list(self._base.get('IAUFHRS', '6').split(',')) - necengrp = self._dict_configs['ecen']['NECENGRP'] + necengrp = self._job_configs['ecen']['NECENGRP'] ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) fhrs = [f'{fhr:03d}' for fhr in fhrs] @@ -595,7 +727,7 @@ def _get_ecengroups(): return grp, dep, lst - eupd_cdump = 'gdas' if 'gdas' in self.eupd_cdumps else 'gfs' + eupd_cdump = 'gdas' if 'gdas' in self.app.eupd_cdumps else 'gfs' deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} @@ -623,7 +755,7 @@ def esfc(self): self.is_this_a_gdas_task(self.cdump, 'esfc') - eupd_cdump = 'gdas' if 'gdas' in self.eupd_cdumps else 'gfs' + eupd_cdump = 'gdas' if 'gdas' in self.app.eupd_cdumps else 'gfs' deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} @@ -651,7 +783,7 @@ def efcs(self): efcsenvars = self.envars efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['efcs']['NMEM_EFCSGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['efcs']['NMEM_EFCSGRP']) task = wfu.create_wf_task('efcs', cdump=self.cdump, envar=efcsenvars, dependency=dependencies, metatask='efmn', varname='grp', varval=groups, @@ -710,7 +842,7 @@ def _get_eposgroups(epos): eposenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_eposgroups(self._dict_configs['epos']) + varval1, varval2, varval3 = _get_eposgroups(self._job_configs['epos']) vardict = {varname2: varval2, varname3: varval3} task = wfu.create_wf_task('epos', cdump=self.cdump, envar=eposenvars, dependency=dependencies, @@ -731,7 +863,7 @@ def earc(self): earcenvars = self.envars earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._dict_configs['efcs']['NMEM_EARCGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['efcs']['NMEM_EARCGRP']) task = wfu.create_wf_task('earc', cdump=self.cdump, envar=earcenvars, dependency=dependencies, metatask='eamn', varname='grp', varval=groups, diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index 17c94899022..a8c7d42c6d4 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -6,62 +6,23 @@ import os import numpy as np from distutils.spawn import find_executable -from typing import List, Dict, Any +from datetime import timedelta +from typing import Dict, Any from rocoto import create_task, create_metatask from hosts import Host -from configuration import Configuration -SERVICE_TASKS = ['arch', 'earc', 'getic'] - - -def source_configs(config: Configuration, tasks) -> dict: - """ - Given the configuration object and tasks, - return a dictionary for each task - Every task depends on "config.base" - """ - - dict_configs = dict() - - # Return config.base as well - dict_configs['base'] = config.parse_config('config.base') - - # Source the list of input tasks - for task in tasks: - - # All tasks must source config.base first - files = ['config.base'] - - if task in ['eobs', 'eomg']: - files.append('config.anal') - files.append('config.eobs') - elif task in ['eupd']: - files.append('config.anal') - files.append('config.eupd') - elif task in ['efcs']: - files.append('config.fcst') - files.append('config.efcs') - elif 'wave' in task: - files.append('config.wave') - files.append(f'config.{task}') - else: - files.append(f'config.{task}') - - print(f'sourcing config.{task}') - dict_configs[task] = config.parse_config(files) - - return dict_configs +#SERVICE_TASKS = ['arch', 'earc', 'getic'] def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, metatask=None, varname=None, varval=None, vardict=None, final=False): - if metatask is None: - taskstr = f'{task}' - else: - taskstr = f'{task}#{varname}#' - metataskstr = f'{cdump}{metatask}' - metatask_dict = {'metataskname': metataskstr, + + taskstr = f'{cdump}{task}' + metatask_dict = None + if metatask is not None: + taskstr = f'{taskstr}#{varname}#' + metatask_dict = {'metataskname': f'{cdump}{metatask}', 'varname': f'{varname}', 'varval': f'{varval}', 'vardict': vardict} @@ -98,6 +59,15 @@ def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=Non return task +def check_expdir(cmd_expdir, cfg_expdir): + + if not os.path.samefile(cmd_expdir, cfg_expdir): + print('MISMATCH in experiment directories!') + print(f'config.base: EXPDIR = {cfg_expdir}') + print(f'input arg: --expdir = {cmd_expdir}') + raise ValueError('Abort!') + + def get_gfs_interval(gfs_cyc: int) -> str: """ return interval in hours based on gfs_cyc @@ -111,14 +81,67 @@ def get_gfs_interval(gfs_cyc: int) -> str: raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') +def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate GFS dates from experiment dates and gfs_cyc choice + """ + + base_out = base.copy() + + gfs_cyc = base['gfs_cyc'] + sdate = base['SDATE'] + edate = base['EDATE'] + + interval_gfs = get_gfs_interval(gfs_cyc) + + # Set GFS cycling dates + hrinc = 0 + hrdet = 0 + if gfs_cyc == 0: + return base_out + elif gfs_cyc == 1: + hrinc = 24 - sdate.hour + hrdet = edate.hour + elif gfs_cyc == 2: + if sdate.hour in [0, 12]: + hrinc = 12 + elif sdate.hour in [6, 18]: + hrinc = 6 + if edate.hour in [6, 18]: + hrdet = 6 + elif gfs_cyc == 4: + hrinc = 6 + sdate_gfs = sdate + timedelta(hours=hrinc) + edate_gfs = edate - timedelta(hours=hrdet) + if sdate_gfs > edate: + print('W A R N I N G!') + print('Starting date for GFS cycles is after Ending date of experiment') + print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') + print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') + gfs_cyc = 0 + + base_out['gfs_cyc'] = gfs_cyc + base_out['SDATE_GFS'] = sdate_gfs + base_out['EDATE_GFS'] = edate_gfs + base_out['INTERVAL_GFS'] = interval_gfs + + fhmax_gfs = {} + for hh in ['00', '06', '12', '18']: + fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', base.get('FHMAX_GFS_00', 120)) + base_out['FHMAX_GFS'] = fhmax_gfs + + return base_out + + def get_resource(task_dict: dict, task_name: str, cdump: str = 'gdas') -> dict: """ - Given a task name (task_name) and its configuration (task_dict), + Given a task name (task_name) and its configuration (task_names), return a dictionary of resources (task_resource) used by the task. Task resource dictionary includes: account, walltime, cores, nodes, ppn, threads, memory, queue, partition, native """ + SERVICE_TASKS = ['arch', 'earc', 'getic'] scheduler = Host.get_scheduler account = task_dict['ACCOUNT'] @@ -153,36 +176,23 @@ def get_resource(task_dict: dict, task_name: str, cdump: str = 'gdas') -> dict: partition = None if scheduler in ['slurm']: - partition = task_dict['PARTITION_SERVICE'] if task_name in SERVICE_TASKS else task_dict['PARTITION_BATCH'] - - task_resource = dict() - task_resource['account'] = account - task_resource['walltime'] = walltime - task_resource['nodes'] = nodes - task_resource['cores'] = cores - task_resource['ppn'] = ppn - task_resource['threads'] = threads - task_resource['compute'] = compute # TODO - remove in the future - task_resource['memory'] = memory - task_resource['native'] = native - task_resource['queue'] = queue - task_resource['partition'] = partition + partition = task_dict['QUEUE_SERVICE'] if task_name in SERVICE_TASKS else task_dict['PARTITION_BATCH'] + + task_resource = {'account': account, + 'walltime': walltime, + 'nodes': nodes, + 'cores': cores, + 'ppn': ppn, + 'threads': threads, + 'compute': compute, + 'memory': memory, + 'native': native, + 'queue': queue, + 'partition': partition} return task_resource -def get_taskplan_resources(dict_configs: List[Dict[str, Any]], task_plan: List[str], cdump='gdas') -> Dict[str, Any]: - """ - Given a list of dictionary containing tasks, and a task plan, - return the resource dictionary for the tasks in the task plan - """ - resources = dict() - for task in task_plan: - task_name = cdump + task - resources[task_name] = get_resource(dict_configs[task], task, cdump=cdump) - return resources - - def create_crontab(base, cronint=5): """ Create crontab to execute rocotorun every cronint (5) minutes diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index 6b989486ed5..807236d6400 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -11,18 +11,16 @@ def get_preamble(): Generate preamble for XML """ - strings = [] - - strings.append('') - strings.append('') + strings = ['', + ''] return '\n'.join(strings) @@ -80,3 +78,62 @@ def get_definitions(base: dict, mode: str) -> str: strings.append('\t' + rocoto.create_entity(key, value)) return '\n'.join(strings) + + +def get_workflow_header(): + """ + Create the workflow header block + """ + + strings = ['', + ']>', + '', + '', + '', + '\t&EXPDIR;/logs/@Y@m@d@H.log', + '', + '\t'] + + return '\n'.join(strings) + + +def get_cycledefs(exp_type, **kwargs): + + cycledef_map = {'cycled': _get_cycledefs_cycled, + 'forecast-only': _get_cycledefs_forecast_only} + + try: + cycledef_map[exp_type](**kwargs) + except KeyError: + raise TypeError(f'{exp_type} is not a valid application.' + + 'Current plot types supported are:\n' + + f'{" | ".join(cycledef_map.keys())}"') + + +def _get_cycledefs_cycled(**kwargs): + strings = ['\t&SDATE; &SDATE; 06:00:00', + '\t&SDATE; &EDATE; 06:00:00', + '\t&SDATE; &EDATE; 06:00:00'] + if base['gfs_cyc'] != 0: + strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;') + + return '\n'.join(strings) + + +def _get_cycledefs_forecast_only(**kwqrgs): + + strings = f'\t&SDATE; &EDATE; &INTERVAL;' + + return strings + + +def get_workflow_footer(): + """ + Generate workflow footer + """ + + return '\n\n' + + +def create_xml(): + return None From 375c8cc542a569094d5e929dd0069a5cdbeec252 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 14 Jun 2022 00:01:58 -0400 Subject: [PATCH 04/41] create tasks with proper resources --- ush/rocoto/applications.py | 32 +-- ush/rocoto/rocoto.py | 25 ++- ush/rocoto/rocoto_viewer.py | 30 +-- ush/rocoto/setup_workflow.py | 16 +- ush/rocoto/setup_workflow_fcstonly.py | 2 +- ush/rocoto/workflow_tasks.py | 261 +++++++++++++++++++------ ush/rocoto/workflow_utils.py | 86 +------- ush/rocoto/workflow_xml.py | 270 ++++++++++++++------------ 8 files changed, 408 insertions(+), 314 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index cb9945ba78e..72cdf044e45 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -3,16 +3,16 @@ from typing import Dict, Any from configuration import Configuration import workflow_utils as wfu +#from workflow_tasks import Tasks class Application: VALID_MODES = ['cycled', 'forecast-only'] - SERVICE_TASKS = ['arch', 'earc', 'getic'] def __init__(self, mode: str, configuration: Configuration) -> None: - self.task_resources = None self.task_names = None + self.task_tasks = None if mode not in self.VALID_MODES: raise NotImplementedError(f'{mode} is not a valid application mode.\n' + 'Valid application modes are:\n' + @@ -50,7 +50,7 @@ def __init__(self, mode: str, configuration: Configuration) -> None: f'{", ".join(self.VALID_MODES)}') # Source the configs for all the jobs in the application - self.job_configs = self.source_configs + self.job_configs = self.source_configs() # Update the base config dictionary based on application upd_base_map = {'cycled': self._cycled_upd_base, @@ -63,6 +63,7 @@ def __init__(self, mode: str, configuration: Configuration) -> None: f'{", ".join(self.VALID_MODES)}') self._base = self.job_configs['base'] + self.gfs_cyc = self._base.get('gfs_cyc') if self.do_hybvar: self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) @@ -169,7 +170,6 @@ def _forecast_only_upd_base(base_in): return base_out - @property def source_configs(self) -> Dict[str, Any]: """ Given the configuration object and jobs, @@ -182,10 +182,10 @@ def source_configs(self) -> Dict[str, Any]: # Return config.base as well dict_jobs['base'] = self.cfg.parse_config('config.base') - # Source the list of input tasks + # Source the list of input task_tasks for job in self.job_names: - # All tasks must source config.base first + # All task_tasks must source config.base first files = ['config.base'] if job in ['eobs', 'eomg']: @@ -206,7 +206,7 @@ def source_configs(self) -> Dict[str, Any]: def get_task_names(self): - # Get a list of all possible tasks that would be part of the application + # Get a list of all possible task_tasks that would be part of the application tasks_map = {'cycled': self._get_cycled_task_names, 'forecast-only': self._get_forecast_only_task_names} try: @@ -229,7 +229,7 @@ def _get_cycled_task_names(self): hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] - # First collect all gdas tasks + # First collect all gdas task_tasks gdas_tasks = tasks + gdas_only_tasks if self.do_gldas: @@ -243,7 +243,7 @@ def _get_cycled_task_names(self): gdas_tasks += hybrid_tasks gdas_tasks += hybrid_gdas_tasks - # Now collect gfs tasks + # Now collect gfs task_tasks gfs_tasks = tasks if self.do_hybvar: @@ -314,10 +314,12 @@ def _get_forecast_only_task_names(self): return {'cdump': tasks} - def get_resources(self): + def get_tasks(self): + + self.task_tasks = dict() + #for cdump in self.task_names.keys(): + # self.task_tasks[cdump] = Tasks(self, cdump) + + def assemble_tasks(self): - self.task_resources = dict() - for cdump, cdump_tasks in self.task_names.items(): - self.task_resources[cdump] = dict() - for task in cdump_tasks: - self.task_resources[cdump][task] = wfu.get_resource(self.job_configs[task], task, cdump=cdump) + return None diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 45cc0d69998..49d8921c4aa 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -7,7 +7,7 @@ rocoto.py ABOUT: - Helper module to create tasks, metatasks, and dependencies for Rocoto + Helper module to create task_tasks, metatasks, and dependencies for Rocoto ''' __all__ = ['create_task', 'create_metatask', @@ -66,14 +66,18 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: final = task_dict.get('final', False) command = task_dict.get('command', 'sleep 10') jobname = task_dict.get('jobname', 'demojob') - account = task_dict.get('account', 'batch') - queue = task_dict.get('queue', 'debug') - partition = task_dict.get('partition', None) - walltime = task_dict.get('walltime', '00:01:00') + resources_dict = task_dict['resources'] + account = resources_dict.get('account', 'batch') + queue = resources_dict.get('queue', 'debug') + partition = resources_dict.get('partition', None) + walltime = resources_dict.get('walltime', '00:01:00') + native = resources_dict.get('native', None) + memory = resources_dict.get('memory', None) + # compute = resources_dict.get('compute', None) # TODO - eliminate + nodes = resources_dict.get('nodes', 1) + ppn = resources_dict.get('ppn', 1) + threads = resources_dict.get('threads', 1) log = task_dict.get('log', 'demo.log') - native = task_dict.get('native', None) - memory = task_dict.get('memory', None) - resources = task_dict.get('resources', None) envar = task_dict.get('envars', None) dependency = task_dict.get('dependency', None) @@ -91,9 +95,10 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: if partition is not None: strings.append(f'\t{partition}\n') - if resources is not None: - strings.append(f'\t{resources}\n') strings.append(f'\t{walltime}\n') + # if compute is not None: # TODO - eliminate this block + # strings.append(f'\t{compute}\n') + strings.append(f'\t{nodes}:ppn={ppn}:tpp={threads}\n') if memory is not None: strings.append(f'\t{memory}\n') if native is not None: diff --git a/ush/rocoto/rocoto_viewer.py b/ush/rocoto/rocoto_viewer.py index 63db6f25383..24e6a78b212 100755 --- a/ush/rocoto/rocoto_viewer.py +++ b/ush/rocoto/rocoto_viewer.py @@ -323,7 +323,7 @@ def usage(message=None): -w workflow.xml -d database.db Optional arguments: - --listtasks --- print out a list of all tasks + --listtasks --- print out a list of all task_tasks --html=filename.html --- creates an HTML document of status --help --- print this usage message''') @@ -441,7 +441,7 @@ def get_arguments(): send_html_to_rzdm = True if list_tasks and workflow_file is None: - usage('In order to list tasks you must supply the XML worflow-file') + usage('In order to list task_tasks you must supply the XML worflow-file') if only_check_point and (workflow_file is None or database_file is None or save_checkfile_path is None): usage('To use the check point output you must specify the workflow, data base, and the specific name of the checkpoint file') @@ -582,7 +582,7 @@ def help_screen(screen): '(->) Next Cycle own (or) Page-dwn to scroll', '(<-) Previous Cycle own (or) Page-up to scroll ', ' ', - ' + Arrow Up to selected multiple tasks', + ' + Arrow Up to selected multiple task_tasks', ' + Arrow Down for using with rocoto utils', 'Double-Click or to expand/collapse metatasks', ' ', @@ -595,7 +595,7 @@ def help_screen(screen): '', 'oads and renews status data (no rocotorun)', 'inds the last cycle with a running task', - 'nloads and clears all previously seleted tasks', + 'nloads and clears all previously seleted task_tasks', 'makes a symlink of log file of highlited task'] for i in range(0, len(helpstr)): @@ -837,7 +837,7 @@ def get_rocoto_check(params, queue_check): def rocoto_boot(params): workflow_file, database_file, cycle, metatask_list, task_list = params - stat = syscall([rocotoboot, '--workflow', workflow_file, '--database', database_file, '--cycles', cycle, '--tasks', task_list]) + stat = syscall([rocotoboot, '--workflow', workflow_file, '--database', database_file, '--cycles', cycle, '--task_tasks', task_list]) if stat is None: display_results('rocotoboot failed!!', '') return stat @@ -1001,7 +1001,7 @@ def get_tasklist(workflow_file): else: metatask_list[first_task_resolved_name].append(task[0]) if list_tasks: - print(f'tasks: , {i}, {task[0]}, {task[1]}, LOG:, {task[2]}') + print(f'task_tasks: , {i}, {task[0]}, {task[1]}, LOG:, {task[2]}') # Default expantion of metatasks True = collapsed # for metatask,metatasks in metatask_list.iteritems(): @@ -2246,30 +2246,30 @@ def main(screen): screen.clear() process = '' if highlight_CYCLE: - screen.addstr(f'Are you sure you want to rewind all the tasks in the cycle {execute_cycle} by running:\n\n') + screen.addstr(f'Are you sure you want to rewind all the task_tasks in the cycle {execute_cycle} by running:\n\n') process = '-a' # highlight_WORKFLOW = False elif execute_metatask_check and len(selected_tasks[execute_cycle]) == 0: for tasks in metatask_list_of_selected_metatask: process += '-t ' + tasks + ' ' - screen.addstr(f'Are you sure you want to rewind all the tasks in the metatask ({execute_task}) by running:\n\n') + screen.addstr(f'Are you sure you want to rewind all the task_tasks in the metatask ({execute_task}) by running:\n\n') elif len(selected_tasks[execute_cycle]) != 0 or len(selected_meta_tasks[execute_cycle]) != 0: if len(selected_tasks[execute_cycle]) != 0: selected_tasks_string = '' - screen.addstr('Selected tasks:\n\n') + screen.addstr('Selected task_tasks:\n\n') for tasks in selected_tasks[execute_cycle]: selected_tasks_string += tasks + '\t' process += '-t ' + tasks + ' ' screen.addstr(selected_tasks_string + '\n\n') if len(selected_meta_tasks[execute_cycle]) != 0: selected_tasks_string = '' - screen.addstr(f'Selected {len(selected_meta_tasks[execute_cycle]):d} entire meta-tasks and their tasks:\n\n') + screen.addstr(f'Selected {len(selected_meta_tasks[execute_cycle]):d} entire meta-task_tasks and their task_tasks:\n\n') for meta_task_selected in selected_meta_tasks[execute_cycle]: for tasks in metatask_list_by_name[meta_task_selected]: selected_tasks_string += tasks + '\t' process += '-t ' + tasks + ' ' screen.addstr(selected_tasks_string + '\n\n') - screen.addstr('\nAre you sure you want to rewind all these seleted tasks by running:\n\n') + screen.addstr('\nAre you sure you want to rewind all these seleted task_tasks by running:\n\n') elif len(selected_tasks[execute_cycle]) == 0: process = '-t ' + execute_task screen.addstr(f'Are you sure you want to rewind the single task {execute_task} by running:\n\n') @@ -2280,7 +2280,7 @@ def main(screen): if event == ord('y') or event == ord('Y'): params = (workflow_file, database_file, execute_cycle, process) results = rocoto_rewind(params) - results_params = ('', '', 'rewind', execute_cycle, 'tasks') + results_params = ('', '', 'rewind', execute_cycle, 'task_tasks') try: display_results(results, screen, results_params) except Exception: @@ -2315,7 +2315,7 @@ def main(screen): screen.addstr(f'You have selected to boot the entire cycle {execute_cycle}:\n\n', curses.A_BOLD) tasks_to_boot = tasks_in_cycle[cycle] elif len(selected_tasks[execute_cycle]) != 0: - screen.addstr('You have a list selected tasks boot:\n\n', curses.A_BOLD) + screen.addstr('You have a list selected task_tasks boot:\n\n', curses.A_BOLD) tasks_to_boot = selected_tasks[execute_cycle] elif len(selected_meta_tasks[execute_cycle]) != 0: screen.addstr(f'Are you sure you want boot the metatask {selected_meta_tasks[execute_cycle][0]} by running rocotoboot with:') @@ -2333,9 +2333,9 @@ def main(screen): boot_task_list = boot_task_list[:-1] screen.addstr(list_of_tasks) - screen.addstr(f'\n\nAre you sure you want to boot all the tasks and/or metatasks in the cycle {execute_cycle} by running:\n\n', curses.A_BOLD) + screen.addstr(f'\n\nAre you sure you want to boot all the task_tasks and/or metatasks in the cycle {execute_cycle} by running:\n\n', curses.A_BOLD) if len(boot_task_list) != 0: - list_of_tasks = ' --tasks ' + "'" + boot_task_list + "'" + list_of_tasks = ' --task_tasks ' + "'" + boot_task_list + "'" screen.addstr(f'rocotoboot -c {execute_cycle} -d {basename(database_file)} -w {basename(workflow_file)} {list_meta_tasks + list_of_tasks}\n\n') screen.addstr('Enter: es or o', curses.A_BOLD) diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 4c1971ed12f..9466d49d094 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -97,7 +97,7 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() reservation = base.get('RESERVATION', 'NONE').upper() - #tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] + #task_tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] tasks = ['prep', 'anal', 'analcalc'] if cdump in ['gdas']: @@ -105,13 +105,13 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: - #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] + #task_tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] tasks += ['fcst', 'post', 'vrfy', 'arch'] if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] + #task_tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: tasks += ['postsnd'] @@ -170,7 +170,7 @@ def get_hyb_resources(dict_configs): dict_resources = OrderedDict() - # These tasks can be run in either or both cycles + # These task_tasks can be run in either or both cycles if lobsdiag_forenkf in ['.T.', '.TRUE.']: tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] else: @@ -206,7 +206,7 @@ def get_hyb_resources(dict_configs): dict_resources[f'{cdump}{task}'] = ''.join(strings) - # These tasks are always run as part of the GDAS cycle + # These task_tasks are always run as part of the GDAS cycle cdump = 'gdas' tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] for task in tasks2: @@ -301,7 +301,7 @@ def create_xml(dict_configs): dict_hyb_resources = get_hyb_resources(dict_configs) dict_hyb_tasks = get_hyb_tasks(dict_configs) - # Removes &MEMORY_JOB_DUMP post mortem from hyb tasks + # Removes &MEMORY_JOB_DUMP post mortem from hyb task_tasks hyp_tasks = {'gdaseobs':'gdaseobs', 'gdasediag':'gdasediag', 'gdaseomg':'gdaseomn', @@ -333,7 +333,7 @@ def create_xml(dict_configs): dict_gfs_resources = get_gdasgfs_resources(dict_configs, cdump='gfs') dict_gfs_tasks = get_gdasgfs_tasks(dict_configs, cdump='gfs') - # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks + # Removes &MEMORY_JOB_DUMP post mortem from gdas task_tasks for each_task, each_resource_string in dict_gdas_resources.items(): if each_task not in dict_gdas_tasks: continue @@ -344,7 +344,7 @@ def create_xml(dict_configs): temp_task_string.append(each_line) dict_gdas_tasks[each_task] = ''.join(temp_task_string) - # Removes &MEMORY_JOB_DUMP post mortem from gfs tasks + # Removes &MEMORY_JOB_DUMP post mortem from gfs task_tasks for each_task, each_resource_string in dict_gfs_resources.items(): if each_task not in dict_gfs_tasks: continue diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index e2671348be1..894c2e637bc 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -166,7 +166,7 @@ def create_xml(dict_configs): resources = get_resources(dict_configs, cdump=base['CDUMP']) workflow = get_workflow_body(dict_configs, cdump=base['CDUMP']) - # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks + # Removes &MEMORY_JOB_DUMP post mortem from gdas task_tasks temp_workflow = '' memory_dict = [] for each_resource_string in re.split(r'(\s+)', resources): diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index c726ca0a8e1..07bfacb47ee 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -2,16 +2,20 @@ import numpy as np import rocoto -import workflow_utils as wfu from applications import Application +from hosts import Host class Tasks: + SERVICE_TASKS = ['arch', 'earc', 'getic'] + def __init__(self, app: Application, cdump: str) -> None: - self.cdump = cdump self.app = app + self.cdump = cdump + + self._scheduler = Host.get_scheduler # Save dict_configs and base in the internal state (never know where it may be needed) self._job_configs = self.app.job_configs @@ -46,6 +50,105 @@ def _get_hybgroups(nens: int, nmem_per_group: int): groups = ' '.join([f'{x:02d}' for x in range(1, int(ngrps) + 1)]) return groups + def get_resource(self, task_name): + """ + Given a task name (task_name) and its configuration (task_names), + return a dictionary of resources (task_resource) used by the task. + Task resource dictionary includes: + account, walltime, cores, nodes, ppn, threads, memory, queue, partition, native + """ + + task_dict = self._job_configs[task_name] + + account = task_dict['ACCOUNT'] + + walltime = task_dict[f'wtime_{task_name}'] + if self.cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_dict.keys(): + walltime = task_dict[f'wtime_{task_name}_gfs'] + + cores = task_dict[f'npe_{task_name}'] + if self.cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_dict.keys(): + cores = task_dict[f'npe_{task_name}_gfs'] + + ppn = task_dict[f'npe_node_{task_name}'] + if self.cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_dict.keys(): + ppn = task_dict[f'npe_node_{task_name}_gfs'] + + nodes = np.int(np.ceil(np.float(cores) / np.float(ppn))) + + threads = task_dict[f'nth_{task_name}'] + if self.cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_dict.keys(): + threads = task_dict[f'nth_{task_name}_gfs'] + + compute = f'{nodes}:ppn={ppn}:tpp={threads}' # TODO - remove dependence on compute + + memory = task_dict.get(f'memory_{task_name}', None) + + native = '--export=NONE' if self._scheduler in ['slurm'] else None + + queue = task_dict['QUEUE'] + if task_name in Tasks.SERVICE_TASKS and self._scheduler not in ['slurm']: + queue = task_dict['QUEUE_SERVICE'] + + partition = None + if self._scheduler in ['slurm']: + partition = task_dict['QUEUE_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_dict['PARTITION_BATCH'] + + task_resource = {'account': account, + 'walltime': walltime, + 'nodes': nodes, + 'cores': cores, + 'ppn': ppn, + 'threads': threads, + 'compute': compute, # TODO - eliminate dependence on compute + 'memory': memory, + 'native': native, + 'queue': queue, + 'partition': partition} + + return task_resource + + def get_task(self, task_name, *args, **kwargs): + return getattr(self, task_name, *args, **kwargs)() + + @staticmethod + def create_wf_task(task_name, resources, + cdump='gdas', cycledef=None, envar=None, dependency=None, + metatask=None, varname=None, varval=None, vardict=None, + final=False): + + tasknamestr = f'{cdump}{task_name}' + metatask_dict = None + if metatask is not None: + tasknamestr = f'{tasknamestr}#{varname}#' + metatask_dict = {'metataskname': f'{cdump}{metatask}', + 'varname': f'{varname}', + 'varval': f'{varval}', + 'vardict': vardict} + + cycledefstr = cdump if cycledef is None else cycledef + + task_dict = {'taskname': f'{tasknamestr}', + 'cycledef': f'{cycledefstr}', + 'maxtries': '&MAXTRIES;', + 'command': f'&JOBS_DIR;/{task_name}.sh', + 'jobname': f'&PSLOT;_{tasknamestr}_@H', + 'resources': resources, + 'log': f'&ROTDIR;/logs/@Y@m@d@H/{tasknamestr}.log', + 'envars': envar, + 'dependency': dependency, + 'final': final} + + task = rocoto.create_task(task_dict) if metatask is None else rocoto.create_metatask(task_dict, metatask_dict) + + return ''.join(task) + + @staticmethod + def is_this_a_gdas_task(cdump, task_name): + if cdump != 'gdas': + raise TypeError(f'{task_name} must be part of the "gdas" cycle and not {cdump}') + + # Specific Tasks begin here def coupled_ic(self): cpl_ic = self._job_configs['coupled_ic'] @@ -91,7 +194,8 @@ def coupled_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('coupled_ic', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('coupled_ic') + task = self.create_wf_task('coupled_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -106,7 +210,8 @@ def getic(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) - task = wfu.create_wf_task('getic', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('getic') + task = self.create_wf_task('getic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -129,7 +234,8 @@ def init(self): dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - task = wfu.create_wf_task('init', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('init') + task = self.create_wf_task('init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -155,14 +261,16 @@ def prep(self): if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: cycledef = 'gdas' - task = wfu.create_wf_task('prep', cdump=self.cdump, envar=self.envars, dependency=dependencies, + resources = self.get_resource('prep') + task = self.create_wf_task('prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) return task def waveinit(self): - task = wfu.create_wf_task('waveinit', cdump=self.cdump, envar=self.envars) + resources = self.get_resource('waveinit') + task = self.create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars) return task @@ -173,7 +281,8 @@ def waveprep(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveprep', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('waveprep') + task = self.create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -204,7 +313,8 @@ def aerosol_init(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('aerosol_init', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('aerosol_init') + task = self.create_wf_task('aerosol_init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -218,7 +328,10 @@ def anal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('anal', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('anal') + task = self.create_wf_task('anal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + return task def analcalc(self): @@ -231,7 +344,8 @@ def analcalc(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('analcalc', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('analcalc') + task = self.create_wf_task('analcalc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -242,7 +356,8 @@ def analdiag(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('analdiag', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('analdiag') + task = self.create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -253,19 +368,20 @@ def gldas(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gldas', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('gldas') + task = self.create_wf_task('gldas', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task - def fcst(self, fcst_type): + def fcst(self): - fcst_map = {'free': self._fcst_free, + fcst_map = {'forecast-only': self._fcst_free, 'cycled': self._fcst_cycled} try: - task = fcst_map[fcst_type] + task = fcst_map[self.app.mode] except KeyError: - raise NotImplementedError(f'{fcst_type} is not a valid type.\n' + + raise NotImplementedError(f'{self.app.mode} is not a valid type.\n' + 'Currently supported forecast types are:\n' + f'{" | ".join(fcst_map.keys())}') @@ -292,7 +408,8 @@ def _fcst_free(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - task = wfu.create_wf_task('fcst', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('fcst') + task = self.create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -313,12 +430,17 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - task = wfu.create_wf_task('fcst', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('fcst') + task = self.create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task def post(self): - return self._post_task('post', add_anl_to_post=True) + add_anl_to_post = False + if self.app.mode in ['cycled']: + add_anl_to_post = True + + return self._post_task('post', add_anl_to_post=add_anl_to_post) def ocnpost(self): return self._post_task('ocnpost', add_anl_to_post=False) @@ -381,7 +503,8 @@ def _get_postgroups(cdump, config, add_anl=False): varval1, varval2, varval3 = _get_postgroups(self.cdump, self._job_configs[task_name], add_anl=add_anl_to_post) vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task(task_name, cdump=self.cdump, envar=postenvars, dependency=dependencies, + resources = self.get_resource(task_name) + task = self.create_wf_task(task_name, resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, metatask=task_name, varname=varname1, varval=varval1, vardict=vardict) return task @@ -394,7 +517,8 @@ def wavepostsbs(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostsbs', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('wavepostsbs') + task = self.create_wf_task('wavepostsbs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -404,7 +528,8 @@ def wavepostbndpnt(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpnt', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('wavepostbndpnt') + task = self.create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -415,7 +540,8 @@ def wavepostbndpntbll(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpntbll', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('wavepostbndpntbll') + task = self.create_wf_task('wavepostbndpntbll', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -428,7 +554,8 @@ def wavepostpnt(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostpnt', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('wavepostpnt') + task = self.create_wf_task('wavepostpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -438,7 +565,8 @@ def wavegempak(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavegempak', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('wavegempak') + task = self.create_wf_task('wavegempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -450,7 +578,8 @@ def waveawipsbulls(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveawipsbulls', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('waveawipsbulls') + task = self.create_wf_task('waveawipsbulls', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -460,7 +589,8 @@ def waveawipsgridded(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveawipsgridded', cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource('waeawipsgridded') + task = self.create_wf_task('waveawipsgridded', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -488,7 +618,8 @@ def _wafs_task(self, task_name): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task(task_name, cdump=self.cdump, envar=self.envars, dependency=dependencies) + resources = self.get_resource(task_name) + task = self.create_wf_task(task_name, resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -497,7 +628,9 @@ def wafsblending(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}wafsgrib2'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('wafsblending') + task = self.create_wf_task('wafsblending', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -506,7 +639,9 @@ def wafsblending0p25(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}wafsgrib20p25'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending0p25', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('wafsblending0p25') + task = self.create_wf_task('wafsblending0p25', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -515,7 +650,9 @@ def postsnd(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('postsnd') + task = self.create_wf_task('postsnd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -573,7 +710,8 @@ def _get_awipsgroups(cdump, config): varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._job_configs['awips']) vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + resources = self.get_resource('awips') + task = self.create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, metatask='awips', varname=varname1, varval=varval1, vardict=vardict) return task @@ -584,7 +722,9 @@ def gempak(self): dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('gempak') + task = self.create_wf_task('gempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -593,7 +733,9 @@ def vrfy(self): dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('vrfy') + task = self.create_wf_task('vrfy', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -613,7 +755,8 @@ def metp(self): varname1 = 'metpcase' varval1 = 'g2g1 g2o1 pcp1' - task = wfu.create_wf_task('metp', cdump=self.cdump, envar=metpenvars, dependency=dependencies, + resources = self.get_resource('metp') + task = self.create_wf_task('metp', resources, cdump=self.cdump, envar=metpenvars, dependency=dependencies, metatask='metp', varname=varname1, varval=varval1) return task @@ -635,11 +778,13 @@ def arch(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=self.cdump, envar=self.envars, dependency=dependencies) + + resources = self.get_resource('arch') + task = self.create_wf_task('arch', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task - # Start of ensemble tasks + # Start of ensemble task_tasks def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} @@ -648,7 +793,8 @@ def eobs(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('eobs', cdump=self.cdump, envar=self.envars, dependency=dependencies, + resources = self.get_resource('eobs') + task = self.create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=self.hyb_cycledef) return task @@ -664,7 +810,8 @@ def eomg(self): groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['eobs']['NMEM_EOMGGRP']) - task = wfu.create_wf_task('eomg', cdump=self.cdump, envar=eomgenvars, dependency=dependencies, + resources = self.get_resource('eomg') + task = self.create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, metatask='eomn', varname='grp', varval=groups, cycledef=self.hyb_cycledef) @@ -676,7 +823,8 @@ def ediag(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('ediag', cdump=self.cdump, envar=self.envars, dependency=dependencies, + resources = self.get_resource('ediag') + task = self.create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=self.hyb_cycledef) return task @@ -690,16 +838,12 @@ def eupd(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('eupd', cdump=self.cdump, envar=self.envars, dependency=dependencies, + resources = self.get_resource('eupd') + task = self.create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=self.hyb_cycledef) return task - @staticmethod - def is_this_a_gdas_task(cdump, task_name): - if cdump != 'gdas': - raise TypeError(f'{task_name} must be part of the "gdas" cycle and not {cdump}') - def ecen(self): self.is_this_a_gdas_task(self.cdump, 'ecen') @@ -712,7 +856,7 @@ def _get_ecengroups(): necengrp = self._job_configs['ecen']['NECENGRP'] ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) - fhrs = [f'{fhr:03d}' for fhr in fhrs] + fhrs = [f'{int(fhr):03d}' for fhr in fhrs] fhrs = np.array_split(fhrs, ngrps) fhrs = [fhr.tolist() for fhr in fhrs] @@ -746,7 +890,8 @@ def _get_ecengroups(): varval1, varval2, varval3 = _get_ecengroups() vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('ecen', cdump=self.cdump, envar=ecenenvars, dependency=dependencies, + resources = self.get_resource('ecen') + task = self.create_wf_task('ecen', resources, cdump=self.cdump, envar=ecenenvars, dependency=dependencies, metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict, cycledef=self.hyb_cycledef) return task @@ -764,7 +909,9 @@ def esfc(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('esfc', cdump='gdas', envar=self.envars, dependency=dependencies, + + resources = self.get_resource('esfc') + task = self.create_wf_task('esfc', resources, cdump='gdas', envar=self.envars, dependency=dependencies, cycledef=self.hyb_cycledef) return task @@ -785,7 +932,8 @@ def efcs(self): groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['efcs']['NMEM_EFCSGRP']) - task = wfu.create_wf_task('efcs', cdump=self.cdump, envar=efcsenvars, dependency=dependencies, + resources = self.get_resource('efcs') + task = self.create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, metatask='efmn', varname='grp', varval=groups, cycledef=self.hyb_cycledef) @@ -802,7 +950,8 @@ def echgres(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('echgres', cdump=self.cdump, envar=self.envars, dependency=dependencies, + resources = self.get_resource('echgres') + task = self.create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=self.hyb_cycledef) return task @@ -845,7 +994,8 @@ def _get_eposgroups(epos): varval1, varval2, varval3 = _get_eposgroups(self._job_configs['epos']) vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('epos', cdump=self.cdump, envar=eposenvars, dependency=dependencies, + resources = self.get_resource('epos') + task = self.create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, cycledef=self.hyb_cycledef) @@ -863,9 +1013,10 @@ def earc(self): earcenvars = self.envars earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['efcs']['NMEM_EARCGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['earc']['NMEM_EARCGRP']) - task = wfu.create_wf_task('earc', cdump=self.cdump, envar=earcenvars, dependency=dependencies, + resources = self.get_resource('earc') + task = self.create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, metatask='eamn', varname='grp', varval=groups, cycledef=self.hyb_cycledef) diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index a8c7d42c6d4..bbe7c96cf7f 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -9,12 +9,10 @@ from datetime import timedelta from typing import Dict, Any from rocoto import create_task, create_metatask -from hosts import Host -#SERVICE_TASKS = ['arch', 'earc', 'getic'] - -def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, +def create_wf_task(task, resources, + cdump='gdas', cycledef=None, envar=None, dependency=None, metatask=None, varname=None, varval=None, vardict=None, final=False): @@ -27,7 +25,6 @@ def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=Non 'varval': f'{varval}', 'vardict': vardict} - taskstr = f'{cdump}{taskstr}' cycledefstr = cdump if cycledef is None else cycledef task_dict = {'taskname': f'{taskstr}', @@ -35,28 +32,15 @@ def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=Non 'maxtries': '&MAXTRIES;', 'command': f'&JOBS_DIR;/{task}.sh', 'jobname': f'&PSLOT;_{taskstr}_@H', - 'account': '&ACCOUNT;', - 'queue': f'&QUEUE_{task.upper()}_{cdump.upper()};', - 'walltime': f'&WALLTIME_{task.upper()}_{cdump.upper()};', - 'native': f'&NATIVE_{task.upper()}_{cdump.upper()};', - 'memory': f'&MEMORY_{task.upper()}_{cdump.upper()};', - 'resources': f'&RESOURCES_{task.upper()}_{cdump.upper()};', + 'resources': resources, 'log': f'&ROTDIR;/logs/@Y@m@d@H/{taskstr}.log', 'envars': envar, 'dependency': dependency, 'final': final} - # Add partition for machines using slurm - if Host.get_scheduler in ['slurm']: - task_dict['partition'] = f'&PARTITION_{task.upper()}_{cdump.upper()};' - - if metatask is None: - task = create_task(task_dict) - else: - task = create_metatask(task_dict, metatask_dict) - task = ''.join(task) + task = create_task(task_dict) if metatask is None else create_metatask(task_dict, metatask_dict) - return task + return ''.join(task) def check_expdir(cmd_expdir, cfg_expdir): @@ -133,66 +117,6 @@ def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: return base_out -def get_resource(task_dict: dict, task_name: str, cdump: str = 'gdas') -> dict: - """ - Given a task name (task_name) and its configuration (task_names), - return a dictionary of resources (task_resource) used by the task. - Task resource dictionary includes: - account, walltime, cores, nodes, ppn, threads, memory, queue, partition, native - """ - - SERVICE_TASKS = ['arch', 'earc', 'getic'] - scheduler = Host.get_scheduler - - account = task_dict['ACCOUNT'] - - walltime = task_dict[f'wtime_{task_name}'] - if cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_dict.keys(): - walltime = task_dict[f'wtime_{task_name}_gfs'] - - cores = task_dict[f'npe_{task_name}'] - if cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_dict.keys(): - cores = task_dict[f'npe_{task_name}_gfs'] - - ppn = task_dict[f'npe_node_{task_name}'] - if cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_dict.keys(): - ppn = task_dict[f'npe_node_{task_name}_gfs'] - - nodes = np.int(np.ceil(np.float(cores) / np.float(ppn))) - - threads = task_dict[f'nth_{task_name}'] - if cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_dict.keys(): - threads = task_dict[f'nth_{task_name}_gfs'] - - compute = f'{nodes}:ppn={ppn}:tpp={threads}' # TODO - remove dependence on compute - - memory = task_dict.get(f'memory_{task_name}', None) - - native = '--export=NONE' if scheduler in ['slurm'] else None - - queue = task_dict['QUEUE'] - if task_name in SERVICE_TASKS and scheduler not in ['slurm']: - queue = task_dict['QUEUE_SERVICE'] - - partition = None - if scheduler in ['slurm']: - partition = task_dict['QUEUE_SERVICE'] if task_name in SERVICE_TASKS else task_dict['PARTITION_BATCH'] - - task_resource = {'account': account, - 'walltime': walltime, - 'nodes': nodes, - 'cores': cores, - 'ppn': ppn, - 'threads': threads, - 'compute': compute, - 'memory': memory, - 'native': native, - 'queue': queue, - 'partition': partition} - - return task_resource - - def create_crontab(base, cronint=5): """ Create crontab to execute rocotorun every cronint (5) minutes diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index 807236d6400..3eaa530cdbb 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -4,135 +4,147 @@ from collections import OrderedDict from hosts import Host import rocoto - - -def get_preamble(): - """ - Generate preamble for XML - """ - - strings = ['', - ''] - - return '\n'.join(strings) - - -def get_definitions(base: dict, mode: str) -> str: - """ - Create entities related to the experiment - """ - - host = Host() - - scheduler = host.scheduler - - entity = OrderedDict() - - entity['PSLOT'] = base['PSLOT'] - entity['SDATE'] = base['SDATE'].strftime('%Y%m%d%H%M') - entity['EDATE'] = base['EDATE'].strftime('%Y%m%d%H%M') - - if mode in ['cycled']: - entity['INTERVAL'] = '06:00:00' - if base['gfs_cyc'] in [1, 2, 4]: - entity['SDATE_GFS'] = base['SDATE_GFS'].strftime('%Y%m%d%H%M') - entity['EDATE_GFS'] = base['EDATE_GFS'].strftime('%Y%m%d%H%M') - entity['INTERVAL_GFS'] = base['INTERVAL_GFS'] - entity['DMPDIR'] = base['DMPDIR'] - - elif mode in ['forecast-only']: - entity['INTERVAL'] = base.get('INTERVAL_GFS', '24:00:00') - entity['CDUMP'] = base['CDUMP'] - entity['CASE'] = base['CASE'] # TODO - is this really used in the XML? - entity['ICSDIR'] = base['ICSDIR'] - - entity['EXPDIR'] = base['EXPDIR'] - entity['ROTDIR'] = base['ROTDIR'] - entity['RUN_ENVIR'] = base.get('RUN_ENVIR', 'emc') - entity['HOMEgfs'] = base['HOMEgfs'] - entity['JOBS_DIR'] = base['BASE_JOB'] - entity['ACCOUNT'] = base['ACCOUNT'] - entity['QUEUE'] = base['QUEUE'] - entity['QUEUE_SERVICE'] = base['QUEUE_SERVICE'] - entity['SCHEDULER'] = scheduler - if scheduler in ['slurm']: - entity['PARTITION_BATCH'] = base['PARTITION_BATCH'] - entity['PARTITION_SERVICE'] = base['PARTITION_SERVICE'] - entity['ARCHIVE_TO_HPSS'] = base.get('HPSSARCH', 'NO') - entity['CYCLETHROTTLE'] = base.get('ROCOTO_CYCLETHROTTLE', 3) - entity['TASKTHROTTLE'] = base.get('ROCOTO_TASKTHROTTLE', 25) - entity['MAXTRIES'] = base.get('ROCOTO_MAXTRIES', 2) - entity['VERBOSITY'] = base.get('ROCOTO_VERBOSITY', 10) - - # Put them all in an XML key-value syntax - strings = [] - for key, value in entity.items(): - strings.append('\t' + rocoto.create_entity(key, value)) - - return '\n'.join(strings) - - -def get_workflow_header(): - """ - Create the workflow header block - """ - - strings = ['', - ']>', - '', - '', - '', - '\t&EXPDIR;/logs/@Y@m@d@H.log', - '', - '\t'] - - return '\n'.join(strings) - - -def get_cycledefs(exp_type, **kwargs): - - cycledef_map = {'cycled': _get_cycledefs_cycled, - 'forecast-only': _get_cycledefs_forecast_only} - - try: - cycledef_map[exp_type](**kwargs) - except KeyError: - raise TypeError(f'{exp_type} is not a valid application.' + - 'Current plot types supported are:\n' + - f'{" | ".join(cycledef_map.keys())}"') - - -def _get_cycledefs_cycled(**kwargs): - strings = ['\t&SDATE; &SDATE; 06:00:00', - '\t&SDATE; &EDATE; 06:00:00', - '\t&SDATE; &EDATE; 06:00:00'] - if base['gfs_cyc'] != 0: - strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;') - - return '\n'.join(strings) - - -def _get_cycledefs_forecast_only(**kwqrgs): - - strings = f'\t&SDATE; &EDATE; &INTERVAL;' - - return strings - - -def get_workflow_footer(): - """ - Generate workflow footer - """ - - return '\n\n' +from applications import Application + + +class RocotoXML: + + def __init__(self, app: Application) -> None: + + self._app = app + + self._base = self._app.job_configs['base'] + + self.preamble = self._get_preamble + self.definitions = self._get_definitions() + self.header = self._get_workflow_header + self.cycledefs = self._get_cycledefs() + self.footer = self._get_workflow_footer + + @staticmethod + def _get_preamble(): + """ + Generate preamble for XML + """ + + strings = ['', + ''] + + return '\n'.join(strings) + + def _get_definitions(self) -> str: + """ + Create entities related to the experiment + """ + + host = Host() + + scheduler = host.scheduler + + entity = OrderedDict() + + entity['PSLOT'] = self._base['PSLOT'] + entity['SDATE'] = self._base['SDATE'].strftime('%Y%m%d%H%M') + entity['EDATE'] = self._base['EDATE'].strftime('%Y%m%d%H%M') + + if self._app.mode in ['cycled']: + entity['INTERVAL'] = '06:00:00' + if self._base['gfs_cyc'] in [1, 2, 4]: + entity['SDATE_GFS'] = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M') + entity['EDATE_GFS'] = self._base['EDATE_GFS'].strftime('%Y%m%d%H%M') + entity['INTERVAL_GFS'] = self._base['INTERVAL_GFS'] + entity['DMPDIR'] = self._base['DMPDIR'] + + elif self._app.mode in ['forecast-only']: + entity['INTERVAL'] = self._base.get('INTERVAL_GFS', '24:00:00') + entity['CDUMP'] = self._base['CDUMP'] + entity['CASE'] = self._base['CASE'] # TODO - is this really used in the XML? + entity['ICSDIR'] = self._base['ICSDIR'] + + entity['EXPDIR'] = self._base['EXPDIR'] + entity['ROTDIR'] = self._base['ROTDIR'] + entity['RUN_ENVIR'] = self._base.get('RUN_ENVIR', 'emc') + entity['HOMEgfs'] = self._base['HOMEgfs'] + entity['JOBS_DIR'] = self._base['BASE_JOB'] + entity['ACCOUNT'] = self._base['ACCOUNT'] + entity['QUEUE'] = self._base['QUEUE'] + entity['QUEUE_SERVICE'] = self._base['QUEUE_SERVICE'] + entity['SCHEDULER'] = scheduler + if scheduler in ['slurm']: + entity['PARTITION_BATCH'] = self._base['PARTITION_BATCH'] + entity['PARTITION_SERVICE'] = self._base['PARTITION_SERVICE'] + entity['ARCHIVE_TO_HPSS'] = self._base.get('HPSSARCH', 'NO') + entity['CYCLETHROTTLE'] = self._base.get('ROCOTO_CYCLETHROTTLE', 3) + entity['TASKTHROTTLE'] = self._base.get('ROCOTO_TASKTHROTTLE', 25) + entity['MAXTRIES'] = self._base.get('ROCOTO_MAXTRIES', 2) + entity['VERBOSITY'] = self._base.get('ROCOTO_VERBOSITY', 10) + + # Put them all in an XML key-value syntax + strings = [] + for key, value in entity.items(): + strings.append('\t' + rocoto.create_entity(key, value)) + + return '\n'.join(strings) + + @staticmethod + def _get_workflow_header(): + """ + Create the workflow header block + """ + + strings = ['', + ']>', + '', + '', + '', + '\t&EXPDIR;/logs/@Y@m@d@H.log', + '', + '\t'] + + return '\n'.join(strings) + + def _get_cycledefs(self): + + cycledef_map = {'cycled': self._get_cycledefs_cycled, + 'forecast-only': self._get_cycledefs_forecast_only} + + try: + cycledef_map[self._app.mode]() + except KeyError: + raise TypeError(f'{self._app.mode} is not a valid application.' + + 'Current plot types supported are:\n' + + f'{" | ".join(cycledef_map.keys())}"') + + def _get_cycledefs_cycled(self): + strings = ['\t&SDATE; &SDATE; 06:00:00', + '\t&SDATE; &EDATE; 06:00:00', + '\t&SDATE; &EDATE; 06:00:00'] + if self._app.gfs_cyc != 0: + strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;') + + return '\n'.join(strings) + + @staticmethod + def _get_cycledefs_forecast_only(): + strings = f'\t&SDATE; &EDATE; &INTERVAL;' + + return strings + + @staticmethod + def _get_workflow_footer(): + """ + Generate workflow footer + """ + + return '\n\n' def create_xml(): From 7567d409a0d9a817c6972687a26648290b2554fc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 14 Jun 2022 23:53:59 -0400 Subject: [PATCH 05/41] creates XML --- ush/rocoto/applications.py | 144 ++++++++------ ush/rocoto/rocoto.py | 2 +- ush/rocoto/workflow_tasks.py | 375 +++++++++++++++++++---------------- ush/rocoto/workflow_utils.py | 1 + ush/rocoto/workflow_xml.py | 101 ++++++---- 5 files changed, 350 insertions(+), 273 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 72cdf044e45..a88ae15d0ef 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -3,25 +3,24 @@ from typing import Dict, Any from configuration import Configuration import workflow_utils as wfu -#from workflow_tasks import Tasks +from hosts import Host +class AppConfig: -class Application: VALID_MODES = ['cycled', 'forecast-only'] def __init__(self, mode: str, configuration: Configuration) -> None: - self.task_names = None - self.task_tasks = None if mode not in self.VALID_MODES: raise NotImplementedError(f'{mode} is not a valid application mode.\n' + 'Valid application modes are:\n' + f'{", ".join(self.VALID_MODES)}') self.mode = mode - self.cfg = configuration - _base = self.cfg.parse_config('config.base') + self.scheduler = Host.get_scheduler + + _base = configuration.parse_config('config.base') self.model_app = _base.get('APP', 'ATM') self.do_hybvar = _base.get('DOHYBVAR', False) @@ -39,30 +38,26 @@ def __init__(self, mode: str, configuration: Configuration) -> None: self.do_hpssarch = _base.get('HPSSARCH', False) - # Get a list of all possible jobs that would be part of the application - jobs_map = {'cycled': self._cycled_jobs, - 'forecast-only': self._forecast_only_jobs} - try: - self.job_names = jobs_map[self.mode] - except KeyError: - raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + - 'Valid application modes are:\n' + - f'{", ".join(self.VALID_MODES)}') + # Get a list of all possible configs that would be part of the application + self.configs_names = self._get_app_configs() - # Source the configs for all the jobs in the application - self.job_configs = self.source_configs() + # Source the configs for the jobs in the application + self.configs = self._source_configs(configuration) # Update the base config dictionary based on application upd_base_map = {'cycled': self._cycled_upd_base, 'forecast-only': self._forecast_only_upd_base} try: - self.job_configs['base'] = upd_base_map[self.mode](self.job_configs['base']) + self.configs['base'] = upd_base_map[self.mode](self.configs['base']) except KeyError: raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + 'Valid application modes are:\n' + f'{", ".join(self.VALID_MODES)}') - self._base = self.job_configs['base'] + # Save base in the internal state since it is often needed + self._base = self.configs['base'] + + # Get more configuration options into the class attributes self.gfs_cyc = self._base.get('gfs_cyc') if self.do_hybvar: @@ -79,83 +74,99 @@ def __init__(self, mode: str, configuration: Configuration) -> None: self.wave_cdumps = ['gfs', 'gdas'] elif wave_cdump in ['gfs', 'gdas']: self.wave_cdumps = [wave_cdump] - self.do_wave_bnd = self.job_configs['wavepostsbs'].get('DOBNDPNT_WAVE', False) + self.do_wave_bnd = self.configs['wavepostsbs'].get('DOBNDPNT_WAVE', False) + + # Finally get task names for the application + self.task_names = self.get_task_names() + + def _get_app_configs(self): + + configs_map = {'cycled': self._cycled_configs, + 'forecast-only': self._forecast_only_configs} + try: + configs_names = configs_map[self.mode] + except KeyError: + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + + return configs_names @property - def _cycled_jobs(self): + def _cycled_configs(self): """ - Returns the jobs that are involved in the cycled app + Returns the configs that are involved in the cycled app_config """ - jobs = ['prep', - 'anal', 'analdiag', 'analcalc', 'gldas', - 'fcst', 'post', 'vrfy', 'arch'] + configs = ['prep', + 'anal', 'analdiag', 'analcalc', 'gldas', + 'fcst', 'post', 'vrfy', 'arch'] if self.do_hybvar: - jobs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + configs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] if self.do_metp: - jobs += ['metp'] + configs += ['metp'] if self.do_gempak: - jobs += ['gempak'] + configs += ['gempak'] if self.do_awips: - jobs += ['awips'] + configs += ['awips'] if self.do_wave: - jobs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] if self.do_gempak: - jobs += ['wavegempak'] + configs += ['wavegempak'] if self.do_awips: - jobs += ['waveawipsbulls', 'waveawipsgridded'] + configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_wafs: - jobs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] + configs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] - return jobs + return configs @property - def _forecast_only_jobs(self): + def _forecast_only_configs(self): """ - Returns the jobs that are involved in the forecast-only app + Returns the configs that are involved in the forecast-only app_config """ - jobs = ['fcst', 'post', 'vrfy', 'arch'] + configs = ['fcst', 'post', 'vrfy', 'arch'] if self.model_app in ['S2S', 'S2SW']: - jobs += ['coupled_ic'] + configs += ['coupled_ic'] else: - jobs += ['init'] + configs += ['init'] if self.do_hpssarch: - jobs += ['getic'] + configs += ['getic'] if self.do_aero: - jobs += ['aerosol_init'] + configs += ['aerosol_init'] if self.do_ocean or self.do_ice: - jobs += ['ocnpost'] + configs += ['ocnpost'] if self.do_metp: - jobs += ['metp'] + configs += ['metp'] if self.do_gempak: - jobs += ['gempak'] + configs += ['gempak'] if self.do_awips: - jobs += ['awips'] + configs += ['awips'] if self.do_wave: - jobs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] if self.do_gempak: - jobs += ['wavegempak'] + configs += ['wavegempak'] if self.do_awips: - jobs += ['waveawipsbulls', 'waveawipsgridded'] + configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_wafs: - jobs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] + configs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] - return jobs + return configs @staticmethod def _cycled_upd_base(base_in): @@ -170,22 +181,22 @@ def _forecast_only_upd_base(base_in): return base_out - def source_configs(self) -> Dict[str, Any]: + def _source_configs(self, configuration: Configuration) -> Dict[str, Any]: """ Given the configuration object and jobs, source the configurations for each job and return a dictionary Every job depends on "config.base" """ - dict_jobs = dict() + configs = dict() # Return config.base as well - dict_jobs['base'] = self.cfg.parse_config('config.base') + configs['base'] = configuration.parse_config('config.base') - # Source the list of input task_tasks - for job in self.job_names: + # Source the list of all configs involved in the application + for job in self.configs_names: - # All task_tasks must source config.base first + # All must source config.base first files = ['config.base'] if job in ['eobs', 'eomg']: @@ -200,9 +211,9 @@ def source_configs(self) -> Dict[str, Any]: files += [f'config.{job}'] print(f'sourcing config.{job}') - dict_jobs[job] = self.cfg.parse_config(files) + configs[job] = configuration.parse_config(files) - return dict_jobs + return configs def get_task_names(self): @@ -210,12 +221,14 @@ def get_task_names(self): tasks_map = {'cycled': self._get_cycled_task_names, 'forecast-only': self._get_forecast_only_task_names} try: - self.task_names = tasks_map[self.mode]() + task_names = tasks_map[self.mode]() except KeyError: raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + 'Valid application modes are:\n' + f'{", ".join(self.VALID_MODES)}') + return task_names + def _get_cycled_task_names(self): tasks = ['prep', 'anal', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] @@ -314,6 +327,12 @@ def _get_forecast_only_task_names(self): return {'cdump': tasks} + +class App: + + def __init__(self, app_config: AppConfig): + self.app_config = app_config + def get_tasks(self): self.task_tasks = dict() @@ -323,3 +342,10 @@ def get_tasks(self): def assemble_tasks(self): return None + + def get_app_configuration(self): + + config = type('AppConfig',(),{}) # Initialize a class; type(classname, superclasses, attributedict) + + config.mode = self.mode + diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 49d8921c4aa..765852879da 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -341,7 +341,7 @@ def create_envar(name: str, value: Union[str, float, int]) -> str: :rtype: str """ - return f'{name}{str(value)}' + return f'{name}{str(value)}' def create_cycledef(group=None, start=None, stop=None, step=None): diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 07bfacb47ee..163a0c19993 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -2,28 +2,36 @@ import numpy as np import rocoto -from applications import Application +from applications import AppConfig from hosts import Host +from typing import List class Tasks: - SERVICE_TASKS = ['arch', 'earc', 'getic'] - - def __init__(self, app: Application, cdump: str) -> None: - - self.app = app + VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init', + 'prep', 'anal', 'analcalc', 'analdiag', 'gldas', 'arch', + 'earc', 'ecen', 'echgres', 'ediag', 'efcs', + 'eobs', 'eomg', 'epos', 'esfc', 'eupd', + 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', + 'postsnd', 'awips', 'gempak', + 'wafs', 'wafsblending', 'wafsblending0p25', + 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25', + 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', + 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep'] + + def __init__(self, app_config: AppConfig, cdump: str) -> None: + + self.app_config = app_config self.cdump = cdump - self._scheduler = Host.get_scheduler - # Save dict_configs and base in the internal state (never know where it may be needed) - self._job_configs = self.app.job_configs - self._base = self._job_configs['base'] + self._configs = self.app_config.configs + self._base = self._configs['base'] self.n_tiles = 6 # TODO - this needs to be elsewhere - if self.app.do_hybvar: + if self.app_config.do_hybvar: self.hyb_cycledef = 'enkf' # TODO - sort this out a bit more envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'), @@ -50,6 +58,11 @@ def _get_hybgroups(nens: int, nmem_per_group: int): groups = ' '.join([f'{x:02d}' for x in range(1, int(ngrps) + 1)]) return groups + @staticmethod + def _is_this_a_gdas_task(cdump, task_name): + if cdump != 'gdas': + raise TypeError(f'{task_name} must be part of the "gdas" cycle and not {cdump}') + def get_resource(self, task_name): """ Given a task name (task_name) and its configuration (task_names), @@ -58,41 +71,44 @@ def get_resource(self, task_name): account, walltime, cores, nodes, ppn, threads, memory, queue, partition, native """ - task_dict = self._job_configs[task_name] + scheduler = self.app_config.scheduler + + task_config = self._configs[task_name] - account = task_dict['ACCOUNT'] + account = task_config['ACCOUNT'] - walltime = task_dict[f'wtime_{task_name}'] - if self.cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_dict.keys(): - walltime = task_dict[f'wtime_{task_name}_gfs'] + walltime = task_config[f'wtime_{task_name}'] + if self.cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_config.keys(): + walltime = task_config[f'wtime_{task_name}_gfs'] - cores = task_dict[f'npe_{task_name}'] - if self.cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_dict.keys(): - cores = task_dict[f'npe_{task_name}_gfs'] + cores = task_config[f'npe_{task_name}'] + if self.cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_config.keys(): + cores = task_config[f'npe_{task_name}_gfs'] - ppn = task_dict[f'npe_node_{task_name}'] - if self.cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_dict.keys(): - ppn = task_dict[f'npe_node_{task_name}_gfs'] + ppn = task_config[f'npe_node_{task_name}'] + if self.cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_config.keys(): + ppn = task_config[f'npe_node_{task_name}_gfs'] nodes = np.int(np.ceil(np.float(cores) / np.float(ppn))) - threads = task_dict[f'nth_{task_name}'] - if self.cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_dict.keys(): - threads = task_dict[f'nth_{task_name}_gfs'] + threads = task_config[f'nth_{task_name}'] + if self.cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_config.keys(): + threads = task_config[f'nth_{task_name}_gfs'] compute = f'{nodes}:ppn={ppn}:tpp={threads}' # TODO - remove dependence on compute - memory = task_dict.get(f'memory_{task_name}', None) + memory = task_config.get(f'memory_{task_name}', None) - native = '--export=NONE' if self._scheduler in ['slurm'] else None + native = '--export=NONE' if scheduler in ['slurm'] else None - queue = task_dict['QUEUE'] - if task_name in Tasks.SERVICE_TASKS and self._scheduler not in ['slurm']: - queue = task_dict['QUEUE_SERVICE'] + queue = task_config['QUEUE'] + if task_name in Tasks.SERVICE_TASKS and scheduler not in ['slurm']: + queue = task_config['QUEUE_SERVICE'] partition = None - if self._scheduler in ['slurm']: - partition = task_dict['QUEUE_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_dict['PARTITION_BATCH'] + if scheduler in ['slurm']: + partition = task_config['QUEUE_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config[ + 'PARTITION_BATCH'] task_resource = {'account': account, 'walltime': walltime, @@ -109,49 +125,20 @@ def get_resource(self, task_name): return task_resource def get_task(self, task_name, *args, **kwargs): - return getattr(self, task_name, *args, **kwargs)() - - @staticmethod - def create_wf_task(task_name, resources, - cdump='gdas', cycledef=None, envar=None, dependency=None, - metatask=None, varname=None, varval=None, vardict=None, - final=False): - - tasknamestr = f'{cdump}{task_name}' - metatask_dict = None - if metatask is not None: - tasknamestr = f'{tasknamestr}#{varname}#' - metatask_dict = {'metataskname': f'{cdump}{metatask}', - 'varname': f'{varname}', - 'varval': f'{varval}', - 'vardict': vardict} - - cycledefstr = cdump if cycledef is None else cycledef - - task_dict = {'taskname': f'{tasknamestr}', - 'cycledef': f'{cycledefstr}', - 'maxtries': '&MAXTRIES;', - 'command': f'&JOBS_DIR;/{task_name}.sh', - 'jobname': f'&PSLOT;_{tasknamestr}_@H', - 'resources': resources, - 'log': f'&ROTDIR;/logs/@Y@m@d@H/{tasknamestr}.log', - 'envars': envar, - 'dependency': dependency, - 'final': final} - - task = rocoto.create_task(task_dict) if metatask is None else rocoto.create_metatask(task_dict, metatask_dict) - - return ''.join(task) - - @staticmethod - def is_this_a_gdas_task(cdump, task_name): - if cdump != 'gdas': - raise TypeError(f'{task_name} must be part of the "gdas" cycle and not {cdump}') + """ + Given a task_name, call the method for that task + """ + try: + return getattr(self, task_name, *args, **kwargs)() + except AttributeError: + raise AttributeError(f'"{task_name}" is not a valid task.\n' + + 'Valid tasks are:\n' + + f'{", ".join(Tasks.VALID_TASKS)}') # Specific Tasks begin here def coupled_ic(self): - cpl_ic = self._job_configs['coupled_ic'] + cpl_ic = self._configs['coupled_ic'] deps = [] @@ -160,14 +147,14 @@ def coupled_ic(self): prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/&CDUMP;" for file in ['gfs_ctrl.nc'] + \ [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: data = f"{prefix}/{atm_res}/INPUT/{file}" dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) # Ocean ICs - if self.app.do_ocean: + if self.app_config.do_ocean: ocn_res = f"{self._base.get('OCNRES', '025'):03d}" prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/ocn" for res in ['res'] + [f'res_{res_index}' for res_index in range(1, 5)]: @@ -176,7 +163,7 @@ def coupled_ic(self): deps.append(rocoto.add_dependency(dep_dict)) # Ice ICs - if self.app.do_ice: + if self.app_config.do_ice: ice_res = f"{self._base.get('ICERES', '025'):03d}" ice_res_dec = f'{float(ice_res) / 100:.2f}' prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/ice" @@ -185,9 +172,9 @@ def coupled_ic(self): deps.append(rocoto.add_dependency(dep_dict)) # Wave ICs - if self.app.do_wave: + if self.app_config.do_wave: prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/wav" - for wave_grid in self._job_configs['waveinit']['waveGRD'].split(): + for wave_grid in self._configs['waveinit']['waveGRD'].split(): data = f"{prefix}/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}" dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -195,7 +182,7 @@ def coupled_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('coupled_ic') - task = self.create_wf_task('coupled_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('coupled_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -211,7 +198,7 @@ def getic(self): dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) resources = self.get_resource('getic') - task = self.create_wf_task('getic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('getic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -229,13 +216,13 @@ def init(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - if self.app.do_hpssarch: + if self.app_config.do_hpssarch: dep_dict = {'type': 'task', 'name': f'{self.cdump}getic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('init') - task = self.create_wf_task('init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -244,7 +231,7 @@ def prep(self): suffix = self._base["SUFFIX"] dump_suffix = self._base["DUMP_SUFFIX"] gfs_cyc = self._base["gfs_cyc"] - gfs_enkf = True if self.app.do_hybvar and 'gfs' in self.app.eupd_cdumps else False + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False deps = [] dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} @@ -262,15 +249,15 @@ def prep(self): cycledef = 'gdas' resources = self.get_resource('prep') - task = self.create_wf_task('prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task = create_wf_task('prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) return task def waveinit(self): resources = self.get_resource('waveinit') - task = self.create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars) + task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars) return task @@ -282,7 +269,7 @@ def waveprep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('waveprep') - task = self.create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -314,7 +301,7 @@ def aerosol_init(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aerosol_init') - task = self.create_wf_task('aerosol_init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('aerosol_init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -322,7 +309,7 @@ def anal(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_hybvar: + if self.app_config.do_hybvar: dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -330,7 +317,7 @@ def anal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('anal') - task = self.create_wf_task('anal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('anal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -339,13 +326,13 @@ def analcalc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_hybvar and self.cdump in ['gdas']: + if self.app_config.do_hybvar and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analcalc') - task = self.create_wf_task('analcalc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('analcalc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -357,7 +344,7 @@ def analdiag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('analdiag') - task = self.create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -369,7 +356,7 @@ def gldas(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gldas') - task = self.create_wf_task('gldas', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('gldas', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -379,9 +366,9 @@ def fcst(self): 'cycled': self._fcst_cycled} try: - task = fcst_map[self.app.mode] + task = fcst_map[self.app_config.mode] except KeyError: - raise NotImplementedError(f'{self.app.mode} is not a valid type.\n' + + raise NotImplementedError(f'{self.app_config.mode} is not a valid type.\n' + 'Currently supported forecast types are:\n' + f'{" | ".join(fcst_map.keys())}') @@ -398,18 +385,18 @@ def _fcst_free(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - if self.app.do_wave and self.cdump in self.app.wave_cdumps: + if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app.do_aero: # TODO - check dependency on cycleexist of previous cycle? + if self.app_config.do_aero: # TODO - check dependency on cycleexist of previous cycle? dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task = self.create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -420,24 +407,24 @@ def _fcst_cycled(self): dep = rocoto.add_dependency(dep_dict) dependencies = rocoto.create_dependency(dep=dep) - if self.app.do_gldas and self.cdump in ['gdas']: + if self.app_config.do_gldas and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.cdump}gldas'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app.do_wave and self.cdump in self.app.wave_cdumps: + if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task = self.create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task def post(self): add_anl_to_post = False - if self.app.mode in ['cycled']: + if self.app_config.mode in ['cycled']: add_anl_to_post = True return self._post_task('post', add_anl_to_post=add_anl_to_post) @@ -500,25 +487,25 @@ def _get_postgroups(cdump, config, add_anl=False): postenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_postgroups(self.cdump, self._job_configs[task_name], add_anl=add_anl_to_post) + varval1, varval2, varval3 = _get_postgroups(self.cdump, self._configs[task_name], add_anl=add_anl_to_post) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource(task_name) - task = self.create_wf_task(task_name, resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - metatask=task_name, varname=varname1, varval=varval1, vardict=vardict) + task = create_wf_task(task_name, resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, + metatask=task_name, varname=varname1, varval=varval1, vardict=vardict) return task def wavepostsbs(self): deps = [] - for wave_grid in self._job_configs['wavepostsbs']['waveGRD'].split(): + for wave_grid in self._configs['wavepostsbs']['waveGRD'].split(): data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/wave/rundata/{self.cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostsbs') - task = self.create_wf_task('wavepostsbs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostsbs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -529,7 +516,7 @@ def wavepostbndpnt(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpnt') - task = self.create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -541,7 +528,8 @@ def wavepostbndpntbll(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpntbll') - task = self.create_wf_task('wavepostbndpntbll', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostbndpntbll', resources, cdump=self.cdump, envar=self.envars, + dependency=dependencies) return task @@ -549,13 +537,13 @@ def wavepostpnt(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_wave_bnd: + if self.app_config.do_wave_bnd: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpntbll'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostpnt') - task = self.create_wf_task('wavepostpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -566,7 +554,7 @@ def wavegempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavegempak') - task = self.create_wf_task('wavegempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavegempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -579,7 +567,7 @@ def waveawipsbulls(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('waveawipsbulls') - task = self.create_wf_task('waveawipsbulls', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveawipsbulls', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -590,7 +578,8 @@ def waveawipsgridded(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('waeawipsgridded') - task = self.create_wf_task('waveawipsgridded', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveawipsgridded', resources, cdump=self.cdump, envar=self.envars, + dependency=dependencies) return task @@ -619,7 +608,7 @@ def _wafs_task(self, task_name): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource(task_name) - task = self.create_wf_task(task_name, resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task(task_name, resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -630,7 +619,7 @@ def wafsblending(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wafsblending') - task = self.create_wf_task('wafsblending', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wafsblending', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -641,7 +630,8 @@ def wafsblending0p25(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wafsblending0p25') - task = self.create_wf_task('wafsblending0p25', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wafsblending0p25', resources, cdump=self.cdump, envar=self.envars, + dependency=dependencies) return task @@ -652,7 +642,7 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('postsnd') - task = self.create_wf_task('postsnd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('postsnd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -707,12 +697,12 @@ def _get_awipsgroups(cdump, config): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._job_configs['awips']) + varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._configs['awips']) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = self.create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + task = create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) return task @@ -724,7 +714,7 @@ def gempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = self.create_wf_task('gempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('gempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -735,7 +725,7 @@ def vrfy(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('vrfy') - task = self.create_wf_task('vrfy', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('vrfy', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -756,31 +746,31 @@ def metp(self): varval1 = 'g2g1 g2o1 pcp1' resources = self.get_resource('metp') - task = self.create_wf_task('metp', resources, cdump=self.cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) + task = create_wf_task('metp', resources, cdump=self.cdump, envar=metpenvars, dependency=dependencies, + metatask='metp', varname=varname1, varval=varval1) return task def arch(self): deps = [] - if self.app.do_vrfy: + if self.app_config.do_vrfy: dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_metp and self.cdump in ['gfs']: + if self.app_config.do_metp and self.cdump in ['gfs']: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_wave: + if self.app_config.do_wave: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app.do_wave_bnd and self.cdump in ['gfs']: + if self.app_config.do_wave_bnd and self.cdump in ['gfs']: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('arch') - task = self.create_wf_task('arch', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('arch', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -794,8 +784,8 @@ def eobs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('eobs') - task = self.create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) return task @@ -808,12 +798,12 @@ def eomg(self): eomgenvars = self.envars eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['eobs']['NMEM_EOMGGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['eobs']['NMEM_EOMGGRP']) resources = self.get_resource('eomg') - task = self.create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + task = create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, + metatask='eomn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) return task @@ -824,14 +814,14 @@ def ediag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ediag') - task = self.create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) return task def eupd(self): deps = [] - if self.app.lobsdiag_forenkf: + if self.app_config.lobsdiag_forenkf: dep_dict = {'type': 'task', 'name': f'{self.cdump}ediag'} else: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}eomn'} @@ -839,21 +829,21 @@ def eupd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('eupd') - task = self.create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) return task def ecen(self): - self.is_this_a_gdas_task(self.cdump, 'ecen') + self._is_this_a_gdas_task(self.cdump, 'ecen') def _get_ecengroups(): if self._base.get('DOIAU_ENKF', False): fhrs = list(self._base.get('IAUFHRS', '6').split(',')) - necengrp = self._job_configs['ecen']['NECENGRP'] + necengrp = self._configs['ecen']['NECENGRP'] ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) fhrs = [f'{int(fhr):03d}' for fhr in fhrs] @@ -871,7 +861,7 @@ def _get_ecengroups(): return grp, dep, lst - eupd_cdump = 'gdas' if 'gdas' in self.app.eupd_cdumps else 'gfs' + eupd_cdump = 'gdas' if 'gdas' in self.app_config.eupd_cdumps else 'gfs' deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} @@ -891,16 +881,16 @@ def _get_ecengroups(): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('ecen') - task = self.create_wf_task('ecen', resources, cdump=self.cdump, envar=ecenenvars, dependency=dependencies, - metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict, - cycledef=self.hyb_cycledef) + task = create_wf_task('ecen', resources, cdump=self.cdump, envar=ecenenvars, dependency=dependencies, + metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict, + cycledef=self.hyb_cycledef) return task def esfc(self): - self.is_this_a_gdas_task(self.cdump, 'esfc') + self._is_this_a_gdas_task(self.cdump, 'esfc') - eupd_cdump = 'gdas' if 'gdas' in self.app.eupd_cdumps else 'gfs' + eupd_cdump = 'gdas' if 'gdas' in self.app_config.eupd_cdumps else 'gfs' deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} @@ -909,16 +899,15 @@ def esfc(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('esfc') - task = self.create_wf_task('esfc', resources, cdump='gdas', envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('esfc', resources, cdump='gdas', envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) return task def efcs(self): - self.is_this_a_gdas_task(self.cdump, 'efcs') + self._is_this_a_gdas_task(self.cdump, 'efcs') deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.cdump}ecmn'} @@ -930,18 +919,18 @@ def efcs(self): efcsenvars = self.envars efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['efcs']['NMEM_EFCSGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['efcs']['NMEM_EFCSGRP']) resources = self.get_resource('efcs') - task = self.create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, - metatask='efmn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + task = create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, + metatask='efmn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) return task def echgres(self): - self.is_this_a_gdas_task(self.cdump, 'echgres') + self._is_this_a_gdas_task(self.cdump, 'echgres') deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} @@ -951,14 +940,14 @@ def echgres(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('echgres') - task = self.create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=self.hyb_cycledef) return task def epos(self): - self.is_this_a_gdas_task(self.cdump, 'epos') + self._is_this_a_gdas_task(self.cdump, 'epos') def _get_eposgroups(epos): fhmin = epos['FHMIN_ENKF'] @@ -991,19 +980,19 @@ def _get_eposgroups(epos): eposenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_eposgroups(self._job_configs['epos']) + varval1, varval2, varval3 = _get_eposgroups(self._configs['epos']) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('epos') - task = self.create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, - cycledef=self.hyb_cycledef) + task = create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, + metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, + cycledef=self.hyb_cycledef) return task def earc(self): - self.is_this_a_gdas_task(self.cdump, 'earc') + self._is_this_a_gdas_task(self.cdump, 'earc') deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.cdump}epmn'} @@ -1013,11 +1002,57 @@ def earc(self): earcenvars = self.envars earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._job_configs['earc']['NMEM_EARCGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['earc']['NMEM_EARCGRP']) resources = self.get_resource('earc') - task = self.create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + task = create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, + metatask='eamn', varname='grp', varval=groups, + cycledef=self.hyb_cycledef) return task + + +def create_wf_task(task_name, resources, + cdump='gdas', cycledef=None, envar=None, dependency=None, + metatask=None, varname=None, varval=None, vardict=None, + final=False): + tasknamestr = f'{cdump}{task_name}' + metatask_dict = None + if metatask is not None: + tasknamestr = f'{tasknamestr}#{varname}#' + metatask_dict = {'metataskname': f'{cdump}{metatask}', + 'varname': f'{varname}', + 'varval': f'{varval}', + 'vardict': vardict} + + cycledefstr = cdump if cycledef is None else cycledef + + task_dict = {'taskname': f'{tasknamestr}', + 'cycledef': f'{cycledefstr}', + 'maxtries': '&MAXTRIES;', + 'command': f'&JOBS_DIR;/{task_name}.sh', + 'jobname': f'&PSLOT;_{tasknamestr}_@H', + 'resources': resources, + 'log': f'&ROTDIR;/logs/@Y@m@d@H/{tasknamestr}.log', + 'envars': envar, + 'dependency': dependency, + 'final': final} + + task = rocoto.create_task(task_dict) if metatask is None else rocoto.create_metatask(task_dict, metatask_dict) + + return ''.join(task) + + +def get_wf_tasks(app_config: AppConfig) -> List: + """ + Take application configuration to return a list of all tasks for that application + """ + + tasks = [] + # Loop over all keys of cycles (CDUMP) + for cdump, cdump_tasks in app_config.task_names.items(): + task_obj = Tasks(app_config, cdump) # create Task object based on cdump + for task_name in cdump_tasks: + tasks.append(task_obj.get_task(task_name)) + + return tasks diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index bbe7c96cf7f..2a93544fef1 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -75,6 +75,7 @@ def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: gfs_cyc = base['gfs_cyc'] sdate = base['SDATE'] edate = base['EDATE'] + base_out['INTERVAL'] = '06:00:00' # Cycled interval is 6 hours interval_gfs = get_gfs_interval(gfs_cyc) diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index 3eaa530cdbb..f3a4ab2d959 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -2,24 +2,31 @@ from datetime import datetime from collections import OrderedDict -from hosts import Host import rocoto -from applications import Application +from applications import AppConfig +from workflow_tasks import get_wf_tasks class RocotoXML: - def __init__(self, app: Application) -> None: + def __init__(self, app_config: AppConfig) -> None: - self._app = app + # Initialize variables + self.xml = None - self._base = self._app.job_configs['base'] + self._app_config = app_config - self.preamble = self._get_preamble + self._base = self._app_config.configs['base'] + + self.preamble = self._get_preamble() self.definitions = self._get_definitions() - self.header = self._get_workflow_header + self.header = self._get_workflow_header() self.cycledefs = self._get_cycledefs() - self.footer = self._get_workflow_footer + task_list = get_wf_tasks(app_config) + self.tasks = ''.join(task_list) + self.footer = self._get_workflow_footer() + + self.xml = self.assemble_xml() @staticmethod def _get_preamble(): @@ -36,7 +43,8 @@ def _get_preamble(): '', '\tNOTES:', f'\t\tThis workflow was automatically generated at {datetime.now()}', - '\t-->'] + '\t-->', + ''] return '\n'.join(strings) @@ -45,25 +53,21 @@ def _get_definitions(self) -> str: Create entities related to the experiment """ - host = Host() - - scheduler = host.scheduler - entity = OrderedDict() entity['PSLOT'] = self._base['PSLOT'] entity['SDATE'] = self._base['SDATE'].strftime('%Y%m%d%H%M') entity['EDATE'] = self._base['EDATE'].strftime('%Y%m%d%H%M') - if self._app.mode in ['cycled']: - entity['INTERVAL'] = '06:00:00' + if self._app_config.mode in ['cycled']: + entity['INTERVAL'] = self._base.get('INTERVAL', '06:00:00') if self._base['gfs_cyc'] in [1, 2, 4]: entity['SDATE_GFS'] = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M') entity['EDATE_GFS'] = self._base['EDATE_GFS'].strftime('%Y%m%d%H%M') entity['INTERVAL_GFS'] = self._base['INTERVAL_GFS'] entity['DMPDIR'] = self._base['DMPDIR'] - elif self._app.mode in ['forecast-only']: + elif self._app_config.mode in ['forecast-only']: entity['INTERVAL'] = self._base.get('INTERVAL_GFS', '24:00:00') entity['CDUMP'] = self._base['CDUMP'] entity['CASE'] = self._base['CASE'] # TODO - is this really used in the XML? @@ -74,18 +78,9 @@ def _get_definitions(self) -> str: entity['RUN_ENVIR'] = self._base.get('RUN_ENVIR', 'emc') entity['HOMEgfs'] = self._base['HOMEgfs'] entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['ACCOUNT'] = self._base['ACCOUNT'] - entity['QUEUE'] = self._base['QUEUE'] - entity['QUEUE_SERVICE'] = self._base['QUEUE_SERVICE'] - entity['SCHEDULER'] = scheduler - if scheduler in ['slurm']: - entity['PARTITION_BATCH'] = self._base['PARTITION_BATCH'] - entity['PARTITION_SERVICE'] = self._base['PARTITION_SERVICE'] entity['ARCHIVE_TO_HPSS'] = self._base.get('HPSSARCH', 'NO') - entity['CYCLETHROTTLE'] = self._base.get('ROCOTO_CYCLETHROTTLE', 3) - entity['TASKTHROTTLE'] = self._base.get('ROCOTO_TASKTHROTTLE', 25) + entity['MAXTRIES'] = self._base.get('ROCOTO_MAXTRIES', 2) - entity['VERBOSITY'] = self._base.get('ROCOTO_VERBOSITY', 10) # Put them all in an XML key-value syntax strings = [] @@ -94,20 +89,25 @@ def _get_definitions(self) -> str: return '\n'.join(strings) - @staticmethod - def _get_workflow_header(): + def _get_workflow_header(self): """ Create the workflow header block """ + scheduler = self._app_config.scheduler + cyclethrottle = self._base.get('ROCOTO_CYCLETHROTTLE', 3) + taskthrottle = self._base.get('ROCOTO_TASKTHROTTLE', 25) + verbosity = self._base.get('ROCOTO_VERBOSITY', 10) + strings = ['', ']>', '', - '', + f'', '', - '\t&EXPDIR;/logs/@Y@m@d@H.log', + f'\t&EXPDIR;/logs/@Y@m@d@H.log', '', - '\t'] + '\t', + ''] return '\n'.join(strings) @@ -117,23 +117,26 @@ def _get_cycledefs(self): 'forecast-only': self._get_cycledefs_forecast_only} try: - cycledef_map[self._app.mode]() + cycledefs = cycledef_map[self._app_config.mode]() except KeyError: - raise TypeError(f'{self._app.mode} is not a valid application.' + - 'Current plot types supported are:\n' + - f'{" | ".join(cycledef_map.keys())}"') + raise KeyError(f'{self._app_config.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(cycledef_map.keys())}') + + return cycledefs def _get_cycledefs_cycled(self): - strings = ['\t&SDATE; &SDATE; 06:00:00', - '\t&SDATE; &EDATE; 06:00:00', - '\t&SDATE; &EDATE; 06:00:00'] - if self._app.gfs_cyc != 0: + strings = ['\t&SDATE; &SDATE; &INTERVAL;', + '\t&SDATE; &EDATE; &INTERVAL;', + '\t&SDATE; &EDATE; &INTERVAL;'] + if self._app_config.gfs_cyc != 0: strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;') + strings.append('') + strings.append('') return '\n'.join(strings) - @staticmethod - def _get_cycledefs_forecast_only(): + def _get_cycledefs_forecast_only(self): strings = f'\t&SDATE; &EDATE; &INTERVAL;' return strings @@ -146,6 +149,18 @@ def _get_workflow_footer(): return '\n\n' + def assemble_xml(self) -> str: + + strings = [self.preamble, + self.definitions, + self.header, + self.cycledefs, + self.tasks, + self.footer] + + return ''.join(strings) + + def write(self, xml_file: str) -> None: -def create_xml(): - return None + with open(xml_file, 'w') as fh: + fh.write(self.xml) From 98ca3f1f82b86ba723e5e6addb85e09513345046 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 14 Jun 2022 23:59:24 -0400 Subject: [PATCH 06/41] Restore setup_workflow.py and setup_workflow_fcstonly.py to `develop` versions. Add tests for configuration and hosts. Add a dummy script for testing setup_workflow.py --- ush/rocoto/setup_workflow.py | 1050 ++++++++++++++++++++++++- ush/rocoto/setup_workflow_fcstonly.py | 819 +++++++++++++++++-- ush/rocoto/test_configuration.py | 29 + ush/rocoto/test_hosts.py | 17 + ush/rocoto/test_setup_workflow.py | 18 + 5 files changed, 1864 insertions(+), 69 deletions(-) create mode 100644 ush/rocoto/test_configuration.py create mode 100644 ush/rocoto/test_hosts.py create mode 100644 ush/rocoto/test_setup_workflow.py diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 9466d49d094..43ee7ab8caa 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -68,7 +68,8 @@ def main(): dict_configs = wfu.source_configs(configs, steps) # Check and set gfs_cyc specific variables - dict_configs['base'] = wfu.get_gfs_cyc_dates(dict_configs['base']) + if dict_configs['base']['gfs_cyc'] != 0: + dict_configs['base'] = get_gfs_cyc_dates(dict_configs['base']) # First create workflow XML create_xml(dict_configs) @@ -79,6 +80,151 @@ def main(): return +def get_gfs_cyc_dates(base): + ''' + Generate GFS dates from experiment dates and gfs_cyc choice + ''' + + base_out = base.copy() + + gfs_cyc = base['gfs_cyc'] + sdate = base['SDATE'] + edate = base['EDATE'] + + interval_gfs = wfu.get_gfs_interval(gfs_cyc) + + # Set GFS cycling dates + hrdet = 0 + if gfs_cyc == 1: + hrinc = 24 - sdate.hour + hrdet = edate.hour + elif gfs_cyc == 2: + if sdate.hour in [0, 12]: + hrinc = 12 + elif sdate.hour in [6, 18]: + hrinc = 6 + if edate.hour in [6, 18]: + hrdet = 6 + elif gfs_cyc == 4: + hrinc = 6 + sdate_gfs = sdate + timedelta(hours=hrinc) + edate_gfs = edate - timedelta(hours=hrdet) + if sdate_gfs > edate: + print('W A R N I N G!') + print('Starting date for GFS cycles is after Ending date of experiment') + print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') + print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') + gfs_cyc = 0 + + base_out['gfs_cyc'] = gfs_cyc + base_out['SDATE_GFS'] = sdate_gfs + base_out['EDATE_GFS'] = edate_gfs + base_out['INTERVAL_GFS'] = interval_gfs + + fhmax_gfs = {} + for hh in ['00', '06', '12', '18']: + fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', 'FHMAX_GFS_00') + base_out['FHMAX_GFS'] = fhmax_gfs + + return base_out + + +def get_preamble(): + ''' + Generate preamble for XML + ''' + + strings = [] + + strings.append('\n') + strings.append('\n') + + return ''.join(strings) + + +def get_definitions(base): + ''' + Create entities related to the experiment + ''' + + machine = base.get('machine', wfu.detectMachine()) + scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() + + strings = [] + + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + + if base['gfs_cyc'] != 0: + strings.append(get_gfs_dates(base)) + strings.append('\n') + + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + if scheduler in ['slurm']: + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'\t\n') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\n') + + return ''.join(strings) + + +def get_gfs_dates(base): + ''' + Generate GFS dates entities + ''' + + strings = [] + + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + + return ''.join(strings) + + def get_gdasgfs_resources(dict_configs, cdump='gdas'): ''' Create GDAS or GFS resource entities @@ -97,7 +243,7 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() reservation = base.get('RESERVATION', 'NONE').upper() - #task_tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] + #tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] tasks = ['prep', 'anal', 'analcalc'] if cdump in ['gdas']: @@ -105,13 +251,13 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: - #task_tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] + #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] tasks += ['fcst', 'post', 'vrfy', 'arch'] if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - #task_tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] + #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: tasks += ['postsnd'] @@ -170,7 +316,7 @@ def get_hyb_resources(dict_configs): dict_resources = OrderedDict() - # These task_tasks can be run in either or both cycles + # These tasks can be run in either or both cycles if lobsdiag_forenkf in ['.T.', '.TRUE.']: tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] else: @@ -206,7 +352,7 @@ def get_hyb_resources(dict_configs): dict_resources[f'{cdump}{task}'] = ''.join(strings) - # These task_tasks are always run as part of the GDAS cycle + # These tasks are always run as part of the GDAS cycle cdump = 'gdas' tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] for task in tasks2: @@ -235,6 +381,766 @@ def get_hyb_resources(dict_configs): return dict_resources + +def get_gdasgfs_tasks(dict_configs, cdump='gdas'): + ''' + Create GDAS or GFS tasks + ''' + + envars = [] + if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: + envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) + envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) + envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) + envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) + envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) + envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) + envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) + envars.append(rocoto.create_envar(name='cyc', value='@H')) + + base = dict_configs['base'] + gfs_cyc = base.get('gfs_cyc', 0) + gldas_cyc = base.get('gldas_cyc', 0) + dohybvar = base.get('DOHYBVAR', 'NO').upper() + eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() + do_gempak = base.get('DO_GEMPAK', 'NO').upper() + do_awips = base.get('DO_AWIPS', 'NO').upper() + do_wafs = base.get('WAFSF', 'NO').upper() + do_metp = base.get('DO_METP', 'NO').upper() + do_gldas = base.get('DO_GLDAS', 'NO').upper() + do_wave = base.get('DO_WAVE', 'NO').upper() + if do_wave in ['YES']: + do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() + do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() + dumpsuffix = base.get('DUMP_SUFFIX', '') + gridsuffix = base.get('SUFFIX', '') + + dict_tasks = OrderedDict() + + # prep + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{gridsuffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&DMPDIR;/{cdump}{dumpsuffix}.@Y@m@d/@H/{cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False + + if gfs_enkf and cdump in ['gfs']: + if gfs_cyc == 4: + task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) + else: + task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') + + else: + task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}prep'] = task + + # wave tasks in gdas or gfs or both + if do_wave_cdump in ['BOTH']: + cdumps = ['gfs', 'gdas'] + elif do_wave_cdump in ['GFS']: + cdumps = ['gfs'] + elif do_wave_cdump in ['GDAS']: + cdumps = ['gdas'] + + # waveinit + if do_wave in ['Y', 'YES'] and cdump in cdumps: + deps = [] + dep_dict = {'type': 'task', 'name': '{cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks['{cdump}waveinit'] = task + + # waveprep + if do_wave in ['Y', 'YES'] and cdump in cdumps: + deps = [] + dep_dict = {'type': 'task', 'name': '{cdump}waveinit'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks['{cdump}waveprep'] = task + + # anal + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if dohybvar in ['y', 'Y', 'yes', 'YES']: + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}anal'] = task + + # analcalc + deps1 = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + dep_dict = {'type': 'data', 'data': data} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + deps.append(rocoto.add_dependency(dep_dict)) + if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': + dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}analcalc'] = task + + # analdiag + if cdump in ['gdas']: + deps1 = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + dep_dict = {'type': 'data', 'data': data} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) + + task = wfu.create_wf_task('analdiag', cdump=cdump, envar=envars, dependency=dependencies2) + + dict_tasks[f'{cdump}analdiag'] = task + + # gldas + if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: + deps1 = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + dep_dict = {'type': 'data', 'data': data} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) + + task = wfu.create_wf_task('gldas', cdump=cdump, envar=envars, dependency=dependencies2) + dict_tasks[f'{cdump}gldas'] = task + + # fcst + deps1 = [] + #data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + #dep_dict = {'type': 'data', 'data': data} + #deps1.append(rocoto.add_dependency(dep_dict)) + if cdump in ['gdas']: + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + deps1.append(rocoto.add_dependency(dep_dict)) + if do_gldas in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}gldas'} + deps1.append(rocoto.add_dependency(dep_dict)) + else: + dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} + deps1.append(rocoto.add_dependency(dep_dict)) + elif cdump in ['gfs']: + dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + + if do_wave in ['Y', 'YES'] and cdump in cdumps: + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) + task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies2) + else: + task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies1) + + dict_tasks[f'{cdump}fcst'] = task + + # post + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, + metatask='post', varname=varname1, varval=varval1, vardict=vardict) + + dict_tasks[f'{cdump}post'] = task + + # wavepostsbs + if do_wave in ['Y', 'YES'] and cdump in cdumps: + deps = [] + for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}wavepostsbs'] = task + + # wavepostbndpnt + if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}wavepostbndpnt'] = task + + # wavepostbndpntbll + if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}wavepostbndpntbll'] = task + + # wavepostpnt + if do_wave in ['Y', 'YES'] and cdump in ['gdas', 'gfs']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave_bnd in ['YES'] and cdump in ['gfs']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}wavepostpnt'] = task + + # wavegempak + if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES'] and cdump in ['gfs']: + deps = [] + dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}wavegempak'] = task + + # waveawipsgridded + if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: + deps = [] + dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}waveawipsgridded'] = task + + # waveawipsbulls + if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: + deps = [] + dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) + dict_tasks[f'{cdump}waveawipsbulls'] = task + + # wavestat + #if do_wave in ['Y', 'YES'] and cdump in cdumps: + # deps = [] + # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} + # deps.append(rocoto.add_dependency(dep_dict)) + # dependencies = rocoto.create_dependency(dep=deps) + # task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies) + # dict_tasks['%swavestat' % cdump] = task + + # vrfy + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}vrfy'] = task + + # metp + if cdump in ['gfs'] and do_metp in ['Y', 'YES']: + deps = [] + dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE_GFS;') + metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') + metpenvars = envars + [sdate_gfs] + [metpcase] + varname1 = 'metpcase' + varval1 = 'g2g1 g2o1 pcp1' + task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, + metatask='metp', varname=varname1, varval=varval1) + dict_tasks[f'{cdump}metp'] = task + + #postsnd + if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}postsnd'] = task + + # awips + if cdump in ['gfs'] and do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + + dict_tasks[f'{cdump}awips'] = task + + # gempak + if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}gempak'] = task + + # wafs + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafs'] = task + + # wafsgcip + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafsgcip'] = task + + # wafsgrib2 + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafsgrib2'] = task + + # wafsgrib20p25 + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafsgrib20p25'] = task + + # wafsblending + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafsblending'] = task + + # wafsblending0p25 + if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}wafsblending0p25'] = task + + # arch + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + if cdump in ['gfs'] and do_metp in ['Y', 'YES']: + dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave_bnd in ['YES'] and cdump in ['gfs']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}arch'] = task + + return dict_tasks + + +def get_hyb_tasks(dict_configs, cycledef='enkf'): + ''' + Create Hybrid tasks + ''' + + # Determine groups based on ensemble size and grouping + base = dict_configs['base'] + nens = base['NMEM_ENKF'] + lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() + eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() + + eobs = dict_configs['eobs'] + nens_eomg = eobs['NMEM_EOMGGRP'] + neomg_grps = nens / nens_eomg + EOMGGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(neomg_grps) + 1)]) + + efcs = dict_configs['efcs'] + nens_efcs = efcs['NMEM_EFCSGRP'] + nefcs_grps = nens / nens_efcs + EFCSGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(nefcs_grps) + 1)]) + + earc = dict_configs['earc'] + nens_earc = earc['NMEM_EARCGRP'] + nearc_grps = nens / nens_earc + EARCGROUPS = ' '.join([f'{x:02d}' for x in range(0, int(nearc_grps) + 1)]) + + envars = [] + if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: + envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) + envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) + envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) + envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) + envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) + #envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) + envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) + envars.append(rocoto.create_envar(name='cyc', value='@H')) + + ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#') + + dict_tasks = OrderedDict() + + if eupd_cyc in ['BOTH']: + cdumps = ['gfs', 'gdas'] + elif eupd_cyc in ['GFS']: + cdumps = ['gfs'] + elif eupd_cyc in ['GDAS']: + cdumps = ['gdas'] + + for cdump in cdumps: + + envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') + envars1 = envars + [envar_cdump] + + # eobs + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}eobs'] = task + + # eomn, eomg + if lobsdiag_forenkf in ['.F.', '.FALSE.']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + eomgenvars= envars1 + [ensgrp] + task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, + metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) + + dict_tasks[f'{cdump}eomn'] = task + + # ediag + else: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('ediag', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}ediag'] = task + + # eupd + deps = [] + if lobsdiag_forenkf in ['.F.', '.FALSE.']: + dep_dict = {'type': 'metatask', 'name': f'{cdump}eomn'} + else: + dep_dict = {'type': 'task', 'name': f'{cdump}ediag'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}eupd'] = task + + # All hybrid tasks beyond this point are always executed in the GDAS cycle + cdump = 'gdas' + envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') + envars1 = envars + [envar_cdump] + cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas' + + # ecmn, ecen + deps1 = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' + dep_dict = {'type': 'data', 'data': data} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) + + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ecenenvars = envars1 + [fhrgrp] + [fhrlst] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_ecengroups(dict_configs, dict_configs['ecen'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('ecen', cdump=cdump, envar=ecenenvars, dependency=dependencies2, + metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict) + + dict_tasks[f'{cdump}ecmn'] = task + + # esfc + deps1 = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' + dep_dict = {'type': 'data', 'data': data} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) + task = wfu.create_wf_task('esfc', cdump=cdump, envar=envars1, dependency=dependencies2, cycledef=cycledef) + + dict_tasks[f'{cdump}esfc'] = task + + # efmn, efcs + deps1 = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}ecmn'} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}esfc'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) + + deps2 = [] + deps2 = dependencies1 + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='or', dep=deps2) + + efcsenvars = envars1 + [ensgrp] + task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies2, + metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef) + + dict_tasks[f'{cdump}efmn'] = task + + # echgres + deps1 = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) + task = wfu.create_wf_task('echgres', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) + + dict_tasks[f'{cdump}echgres'] = task + + # epmn, epos + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + eposenvars = envars1 + [fhrgrp] + [fhrlst] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies, + metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) + + dict_tasks[f'{cdump}epmn'] = task + + # eamn, earc + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}epmn'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + earcenvars = envars1 + [ensgrp] + task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies, + metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef) + + dict_tasks[f'{cdump}eamn'] = task + + return dict_tasks + + def get_workflow_header(base): ''' Create the workflow header block @@ -261,6 +1167,132 @@ def get_workflow_header(base): return ''.join(strings) +def get_workflow_footer(): + ''' + Generate workflow footer + ''' + + strings = [] + strings.append('\n\n') + + return ''.join(strings) + + +def get_postgroups(post, cdump='gdas'): + + fhmin = post['FHMIN'] + fhmax = post['FHMAX'] + fhout = post['FHOUT'] + + # Get a list of all forecast hours + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax+fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max([post['FHMAX_GFS_00'],post['FHMAX_GFS_06'],post['FHMAX_GFS_12'],post['FHMAX_GFS_18']]) + fhout = post['FHOUT_GFS'] + fhmax_hf = post['FHMAX_HF_GFS'] + fhout_hf = post['FHOUT_HF_GFS'] + fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) + fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout)) + + npostgrp = post['NPOSTGRP'] + ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) + + fhrs = [f'f{f:03d}' for f in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [f.tolist() for f in fhrs] + + fhrgrp = ' '.join(['anl'] + [f'_{f[0]}-{f[-1]}' for f in fhrs]) + fhrdep = ' '.join(['anl'] + [f[-1] for f in fhrs]) + fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) + + return fhrgrp, fhrdep, fhrlst + +def get_awipsgroups(awips, cdump='gdas'): + + fhmin = awips['FHMIN'] + fhmax = awips['FHMAX'] + fhout = awips['FHOUT'] + + # Get a list of all forecast hours + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax+fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max([awips['FHMAX_GFS_00'],awips['FHMAX_GFS_06'],awips['FHMAX_GFS_12'],awips['FHMAX_GFS_18']]) + fhout = awips['FHOUT_GFS'] + fhmax_hf = awips['FHMAX_HF_GFS'] + fhout_hf = awips['FHOUT_HF_GFS'] + if fhmax > 240: + fhmax = 240 + if fhmax_hf > 240: + fhmax_hf = 240 + fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) + fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) + + nawipsgrp = awips['NAWIPSGRP'] + ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) + + fhrs = [f'f{f:03d}' for f in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [f.tolist() for f in fhrs] + + fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) + fhrdep = ' '.join([f[-1] for f in fhrs]) + fhrlst = ' '.join(['_'.join(f) for f in fhrs]) + + return fhrgrp, fhrdep, fhrlst + +def get_ecengroups(dict_configs, ecen, cdump='gdas'): + + base = dict_configs['base'] + + if base.get('DOIAU_ENKF', 'NO') == 'YES' : + fhrs = list(base.get('IAUFHRS','6').split(',')) + ifhrs = [f'f00{ff}' for ff in fhrs] + ifhrs0 = ifhrs[0] + nfhrs = len(fhrs) + + ifhrs = [f'f00{ff}' for ff in fhrs] + ifhrs0 = ifhrs[0] + nfhrs = len(fhrs) + + necengrp = ecen['NECENGRP'] + ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) + + ifhrs = np.array_split(ifhrs, ngrps) + + fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) + fhrdep = ' '.join([f[-1] for f in ifhrs]) + fhrlst = ' '.join(['_'.join(f) for f in ifhrs]) + + else: + fhrgrp='000' + fhrdep='f006' + fhrlst='f006' + + return fhrgrp, fhrdep, fhrlst + +def get_eposgroups(epos, cdump='gdas'): + + fhmin = epos['FHMIN_ENKF'] + fhmax = epos['FHMAX_ENKF'] + fhout = epos['FHOUT_ENKF'] + fhrs = range(fhmin, fhmax+fhout, fhout) + + neposgrp = epos['NEPOSGRP'] + ngrps = neposgrp if len(fhrs) > neposgrp else len(fhrs) + + fhrs = [f'f{f:03d}' for f in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [f.tolist() for f in fhrs] + + fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) + fhrdep = ' '.join([f[-1] for f in fhrs]) + fhrlst = ' '.join(['_'.join(f) for f in fhrs]) + + return fhrgrp, fhrdep, fhrlst + + def dict_to_strings(dict_in): strings = [] @@ -301,7 +1333,7 @@ def create_xml(dict_configs): dict_hyb_resources = get_hyb_resources(dict_configs) dict_hyb_tasks = get_hyb_tasks(dict_configs) - # Removes &MEMORY_JOB_DUMP post mortem from hyb task_tasks + # Removes &MEMORY_JOB_DUMP post mortem from hyb tasks hyp_tasks = {'gdaseobs':'gdaseobs', 'gdasediag':'gdasediag', 'gdaseomg':'gdaseomn', @@ -333,7 +1365,7 @@ def create_xml(dict_configs): dict_gfs_resources = get_gdasgfs_resources(dict_configs, cdump='gfs') dict_gfs_tasks = get_gdasgfs_tasks(dict_configs, cdump='gfs') - # Removes &MEMORY_JOB_DUMP post mortem from gdas task_tasks + # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks for each_task, each_resource_string in dict_gdas_resources.items(): if each_task not in dict_gdas_tasks: continue @@ -344,7 +1376,7 @@ def create_xml(dict_configs): temp_task_string.append(each_line) dict_gdas_tasks[each_task] = ''.join(temp_task_string) - # Removes &MEMORY_JOB_DUMP post mortem from gfs task_tasks + # Removes &MEMORY_JOB_DUMP post mortem from gfs tasks for each_task, each_resource_string in dict_gfs_resources.items(): if each_task not in dict_gfs_tasks: continue diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index 894c2e637bc..e0fd8021231 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -1,8 +1,12 @@ #!/usr/bin/env python3 -""" +''' PROGRAM: - Create the ROCOTO workflow for any experiment given the configuration of the GFS parallel + Create the ROCOTO workflow for a forecast only experiment given the configuration of the GFS parallel + + AUTHOR: + Rahul.Mahajan + rahul.mahajan@noaa.gov FILE DEPENDENCIES: 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. @@ -12,72 +16,37 @@ 1. PSLOT.xml: XML workflow 2. PSLOT.crontab: crontab for ROCOTO run command -""" +''' import os import sys import re - +import numpy as np +from datetime import datetime from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from hosts import Host +import rocoto import workflow_utils as wfu -from configuration import Configuration - - -@property -def input_args(): - """ - Method to collect user arguments for `setup_workflow.py` - """ - - here = os.path.dirname(__file__) - top = os.path.abspath(os.path.join( - os.path.abspath(here), '../..')) - - description = """ - Sources configuration files based on application and\n - creates "PSLOT.xml" for use with Rocoto.\n - """ - - parser = ArgumentParser(description=description, - formatter_class=ArgumentDefaultsHelpFormatter) - - parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, - required=False, default=os.environ['PWD']) - - # Set up sub-parsers for various modes of experimentation - subparser = parser.add_subparsers(dest='mode') - cycled = subparser.add_parser( - 'cycled', help='arguments for cycled mode') - forecasts = subparser.add_parser( - 'forecast-only', help='arguments for forecast-only mode') - - forecasts.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', - required=False) - - args = parser.parse_args() - - return args +taskplan = ['getic', 'init', 'coupled_ic', 'aerosol_init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch', 'ocnpost'] def main(): + parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument('--expdir',help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) + parser.add_argument('--cdump',help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) - user_inputs = input_args - - cfg = Configuration(user_inputs.expdir) - + args = parser.parse_args() - sys.exit(0) + configs = wfu.get_configs(args.expdir) - _base = cfg.parse_config('config.base') + _base = wfu.config_parser([wfu.find_config('config.base', configs)]) - if not os.path.samefile(args.expdir, _base['EXPDIR']): + if not os.path.samefile(args.expdir,_base['EXPDIR']): print('MISMATCH in experiment directories!') print(f'''config.base: EXPDIR = {repr(_base['EXPDIR'])}''') print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - dict_configs = wfu.source_configs(cfg, taskplan) + dict_configs = wfu.source_configs(configs, taskplan) dict_configs['base']['CDUMP'] = args.cdump @@ -90,20 +59,115 @@ def main(): return -def get_resources(dict_configs, task_plan, cdump='gdas'): +def get_preamble(): ''' - Create resource entities + Generate preamble for XML ''' - resources = wfu.get_taskplan_resources(dict_configs, task_plan, cdump=cdump) + strings = [] + + strings.append('\n') + strings.append('\n') + + return ''.join(strings) + + +def get_definitions(base): + ''' + Create entities related to the experiment + ''' + + machine = base.get('machine', wfu.detectMachine()) + scheduler = wfu.get_scheduler(machine) + hpssarch = base.get('HPSSARCH', 'NO').upper() + + strings = [] + + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + if base['INTERVAL'] is None: + print('cycle INTERVAL cannot be None') + sys.exit(1) + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + if scheduler in ['slurm']: + strings.append(f'''\t\n''') + strings.append(f'''\t\n''') + strings.append(f'\t\n') + strings.append('\n') + strings.append('\t\n') + strings.append(f'''\t\n''') + strings.append('\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\t\n') + strings.append('\n') + + return ''.join(strings) + - scheduler = Host.get_scheduler +def get_resources(dict_configs, cdump='gdas'): + ''' + Create resource entities + ''' strings = [] + strings.append('\t\n') strings.append('\n') - for task in task_plan: + base = dict_configs['base'] + machine = base.get('machine', wfu.detectMachine()) + reservation = base.get('RESERVATION', 'NONE').upper() + scheduler = wfu.get_scheduler(machine) + + do_wave = base.get('DO_WAVE', 'NO').upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() + do_gempak = base.get('DO_GEMPAK', 'NO').upper() + do_awips = base.get('DO_AWIPS', 'NO').upper() + do_metp = base.get('DO_METP', 'NO').upper() + + for task in taskplan: + + cfg = dict_configs[task] + + wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) taskstr = f'{task.upper()}_{cdump.upper()}' @@ -127,6 +191,642 @@ def get_resources(dict_configs, task_plan, cdump='gdas'): return ''.join(strings) +def get_postgroups(post, cdump='gdas'): + + fhmin = post['FHMIN'] + fhmax = post['FHMAX'] + fhout = post['FHOUT'] + + # Get a list of all forecast hours + if cdump in ['gdas']: + fhrs = list(range(fhmin, fhmax + fhout, fhout)) + elif cdump in ['gfs']: + fhmax = np.max([post['FHMAX_GFS_00'], post['FHMAX_GFS_06'], post['FHMAX_GFS_12'], post['FHMAX_GFS_18']]) + fhout = post['FHOUT_GFS'] + fhmax_hf = post['FHMAX_HF_GFS'] + fhout_hf = post['FHOUT_HF_GFS'] + fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) + fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + + npostgrp = post['NPOSTGRP'] + ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) + + fhrs = [f'f{f:03d}' for f in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [f.tolist() for f in fhrs] + + fhrgrp = ' '.join([f'_{f[0]}-{f[-1]}' for f in fhrs]) + fhrdep = ' '.join([f[-1] for f in fhrs]) + fhrlst = ' '.join(['_'.join(f) for f in fhrs]) + + return fhrgrp, fhrdep, fhrlst + + +def get_workflow(dict_configs, cdump='gdas'): + ''' + Create tasks for forecast only workflow + ''' + + envars = [] + envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) + envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) + envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) + envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) + envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;')) + envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) + envars.append(rocoto.create_envar(name='cyc', value='@H')) + + base = dict_configs['base'] + machine = base.get('machine', wfu.detectMachine()) + hpssarch = base.get('HPSSARCH', 'NO').upper() + app = base.get('APP', "ATM").upper() + do_wave = base.get('DO_WAVE', 'NO').upper() + do_ocean = base.get('DO_OCN', 'NO').upper() + do_ice = base.get('DO_ICE', 'NO').upper() + do_aero = base.get('DO_AERO', 'NO').upper() + do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() + if do_wave in ['YES']: + do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() + do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() + do_gempak = base.get('DO_GEMPAK', 'NO').upper() + do_awips = base.get('DO_AWIPS', 'NO').upper() + do_wafs = base.get('WAFSF', 'NO').upper() + do_vrfy = base.get('DO_VRFY', 'YES').upper() + do_metp = base.get('DO_METP', 'NO').upper() + n_tiles = 6 + + tasks = [] + + if app in ['S2S', 'S2SW']: + # Copy prototype ICs + deps = [] + base_cplic = dict_configs['coupled_ic']['BASE_CPLIC'] + + # ATM ICs + for file in ['gfs_ctrl.nc'] + [f'{datatype}_data.tile{tile_index}.nc' for datatype in ['gfs', 'sfc'] for tile_index in range(1, n_tiles + 1)]: + data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ATMIC']}/@Y@m@d@H/&CDUMP;/{base.get('CASE','C384')}/INPUT/{file}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Ocean ICs + if do_ocean in ["YES"]: + ocn_res = base.get('OCNRES', '025') + for res in ['res'] + [f'res_{res_index}' for res_index in range(1, 5)]: + data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_OCNIC']}/@Y@m@d@H/ocn/{ocn_res:03d}/MOM.{res}.nc" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Ice ICs + if do_ice in ["YES"]: + ice_res = base.get('ICERES', '025') + ice_res_dec = f'{float(ice_res)/100:.2f}' + data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ICEIC']}/@Y@m@d@H/ice/{ice_res:03d}/cice5_model_{ice_res_dec}.res_@Y@m@d@H.nc" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Wave ICs + if do_wave in ["YES"]: + for wave_grid in dict_configs['waveinit']['waveGRD'].split(): + data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_WAVIC']}/@Y@m@d@H/wav/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('coupled_ic', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + else: + if hpssarch in ['YES']: + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) + + task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # init + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + if hpssarch in ['YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}getic'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep=deps) + + deps = [] + deps.append(dependencies) + if hpssarch in ['YES']: + deps.append(dependencies2) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # waveinit + if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: + task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars) + tasks.append(task) + tasks.append('\n') + + # waveprep + if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}waveinit'} + deps.append(rocoto.add_dependency(dep_dict)) + if app not in ['S2S', 'S2SW']: + dep_dict = {'type': 'task', 'name': f'{cdump}init'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # aerosol_init + if do_aero in ['Y', 'YES']: + deps = [] + if app in ['S2S', 'S2SW']: + dep_dict = {'type': 'task', 'name': 'coupled_ic'} + else: + dep_dict = {'type': 'task', 'name': f'{cdump}init'} + + deps.append(rocoto.add_dependency(dep_dict)) + + # Files from current cycle + files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] + for file in files: + data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/{file}' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + # Files from previous cycle + dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} + deps.append(rocoto.add_dependency(dep_dict)) + + files = [f'@Y@m@d.@H0000.fv_core.res.nc'] + \ + [f'@Y@m@d.@H0000.fv_core.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] + \ + [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] + + data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RERUN_RESTART/' + dep_dict = {'type': 'data', 'data': data, 'offset': f'-{base["INTERVAL"]}'} + # Hack off the trailing tag because we are going to concatenate with the rest + dependency1 = rocoto.add_dependency(dep_dict)[:-10] + for file in files: + dep_dict = {'type': 'data', 'data': file} + # Hack off the leading tag to join with the earlier one + dependency2 = rocoto.add_dependency(dep_dict)[9:] + # Combine the two into a dependency with two different cyclestr tags + deps.append(dependency1 + dependency2) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('aerosol_init', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # fcst + deps = [] + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + dep_dict = {'type':'data', 'data':data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps) + + if do_aero in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}aerosol_init'} + deps.append(rocoto.add_dependency(dep_dict)) + deps2 = [] + dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='not', dep=deps2)) + dependencies3 = rocoto.create_dependency(dep_condition='or', dep=deps) + + deps = [] + deps.append(dependencies) + if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: + deps.append(dependencies2) + if do_aero in ['Y', 'YES']: + deps.append(dependencies3) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # post + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, + metatask='post', varname=varname1, varval=varval1, vardict=vardict) + tasks.append(task) + tasks.append('\n') + + # wavepostsbs + if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: + deps = [] + for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wavepostbndpnt + if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wavepostbndpntbll + if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wavepostpnt + if do_wave in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave_bnd in ['YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wavegempak + if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # waveawipsbulls + if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # waveawipsgridded + if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # ocnpost + if do_ocean in ['YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_postgroups(dict_configs['ocnpost'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('ocnpost', cdump=cdump, envar=postenvars, dependency=dependencies, + metatask='ocnpost', varname=varname1, varval=varval1, vardict=vardict) + tasks.append(task) + tasks.append('\n') + + # wafs + if do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wafsgcip + if do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wafsgrib2 + if do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wafsgrib20p25 + if do_wafs in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wafsblending + if do_wafs in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # wafsblending0p25 + if do_wafs in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + #postsnd + if do_bufrsnd in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # awips + if do_awips in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') + fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + tasks.append(task) + tasks.append('\n') + + # gempak + if do_gempak in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # vrfy + if do_vrfy in ['Y', 'YES']: + deps = [] + dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) + tasks.append(task) + tasks.append('\n') + + # metp + if do_metp in ['Y', 'YES']: + deps = [] + dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;') + metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') + metpenvars = envars + [sdate_gfs] + [metpcase] + varname1 = 'metpcase' + varval1 = 'g2g1 g2o1 pcp1' + task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, + metatask='metp', varname=varname1, varval=varval1) + tasks.append(task) + tasks.append('\n') + + # arch + deps = [] + dep_dict = {'type':'metatask', 'name':f'{cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_vrfy in ['Y', 'YES']: + dep_dict = {'type':'task', 'name':f'{cdump}vrfy'} + deps.append(rocoto.add_dependency(dep_dict)) + if cdump in ['gfs'] and do_metp in ['Y', 'YES']: + dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':f'{hpssarch}'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_wave_bnd in ['YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if do_ocean in ['Y', 'YES']: + dep_dict = {'type': 'metatask', 'name': f'{cdump}ocnpost'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) + tasks.append(task) + tasks.append('\n') + + return ''.join(tasks) + + def get_workflow_body(dict_configs, cdump='gdas'): ''' Create the workflow body @@ -137,8 +837,7 @@ def get_workflow_body(dict_configs, cdump='gdas'): strings.append('\n') strings.append(']>\n') strings.append('\n') - strings.append( - '\n') + strings.append('\n') strings.append('\n') strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') strings.append('\n') @@ -158,6 +857,7 @@ def create_xml(dict_configs): XML directory containing XML templates, create the workflow XML ''' + dict_configs['base']['INTERVAL'] = wfu.get_gfs_interval(dict_configs['base']['gfs_cyc']) base = dict_configs['base'] @@ -166,7 +866,7 @@ def create_xml(dict_configs): resources = get_resources(dict_configs, cdump=base['CDUMP']) workflow = get_workflow_body(dict_configs, cdump=base['CDUMP']) - # Removes &MEMORY_JOB_DUMP post mortem from gdas task_tasks + # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks temp_workflow = '' memory_dict = [] for each_resource_string in re.split(r'(\s+)', resources): @@ -176,7 +876,7 @@ def create_xml(dict_configs): if 'MEMORY' not in each_line: temp_workflow += each_line else: - if any(substring in each_line for substring in memory_dict): + if any( substring in each_line for substring in memory_dict): temp_workflow += each_line workflow = temp_workflow @@ -192,7 +892,6 @@ def create_xml(dict_configs): return - if __name__ == '__main__': main() sys.exit(0) diff --git a/ush/rocoto/test_configuration.py b/ush/rocoto/test_configuration.py new file mode 100644 index 00000000000..9dbe107d1af --- /dev/null +++ b/ush/rocoto/test_configuration.py @@ -0,0 +1,29 @@ +from configuration import Configuration + + +expdir = '/Users/rmahajan/scratch/gwWork/EXPDIR/test-workflow' + +cfg = Configuration(expdir) + +print(f'experiment dir: {cfg.config_dir}') + +print('configuration files ...') +line_separator = '\n' # \escapes are not allowed inside f-strings +print(f'{line_separator.join(cfg.configs)}') + +print(f'config.base: {cfg.find_config("config.base")}') + +print('*'*80) +print('config.base ...') +base = cfg.parse_config('config.base') +cfg.print_config('config.base') + +print('*'*80) +print('config.anal...') +cfg.print_config(['config.base', 'config.anal']) + + +print('*'*80) +print('config.efcs ...') +configs = ['config.base', 'config.fcst', 'config.efcs'] +cfg.print_config(configs) diff --git a/ush/rocoto/test_hosts.py b/ush/rocoto/test_hosts.py new file mode 100644 index 00000000000..89e0daa7e0e --- /dev/null +++ b/ush/rocoto/test_hosts.py @@ -0,0 +1,17 @@ +from hosts import Host + +print(f'supported hosts are: {", ".join(Host.SUPPORTED_HOSTS)}') + +print(f'host detected as: {Host.detect}') +print(f'scheduler on host: {Host.get_scheduler}') + +print('initializing host ...') +host = Host() + +print(f'hostname: {host.machine}') + +print(f'scheduler on host: {host.scheduler}') + +print('host information ...') +line_separator = '\n' # \escapes are not allowed inside f-strings +print(f'{line_separator.join(f"{key}: {host.info[key]}" for key in host.info.keys())}') diff --git a/ush/rocoto/test_setup_workflow.py b/ush/rocoto/test_setup_workflow.py new file mode 100644 index 00000000000..bf37626e355 --- /dev/null +++ b/ush/rocoto/test_setup_workflow.py @@ -0,0 +1,18 @@ +import workflow_utils as wfu + +from configuration import Configuration +from applications import AppConfig +from workflow_xml import RocotoXML + +expdir = '/Users/rmahajan/scratch/gwWork/EXPDIR/test-workflow' +#mode = 'forecast-only' +mode = 'cycled' + +cfg = Configuration(expdir) + +_base = cfg.parse_config('config.base') +wfu.check_expdir(expdir, _base['EXPDIR']) + +app_config = AppConfig(mode, cfg) +xml = RocotoXML(app_config) +xml.write('text.xml') \ No newline at end of file From d9e7b72545330b786ae33043bacc83c9c77af0d2 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 15 Jun 2022 00:14:34 -0400 Subject: [PATCH 07/41] revert task_tasks saldfhjsafads --- ush/rocoto/applications.py | 28 +++------------------------- ush/rocoto/rocoto.py | 2 +- ush/rocoto/rocoto_viewer.py | 30 +++++++++++++++--------------- ush/rocoto/workflow_tasks.py | 2 +- 4 files changed, 20 insertions(+), 42 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index a88ae15d0ef..0ea5a65e431 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -217,7 +217,7 @@ def _source_configs(self, configuration: Configuration) -> Dict[str, Any]: def get_task_names(self): - # Get a list of all possible task_tasks that would be part of the application + # Get a list of all possible tasks that would be part of the application tasks_map = {'cycled': self._get_cycled_task_names, 'forecast-only': self._get_forecast_only_task_names} try: @@ -242,7 +242,7 @@ def _get_cycled_task_names(self): hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] - # First collect all gdas task_tasks + # First collect all gdas tasks gdas_tasks = tasks + gdas_only_tasks if self.do_gldas: @@ -256,7 +256,7 @@ def _get_cycled_task_names(self): gdas_tasks += hybrid_tasks gdas_tasks += hybrid_gdas_tasks - # Now collect gfs task_tasks + # Now collect gfs tasks gfs_tasks = tasks if self.do_hybvar: @@ -327,25 +327,3 @@ def _get_forecast_only_task_names(self): return {'cdump': tasks} - -class App: - - def __init__(self, app_config: AppConfig): - self.app_config = app_config - - def get_tasks(self): - - self.task_tasks = dict() - #for cdump in self.task_names.keys(): - # self.task_tasks[cdump] = Tasks(self, cdump) - - def assemble_tasks(self): - - return None - - def get_app_configuration(self): - - config = type('AppConfig',(),{}) # Initialize a class; type(classname, superclasses, attributedict) - - config.mode = self.mode - diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 765852879da..9571d7e9fcd 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -7,7 +7,7 @@ rocoto.py ABOUT: - Helper module to create task_tasks, metatasks, and dependencies for Rocoto + Helper module to create tasks, metatasks, and dependencies for Rocoto ''' __all__ = ['create_task', 'create_metatask', diff --git a/ush/rocoto/rocoto_viewer.py b/ush/rocoto/rocoto_viewer.py index 24e6a78b212..63db6f25383 100755 --- a/ush/rocoto/rocoto_viewer.py +++ b/ush/rocoto/rocoto_viewer.py @@ -323,7 +323,7 @@ def usage(message=None): -w workflow.xml -d database.db Optional arguments: - --listtasks --- print out a list of all task_tasks + --listtasks --- print out a list of all tasks --html=filename.html --- creates an HTML document of status --help --- print this usage message''') @@ -441,7 +441,7 @@ def get_arguments(): send_html_to_rzdm = True if list_tasks and workflow_file is None: - usage('In order to list task_tasks you must supply the XML worflow-file') + usage('In order to list tasks you must supply the XML worflow-file') if only_check_point and (workflow_file is None or database_file is None or save_checkfile_path is None): usage('To use the check point output you must specify the workflow, data base, and the specific name of the checkpoint file') @@ -582,7 +582,7 @@ def help_screen(screen): '(->) Next Cycle own (or) Page-dwn to scroll', '(<-) Previous Cycle own (or) Page-up to scroll ', ' ', - ' + Arrow Up to selected multiple task_tasks', + ' + Arrow Up to selected multiple tasks', ' + Arrow Down for using with rocoto utils', 'Double-Click or to expand/collapse metatasks', ' ', @@ -595,7 +595,7 @@ def help_screen(screen): '', 'oads and renews status data (no rocotorun)', 'inds the last cycle with a running task', - 'nloads and clears all previously seleted task_tasks', + 'nloads and clears all previously seleted tasks', 'makes a symlink of log file of highlited task'] for i in range(0, len(helpstr)): @@ -837,7 +837,7 @@ def get_rocoto_check(params, queue_check): def rocoto_boot(params): workflow_file, database_file, cycle, metatask_list, task_list = params - stat = syscall([rocotoboot, '--workflow', workflow_file, '--database', database_file, '--cycles', cycle, '--task_tasks', task_list]) + stat = syscall([rocotoboot, '--workflow', workflow_file, '--database', database_file, '--cycles', cycle, '--tasks', task_list]) if stat is None: display_results('rocotoboot failed!!', '') return stat @@ -1001,7 +1001,7 @@ def get_tasklist(workflow_file): else: metatask_list[first_task_resolved_name].append(task[0]) if list_tasks: - print(f'task_tasks: , {i}, {task[0]}, {task[1]}, LOG:, {task[2]}') + print(f'tasks: , {i}, {task[0]}, {task[1]}, LOG:, {task[2]}') # Default expantion of metatasks True = collapsed # for metatask,metatasks in metatask_list.iteritems(): @@ -2246,30 +2246,30 @@ def main(screen): screen.clear() process = '' if highlight_CYCLE: - screen.addstr(f'Are you sure you want to rewind all the task_tasks in the cycle {execute_cycle} by running:\n\n') + screen.addstr(f'Are you sure you want to rewind all the tasks in the cycle {execute_cycle} by running:\n\n') process = '-a' # highlight_WORKFLOW = False elif execute_metatask_check and len(selected_tasks[execute_cycle]) == 0: for tasks in metatask_list_of_selected_metatask: process += '-t ' + tasks + ' ' - screen.addstr(f'Are you sure you want to rewind all the task_tasks in the metatask ({execute_task}) by running:\n\n') + screen.addstr(f'Are you sure you want to rewind all the tasks in the metatask ({execute_task}) by running:\n\n') elif len(selected_tasks[execute_cycle]) != 0 or len(selected_meta_tasks[execute_cycle]) != 0: if len(selected_tasks[execute_cycle]) != 0: selected_tasks_string = '' - screen.addstr('Selected task_tasks:\n\n') + screen.addstr('Selected tasks:\n\n') for tasks in selected_tasks[execute_cycle]: selected_tasks_string += tasks + '\t' process += '-t ' + tasks + ' ' screen.addstr(selected_tasks_string + '\n\n') if len(selected_meta_tasks[execute_cycle]) != 0: selected_tasks_string = '' - screen.addstr(f'Selected {len(selected_meta_tasks[execute_cycle]):d} entire meta-task_tasks and their task_tasks:\n\n') + screen.addstr(f'Selected {len(selected_meta_tasks[execute_cycle]):d} entire meta-tasks and their tasks:\n\n') for meta_task_selected in selected_meta_tasks[execute_cycle]: for tasks in metatask_list_by_name[meta_task_selected]: selected_tasks_string += tasks + '\t' process += '-t ' + tasks + ' ' screen.addstr(selected_tasks_string + '\n\n') - screen.addstr('\nAre you sure you want to rewind all these seleted task_tasks by running:\n\n') + screen.addstr('\nAre you sure you want to rewind all these seleted tasks by running:\n\n') elif len(selected_tasks[execute_cycle]) == 0: process = '-t ' + execute_task screen.addstr(f'Are you sure you want to rewind the single task {execute_task} by running:\n\n') @@ -2280,7 +2280,7 @@ def main(screen): if event == ord('y') or event == ord('Y'): params = (workflow_file, database_file, execute_cycle, process) results = rocoto_rewind(params) - results_params = ('', '', 'rewind', execute_cycle, 'task_tasks') + results_params = ('', '', 'rewind', execute_cycle, 'tasks') try: display_results(results, screen, results_params) except Exception: @@ -2315,7 +2315,7 @@ def main(screen): screen.addstr(f'You have selected to boot the entire cycle {execute_cycle}:\n\n', curses.A_BOLD) tasks_to_boot = tasks_in_cycle[cycle] elif len(selected_tasks[execute_cycle]) != 0: - screen.addstr('You have a list selected task_tasks boot:\n\n', curses.A_BOLD) + screen.addstr('You have a list selected tasks boot:\n\n', curses.A_BOLD) tasks_to_boot = selected_tasks[execute_cycle] elif len(selected_meta_tasks[execute_cycle]) != 0: screen.addstr(f'Are you sure you want boot the metatask {selected_meta_tasks[execute_cycle][0]} by running rocotoboot with:') @@ -2333,9 +2333,9 @@ def main(screen): boot_task_list = boot_task_list[:-1] screen.addstr(list_of_tasks) - screen.addstr(f'\n\nAre you sure you want to boot all the task_tasks and/or metatasks in the cycle {execute_cycle} by running:\n\n', curses.A_BOLD) + screen.addstr(f'\n\nAre you sure you want to boot all the tasks and/or metatasks in the cycle {execute_cycle} by running:\n\n', curses.A_BOLD) if len(boot_task_list) != 0: - list_of_tasks = ' --task_tasks ' + "'" + boot_task_list + "'" + list_of_tasks = ' --tasks ' + "'" + boot_task_list + "'" screen.addstr(f'rocotoboot -c {execute_cycle} -d {basename(database_file)} -w {basename(workflow_file)} {list_meta_tasks + list_of_tasks}\n\n') screen.addstr('Enter: es or o', curses.A_BOLD) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 163a0c19993..dfe23506c0f 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -774,7 +774,7 @@ def arch(self): return task - # Start of ensemble task_tasks + # Start of ensemble tasks def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} From a17eea139c9a06e272bf6d18049813f12780d93b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 15 Jun 2022 10:34:02 -0400 Subject: [PATCH 08/41] front-end for RocotoXML generation --- ush/rocoto/setup_workflow.py | 1432 ------------------------- ush/rocoto/setup_workflow_fcstonly.py | 897 ---------------- ush/rocoto/setup_xml.py | 66 ++ ush/rocoto/test_setup_workflow.py | 18 - ush/rocoto/workflow_utils.py | 34 - ush/rocoto/workflow_xml.py | 5 +- 6 files changed, 70 insertions(+), 2382 deletions(-) delete mode 100755 ush/rocoto/setup_workflow.py delete mode 100755 ush/rocoto/setup_workflow_fcstonly.py create mode 100644 ush/rocoto/setup_xml.py delete mode 100644 ush/rocoto/test_setup_workflow.py diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py deleted file mode 100755 index 43ee7ab8caa..00000000000 --- a/ush/rocoto/setup_workflow.py +++ /dev/null @@ -1,1432 +0,0 @@ -#!/usr/bin/env python3 - -''' - PROGRAM: - Create the ROCOTO workflow given the configuration of the GFS parallel - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov - - FILE DEPENDENCIES: - 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. - Without these dependencies, the script will fail - - OUTPUT: - 1. PSLOT.xml: XML workflow - 2. PSLOT.crontab: crontab for ROCOTO run command -''' - -import os -import sys -import re -import numpy as np -from datetime import datetime, timedelta -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from collections import OrderedDict -import rocoto -import workflow_utils as wfu - -def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a GFS parallel.', formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - args = parser.parse_args() - - configs = wfu.get_configs(args.expdir) - - _base = wfu.config_parser([wfu.find_config('config.base', configs)]) - - if not os.path.samefile(args.expdir, _base['EXPDIR']): - print('MISMATCH in experiment directories!') - print(f'config.base: EXPDIR = {repr(_base["EXPDIR"])}') - print(f'input arg: --expdir = {repr(args.expdir)}') - sys.exit(1) - - gfs_steps = ['prep', 'anal', 'analdiag', 'analcalc', 'gldas', 'fcst', 'postsnd', 'post', 'vrfy', 'arch'] - gfs_steps_gempak = ['gempak'] - gfs_steps_awips = ['awips'] - gfs_steps_wafs = ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] - #hyb_steps = ['eobs', 'eomg', 'eupd', 'ecen', 'efcs', 'epos', 'earc'] - metp_steps = ['metp'] - wav_steps = ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - #Implement additional wave jobs at later date - wav_steps_gempak = ['wavegempak'] - wav_steps_awips = ['waveawipsbulls', 'waveawipsgridded'] -# From gfsv16b latest -# gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] - hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] - - steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps - steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps - steps = steps + gfs_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps - steps = steps + gfs_steps_awips if _base.get('DO_AWIPS', 'NO') == 'YES' else steps - steps = steps + gfs_steps_wafs if _base.get('WAFSF', 'NO') == 'YES' else steps - steps = steps + wav_steps if _base.get('DO_WAVE', 'NO') == 'YES' else steps - steps = steps + wav_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps - steps = steps + wav_steps_awips if _base.get('DO_AWIPS', 'NO') == 'YES' else steps - - dict_configs = wfu.source_configs(configs, steps) - - # Check and set gfs_cyc specific variables - if dict_configs['base']['gfs_cyc'] != 0: - dict_configs['base'] = get_gfs_cyc_dates(dict_configs['base']) - - # First create workflow XML - create_xml(dict_configs) - - # Next create the crontab - wfu.create_crontab(dict_configs['base']) - - return - - -def get_gfs_cyc_dates(base): - ''' - Generate GFS dates from experiment dates and gfs_cyc choice - ''' - - base_out = base.copy() - - gfs_cyc = base['gfs_cyc'] - sdate = base['SDATE'] - edate = base['EDATE'] - - interval_gfs = wfu.get_gfs_interval(gfs_cyc) - - # Set GFS cycling dates - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - fhmax_gfs = {} - for hh in ['00', '06', '12', '18']: - fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', 'FHMAX_GFS_00') - base_out['FHMAX_GFS'] = fhmax_gfs - - return base_out - - -def get_preamble(): - ''' - Generate preamble for XML - ''' - - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' - - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - hpssarch = base.get('HPSSARCH', 'NO').upper() - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - - if base['gfs_cyc'] != 0: - strings.append(get_gfs_dates(base)) - strings.append('\n') - - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if scheduler in ['slurm']: - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_gfs_dates(base): - ''' - Generate GFS dates entities - ''' - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - - return ''.join(strings) - - -def get_gdasgfs_resources(dict_configs, cdump='gdas'): - ''' - Create GDAS or GFS resource entities - ''' - - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_wafs = base.get('WAFSF', 'NO').upper() - do_metp = base.get('DO_METP', 'NO').upper() - do_gldas = base.get('DO_GLDAS', 'NO').upper() - do_wave = base.get('DO_WAVE', 'NO').upper() - do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - reservation = base.get('RESERVATION', 'NONE').upper() - - #tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] - tasks = ['prep', 'anal', 'analcalc'] - - if cdump in ['gdas']: - tasks += ['analdiag'] - if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: - tasks += ['gldas'] - if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: - #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] - tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - - tasks += ['fcst', 'post', 'vrfy', 'arch'] - - if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] - tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: - tasks += ['postsnd'] - if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: - tasks += ['gempak'] - if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: - tasks += ['wavegempak'] - if cdump in ['gfs'] and do_awips in ['Y', 'YES']: - tasks += ['awips'] - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - tasks += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - tasks += ['metp'] - if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: - tasks += ['waveawipsbulls', 'waveawipsgridded'] - - dict_resources = OrderedDict() - - for task in tasks: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) - taskstr = f'{task.upper()}_{cdump.upper()}' - - strings = [] - strings.append(f'\t\n') - if scheduler in ['slurm']: - if task in ['arch']: - strings.append(f'\t\n') - else: - strings.append(f'\t\n') - - strings.append(f'\t\n') - strings.append(f'\t\n') - if len(memstr) != 0: - strings.append(f'\t\n') - strings.append(f'\t\n') - - dict_resources[f'{cdump}{task}'] = ''.join(strings) - - return dict_resources - - -def get_hyb_resources(dict_configs): - ''' - Create hybrid resource entities - ''' - - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() - eupd_cyc= base.get('EUPD_CYC', 'gdas').upper() - reservation = base.get('RESERVATION', 'NONE').upper() - - dict_resources = OrderedDict() - - # These tasks can be run in either or both cycles - if lobsdiag_forenkf in ['.T.', '.TRUE.']: - tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] - else: - tasks1 = ['eobs', 'eomg', 'eupd', 'echgres'] - - if eupd_cyc in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif eupd_cyc in ['GFS']: - cdumps = ['gfs'] - elif eupd_cyc in ['GDAS']: - cdumps = ['gdas'] - - for cdump in cdumps: - for task in tasks1: - - cfg = dict_configs['eobs'] if task in ['eomg'] else dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) - - taskstr = f'{task.upper()}_{cdump.upper()}' - - strings = [] - - strings.append(f'\t\n') - if scheduler in ['slurm']: - strings.append(f'\t\n') - strings.append(f'\t\n') - strings.append(f'\t\n') - if len(memstr) != 0: - strings.appendf(f'\t\n') - strings.append(f'\t\n') - - dict_resources[f'{cdump}{task}'] = ''.join(strings) - - - # These tasks are always run as part of the GDAS cycle - cdump = 'gdas' - tasks2 = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] - for task in tasks2: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) - - taskstr = f'{task.upper()}_{cdump.upper()}' - - strings = [] - strings.append(f'\t\n') - if scheduler in ['slurm']: - if task in ['earc']: - strings.append(f'\t\n') - else: - strings.append(f'\t\n') - - strings.append(f'\t\n') - strings.append(f'\t\n') - if len(memstr) != 0: - strings.append(f'\t\n') - strings.append(f'\t\n') - - dict_resources[f'{cdump}{task}'] = ''.join(strings) - - return dict_resources - - -def get_gdasgfs_tasks(dict_configs, cdump='gdas'): - ''' - Create GDAS or GFS tasks - ''' - - envars = [] - if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: - envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - base = dict_configs['base'] - gfs_cyc = base.get('gfs_cyc', 0) - gldas_cyc = base.get('gldas_cyc', 0) - dohybvar = base.get('DOHYBVAR', 'NO').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_wafs = base.get('WAFSF', 'NO').upper() - do_metp = base.get('DO_METP', 'NO').upper() - do_gldas = base.get('DO_GLDAS', 'NO').upper() - do_wave = base.get('DO_WAVE', 'NO').upper() - if do_wave in ['YES']: - do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() - do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - dumpsuffix = base.get('DUMP_SUFFIX', '') - gridsuffix = base.get('SUFFIX', '') - - dict_tasks = OrderedDict() - - # prep - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{gridsuffix}' - dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&DMPDIR;/{cdump}{dumpsuffix}.@Y@m@d/@H/{cdump}.t@Hz.updated.status.tm00.bufr_d' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False - - if gfs_enkf and cdump in ['gfs']: - if gfs_cyc == 4: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') - - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}prep'] = task - - # wave tasks in gdas or gfs or both - if do_wave_cdump in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif do_wave_cdump in ['GFS']: - cdumps = ['gfs'] - elif do_wave_cdump in ['GDAS']: - cdumps = ['gdas'] - - # waveinit - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks['{cdump}waveinit'] = task - - # waveprep - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}waveinit'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks['{cdump}waveprep'] = task - - # anal - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - else: - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}anal'] = task - - # analcalc - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': - dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - else: - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('analcalc', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}analcalc'] = task - - # analdiag - if cdump in ['gdas']: - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - task = wfu.create_wf_task('analdiag', cdump=cdump, envar=envars, dependency=dependencies2) - - dict_tasks[f'{cdump}analdiag'] = task - - # gldas - if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - task = wfu.create_wf_task('gldas', cdump=cdump, envar=envars, dependency=dependencies2) - dict_tasks[f'{cdump}gldas'] = task - - # fcst - deps1 = [] - #data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - #dep_dict = {'type': 'data', 'data': data} - #deps1.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps1.append(rocoto.add_dependency(dep_dict)) - if do_gldas in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}gldas'} - deps1.append(rocoto.add_dependency(dep_dict)) - else: - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - elif cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies2) - else: - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies1) - - dict_tasks[f'{cdump}fcst'] = task - - # post - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}post'] = task - - # wavepostsbs - if do_wave in ['Y', 'YES'] and cdump in cdumps: - deps = [] - for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostsbs'] = task - - # wavepostbndpnt - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostbndpnt'] = task - - # wavepostbndpntbll - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES'] and cdump in ['gfs']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostbndpntbll'] = task - - # wavepostpnt - if do_wave in ['Y', 'YES'] and cdump in ['gdas', 'gfs']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES'] and cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavepostpnt'] = task - - # wavegempak - if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}wavegempak'] = task - - # waveawipsgridded - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}waveawipsgridded'] = task - - # waveawipsbulls - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES'] and cdump in ['gfs']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}waveawipsbulls'] = task - - # wavestat - #if do_wave in ['Y', 'YES'] and cdump in cdumps: - # deps = [] - # dep_dict = {'type':'task', 'name':'%swavepost' % cdump} - # deps.append(rocoto.add_dependency(dep_dict)) - # dependencies = rocoto.create_dependency(dep=deps) - # task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies) - # dict_tasks['%swavestat' % cdump] = task - - # vrfy - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}vrfy'] = task - - # metp - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE_GFS;') - metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') - metpenvars = envars + [sdate_gfs] + [metpcase] - varname1 = 'metpcase' - varval1 = 'g2g1 g2o1 pcp1' - task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) - dict_tasks[f'{cdump}metp'] = task - - #postsnd - if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}postsnd'] = task - - # awips - if cdump in ['gfs'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}awips'] = task - - # gempak - if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}gempak'] = task - - # wafs - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafs'] = task - - # wafsgcip - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgcip'] = task - - # wafsgrib2 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgrib2'] = task - - # wafsgrib20p25 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsgrib20p25'] = task - - # wafsblending - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsblending'] = task - - # wafsblending0p25 - if cdump in ['gfs'] and do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}wafsblending0p25'] = task - - # arch - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES'] and cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks[f'{cdump}arch'] = task - - return dict_tasks - - -def get_hyb_tasks(dict_configs, cycledef='enkf'): - ''' - Create Hybrid tasks - ''' - - # Determine groups based on ensemble size and grouping - base = dict_configs['base'] - nens = base['NMEM_ENKF'] - lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - - eobs = dict_configs['eobs'] - nens_eomg = eobs['NMEM_EOMGGRP'] - neomg_grps = nens / nens_eomg - EOMGGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(neomg_grps) + 1)]) - - efcs = dict_configs['efcs'] - nens_efcs = efcs['NMEM_EFCSGRP'] - nefcs_grps = nens / nens_efcs - EFCSGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(nefcs_grps) + 1)]) - - earc = dict_configs['earc'] - nens_earc = earc['NMEM_EARCGRP'] - nearc_grps = nens / nens_earc - EARCGROUPS = ' '.join([f'{x:02d}' for x in range(0, int(nearc_grps) + 1)]) - - envars = [] - if wfu.get_scheduler(wfu.detectMachine()) in ['slurm']: - envars.append(rocoto.create_envar(name='SLURM_SET', value='YES')) - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - #envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#') - - dict_tasks = OrderedDict() - - if eupd_cyc in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif eupd_cyc in ['GFS']: - cdumps = ['gfs'] - elif eupd_cyc in ['GDAS']: - cdumps = ['gdas'] - - for cdump in cdumps: - - envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') - envars1 = envars + [envar_cdump] - - # eobs - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}eobs'] = task - - # eomn, eomg - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - eomgenvars= envars1 + [ensgrp] - task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}eomn'] = task - - # ediag - else: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('ediag', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}ediag'] = task - - # eupd - deps = [] - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}eomn'} - else: - dep_dict = {'type': 'task', 'name': f'{cdump}ediag'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}eupd'] = task - - # All hybrid tasks beyond this point are always executed in the GDAS cycle - cdump = 'gdas' - envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') - envars1 = envars + [envar_cdump] - cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas' - - # ecmn, ecen - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ecenenvars = envars1 + [fhrgrp] + [fhrlst] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_ecengroups(dict_configs, dict_configs['ecen'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('ecen', cdump=cdump, envar=ecenenvars, dependency=dependencies2, - metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}ecmn'] = task - - # esfc - deps1 = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' - dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - task = wfu.create_wf_task('esfc', cdump=cdump, envar=envars1, dependency=dependencies2, cycledef=cycledef) - - dict_tasks[f'{cdump}esfc'] = task - - # efmn, efcs - deps1 = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}ecmn'} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}esfc'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - - deps2 = [] - deps2 = dependencies1 - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='or', dep=deps2) - - efcsenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies2, - metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}efmn'] = task - - # echgres - deps1 = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} - deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) - task = wfu.create_wf_task('echgres', cdump=cdump, envar=envars1, dependency=dependencies1, cycledef=cycledef) - - dict_tasks[f'{cdump}echgres'] = task - - # epmn, epos - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}efmn'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - eposenvars = envars1 + [fhrgrp] + [fhrlst] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks[f'{cdump}epmn'] = task - - # eamn, earc - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}epmn'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - earcenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef) - - dict_tasks[f'{cdump}eamn'] = task - - return dict_tasks - - -def get_workflow_header(base): - ''' - Create the workflow header block - ''' - - strings = [] - - strings.append('\n') - strings.append(']>\n') - strings.append('\n') - strings.append('\n') - strings.append('\n') - strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') - strings.append('\n') - strings.append('\t\n') - strings.append('\t&SDATE; &SDATE; 06:00:00\n') - strings.append('\t&SDATE; &EDATE; 06:00:00\n') - strings.append('\t&SDATE; &EDATE; 06:00:00\n') - if base['gfs_cyc'] != 0: - strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;\n') - - strings.append('\n') - - return ''.join(strings) - - -def get_workflow_footer(): - ''' - Generate workflow footer - ''' - - strings = [] - strings.append('\n\n') - - return ''.join(strings) - - -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'],post['FHMAX_GFS_06'],post['FHMAX_GFS_12'],post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout)) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['anl'] + [f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join(['anl'] + [f[-1] for f in fhrs]) - fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_awipsgroups(awips, cdump='gdas'): - - fhmin = awips['FHMIN'] - fhmax = awips['FHMAX'] - fhout = awips['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([awips['FHMAX_GFS_00'],awips['FHMAX_GFS_06'],awips['FHMAX_GFS_12'],awips['FHMAX_GFS_18']]) - fhout = awips['FHOUT_GFS'] - fhmax_hf = awips['FHMAX_HF_GFS'] - fhout_hf = awips['FHOUT_HF_GFS'] - if fhmax > 240: - fhmax = 240 - if fhmax_hf > 240: - fhmax_hf = 240 - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) - - nawipsgrp = awips['NAWIPSGRP'] - ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_ecengroups(dict_configs, ecen, cdump='gdas'): - - base = dict_configs['base'] - - if base.get('DOIAU_ENKF', 'NO') == 'YES' : - fhrs = list(base.get('IAUFHRS','6').split(',')) - ifhrs = [f'f00{ff}' for ff in fhrs] - ifhrs0 = ifhrs[0] - nfhrs = len(fhrs) - - ifhrs = [f'f00{ff}' for ff in fhrs] - ifhrs0 = ifhrs[0] - nfhrs = len(fhrs) - - necengrp = ecen['NECENGRP'] - ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) - - ifhrs = np.array_split(ifhrs, ngrps) - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in ifhrs]) - fhrlst = ' '.join(['_'.join(f) for f in ifhrs]) - - else: - fhrgrp='000' - fhrdep='f006' - fhrlst='f006' - - return fhrgrp, fhrdep, fhrlst - -def get_eposgroups(epos, cdump='gdas'): - - fhmin = epos['FHMIN_ENKF'] - fhmax = epos['FHMAX_ENKF'] - fhout = epos['FHOUT_ENKF'] - fhrs = range(fhmin, fhmax+fhout, fhout) - - neposgrp = epos['NEPOSGRP'] - ngrps = neposgrp if len(fhrs) > neposgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'{x:03d}' for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - -def dict_to_strings(dict_in): - - strings = [] - for key in dict_in.keys(): - strings.append(dict_in[key]) - strings.append('\n') - - return ''.join(strings) - - -def create_xml(dict_configs): - ''' - Given an dictionary of sourced config files, - create the workflow XML - ''' - - from builtins import any as b_any - #from __builtin__ import any as b_any - - base = dict_configs['base'] - dohybvar = base.get('DOHYBVAR', 'NO').upper() - gfs_cyc = base.get('gfs_cyc', 0) - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - - # Start collecting workflow pieces - preamble = get_preamble() - definitions = get_definitions(base) - workflow_header = get_workflow_header(base) - workflow_footer = get_workflow_footer() - - # Get GDAS related entities, resources, workflow - dict_gdas_resources = get_gdasgfs_resources(dict_configs) - dict_gdas_tasks = get_gdasgfs_tasks(dict_configs) - - # Get hybrid related entities, resources, workflow - if dohybvar in ['Y', 'YES']: - - dict_hyb_resources = get_hyb_resources(dict_configs) - dict_hyb_tasks = get_hyb_tasks(dict_configs) - - # Removes &MEMORY_JOB_DUMP post mortem from hyb tasks - hyp_tasks = {'gdaseobs':'gdaseobs', - 'gdasediag':'gdasediag', - 'gdaseomg':'gdaseomn', - 'gdaseupd':'gdaseupd', - 'gdasecen':'gdasecmn', - 'gdasesfc':'gdasesfc', - 'gdasefcs':'gdasefmn', - 'gdasepos':'gdasepmn', - 'gdasearc':'gdaseamn', - 'gdasechgres':'gdasechgres'} - for each_task, each_resource_string in dict_hyb_resources.items(): - #print(each_task,hyp_tasks[each_task]) - #print(dict_hyb_tasks[hyp_tasks[each_task]]) - if 'MEMORY' not in each_resource_string: - if each_task in dict_hyb_tasks: - temp_task_string = [] - for each_line in re.split(r'(\s+)', dict_hyb_tasks[each_task]): - if 'memory' not in each_line: - temp_task_string.append(each_line) - dict_hyb_tasks[each_task] = ''.join(temp_task_string) - if hyp_tasks[each_task] in dict_hyb_tasks: - temp_task_string = [] - for each_line in re.split(r'(\s+)', dict_hyb_tasks[hyp_tasks[each_task]]): - if 'memory' not in each_line: - temp_task_string.append(each_line) - dict_hyb_tasks[hyp_tasks[each_task]] = ''.join(temp_task_string) - - # Get GFS cycle related entities, resources, workflow - dict_gfs_resources = get_gdasgfs_resources(dict_configs, cdump='gfs') - dict_gfs_tasks = get_gdasgfs_tasks(dict_configs, cdump='gfs') - - # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks - for each_task, each_resource_string in dict_gdas_resources.items(): - if each_task not in dict_gdas_tasks: - continue - if 'MEMORY' not in each_resource_string: - temp_task_string = [] - for each_line in re.split(r'(\s+)', dict_gdas_tasks[each_task]): - if 'memory' not in each_line: - temp_task_string.append(each_line) - dict_gdas_tasks[each_task] = ''.join(temp_task_string) - - # Removes &MEMORY_JOB_DUMP post mortem from gfs tasks - for each_task, each_resource_string in dict_gfs_resources.items(): - if each_task not in dict_gfs_tasks: - continue - if 'MEMORY' not in each_resource_string: - temp_task_string = [] - for each_line in re.split(r'(\s+)', dict_gfs_tasks[each_task]): - if 'memory' not in each_line: - temp_task_string.append(each_line) - dict_gfs_tasks[each_task] = ''.join(temp_task_string) - - # Put together the XML file - xmlfile = [] - - xmlfile.append(preamble) - - xmlfile.append(definitions) - - xmlfile.append(dict_to_strings(dict_gdas_resources)) - - if dohybvar in ['Y', 'YES']: - xmlfile.append(dict_to_strings(dict_hyb_resources)) - - if gfs_cyc != 0: - xmlfile.append(dict_to_strings(dict_gfs_resources)) - elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_resources['gfsprep']) - - xmlfile.append(workflow_header) - - xmlfile.append(dict_to_strings(dict_gdas_tasks)) - - if dohybvar in ['Y', 'YES']: - xmlfile.append(dict_to_strings(dict_hyb_tasks)) - - if gfs_cyc != 0: - xmlfile.append(dict_to_strings(dict_gfs_tasks)) - elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_tasks['gfsprep']) - xmlfile.append('\n') - - xmlfile.append(workflow_footer) - - # Write the XML file - fh = open(f'{base["EXPDIR"]}/{base["PSLOT"]}.xml', 'w') - fh.write(''.join(xmlfile)) - fh.close() - - return - - -if __name__ == '__main__': - main() - sys.exit(0) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py deleted file mode 100755 index e0fd8021231..00000000000 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ /dev/null @@ -1,897 +0,0 @@ -#!/usr/bin/env python3 - -''' - PROGRAM: - Create the ROCOTO workflow for a forecast only experiment given the configuration of the GFS parallel - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov - - FILE DEPENDENCIES: - 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. - Without this dependency, the script will fail - - OUTPUT: - 1. PSLOT.xml: XML workflow - 2. PSLOT.crontab: crontab for ROCOTO run command - -''' - -import os -import sys -import re -import numpy as np -from datetime import datetime -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import rocoto -import workflow_utils as wfu - -taskplan = ['getic', 'init', 'coupled_ic', 'aerosol_init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch', 'ocnpost'] - -def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir',help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - parser.add_argument('--cdump',help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) - - args = parser.parse_args() - - configs = wfu.get_configs(args.expdir) - - _base = wfu.config_parser([wfu.find_config('config.base', configs)]) - - if not os.path.samefile(args.expdir,_base['EXPDIR']): - print('MISMATCH in experiment directories!') - print(f'''config.base: EXPDIR = {repr(_base['EXPDIR'])}''') - print(f'input arg: --expdir = {repr(args.expdir)}') - sys.exit(1) - - dict_configs = wfu.source_configs(configs, taskplan) - - dict_configs['base']['CDUMP'] = args.cdump - - # First create workflow XML - create_xml(dict_configs) - - # Next create the crontab - wfu.create_crontab(dict_configs['base']) - - return - - -def get_preamble(): - ''' - Generate preamble for XML - ''' - - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' - - machine = base.get('machine', wfu.detectMachine()) - scheduler = wfu.get_scheduler(machine) - hpssarch = base.get('HPSSARCH', 'NO').upper() - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if base['INTERVAL'] is None: - print('cycle INTERVAL cannot be None') - sys.exit(1) - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - if scheduler in ['slurm']: - strings.append(f'''\t\n''') - strings.append(f'''\t\n''') - strings.append(f'\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append(f'''\t\n''') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_resources(dict_configs, cdump='gdas'): - ''' - Create resource entities - ''' - - strings = [] - - strings.append('\t\n') - strings.append('\n') - - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - reservation = base.get('RESERVATION', 'NONE').upper() - scheduler = wfu.get_scheduler(machine) - - do_wave = base.get('DO_WAVE', 'NO').upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_metp = base.get('DO_METP', 'NO').upper() - - for task in taskplan: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, reservation, cdump=cdump) - - taskstr = f'{task.upper()}_{cdump.upper()}' - - strings.append(f'\t\n') - if scheduler in ['slurm']: - if task in ['getic', 'arch']: - strings.append(f'\t\n') - else: - strings.append(f'\t\n') - - strings.append(f'\t\n') - strings.append(f'\t\n') - if len(memstr) != 0: - strings.append(f'\t\n') - strings.append(f'\t\n') - - strings.append('\n') - - strings.append('\t\n') - - return ''.join(strings) - - -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = list(range(fhmin, fhmax + fhout, fhout)) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'], post['FHMAX_GFS_06'], post['FHMAX_GFS_12'], post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) - fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = [f'f{f:03d}' for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join([f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - -def get_workflow(dict_configs, cdump='gdas'): - ''' - Create tasks for forecast only workflow - ''' - - envars = [] - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - base = dict_configs['base'] - machine = base.get('machine', wfu.detectMachine()) - hpssarch = base.get('HPSSARCH', 'NO').upper() - app = base.get('APP', "ATM").upper() - do_wave = base.get('DO_WAVE', 'NO').upper() - do_ocean = base.get('DO_OCN', 'NO').upper() - do_ice = base.get('DO_ICE', 'NO').upper() - do_aero = base.get('DO_AERO', 'NO').upper() - do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - if do_wave in ['YES']: - do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - do_wafs = base.get('WAFSF', 'NO').upper() - do_vrfy = base.get('DO_VRFY', 'YES').upper() - do_metp = base.get('DO_METP', 'NO').upper() - n_tiles = 6 - - tasks = [] - - if app in ['S2S', 'S2SW']: - # Copy prototype ICs - deps = [] - base_cplic = dict_configs['coupled_ic']['BASE_CPLIC'] - - # ATM ICs - for file in ['gfs_ctrl.nc'] + [f'{datatype}_data.tile{tile_index}.nc' for datatype in ['gfs', 'sfc'] for tile_index in range(1, n_tiles + 1)]: - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ATMIC']}/@Y@m@d@H/&CDUMP;/{base.get('CASE','C384')}/INPUT/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if do_ocean in ["YES"]: - ocn_res = base.get('OCNRES', '025') - for res in ['res'] + [f'res_{res_index}' for res_index in range(1, 5)]: - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_OCNIC']}/@Y@m@d@H/ocn/{ocn_res:03d}/MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if do_ice in ["YES"]: - ice_res = base.get('ICERES', '025') - ice_res_dec = f'{float(ice_res)/100:.2f}' - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_ICEIC']}/@Y@m@d@H/ice/{ice_res:03d}/cice5_model_{ice_res_dec}.res_@Y@m@d@H.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if do_wave in ["YES"]: - for wave_grid in dict_configs['waveinit']['waveGRD'].split(): - data = f"{base_cplic}/{dict_configs['coupled_ic'][f'CPL_WAVIC']}/@Y@m@d@H/wav/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('coupled_ic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - else: - if hpssarch in ['YES']: - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) - - task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # init - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - if hpssarch in ['YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}getic'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep=deps) - - deps = [] - deps.append(dependencies) - if hpssarch in ['YES']: - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveinit - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - task = wfu.create_wf_task('waveinit', cdump=cdump, envar=envars) - tasks.append(task) - tasks.append('\n') - - # waveprep - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}waveinit'} - deps.append(rocoto.add_dependency(dep_dict)) - if app not in ['S2S', 'S2SW']: - dep_dict = {'type': 'task', 'name': f'{cdump}init'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # aerosol_init - if do_aero in ['Y', 'YES']: - deps = [] - if app in ['S2S', 'S2SW']: - dep_dict = {'type': 'task', 'name': 'coupled_ic'} - else: - dep_dict = {'type': 'task', 'name': f'{cdump}init'} - - deps.append(rocoto.add_dependency(dep_dict)) - - # Files from current cycle - files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] - for file in files: - data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/{file}' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Files from previous cycle - dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} - deps.append(rocoto.add_dependency(dep_dict)) - - files = [f'@Y@m@d.@H0000.fv_core.res.nc'] + \ - [f'@Y@m@d.@H0000.fv_core.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] + \ - [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile_index}.nc' for tile_index in range(1, n_tiles + 1)] - - data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RERUN_RESTART/' - dep_dict = {'type': 'data', 'data': data, 'offset': f'-{base["INTERVAL"]}'} - # Hack off the trailing tag because we are going to concatenate with the rest - dependency1 = rocoto.add_dependency(dep_dict)[:-10] - for file in files: - dep_dict = {'type': 'data', 'data': file} - # Hack off the leading tag to join with the earlier one - dependency2 = rocoto.add_dependency(dep_dict)[9:] - # Combine the two into a dependency with two different cyclestr tags - deps.append(dependency1 + dependency2) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('aerosol_init', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # fcst - deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}waveprep'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps) - - if do_aero in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}aerosol_init'} - deps.append(rocoto.add_dependency(dep_dict)) - deps2 = [] - dep_dict = {'type': 'cycleexist', 'offset': f'-{base["INTERVAL"]}'} - deps2.append(rocoto.add_dependency(dep_dict)) - deps.append(rocoto.create_dependency(dep_condition='not', dep=deps2)) - dependencies3 = rocoto.create_dependency(dep_condition='or', dep=deps) - - deps = [] - deps.append(dependencies) - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps.append(dependencies2) - if do_aero in ['Y', 'YES']: - deps.append(dependencies3) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # post - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # wavepostsbs - if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: - deps = [] - for wave_grid in dict_configs['wavepostsbs']['waveGRD'].split(): - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/wave/rundata/{cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostsbs', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostbndpnt - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpnt', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostbndpntbll - if do_wave in ['Y', 'YES'] and do_wave_bnd in ['YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.logf180.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavepostbndpntbll', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavepostpnt - if do_wave in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpntbll'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wavepostpnt', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wavegempak - if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wavegempak', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveawipsbulls - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'task', 'name':f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('waveawipsbulls', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # waveawipsgridded - if do_wave in ['Y', 'YES'] and do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'task', 'name':f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('waveawipsgridded', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # ocnpost - if do_ocean in ['YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['ocnpost'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('ocnpost', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='ocnpost', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # wafs - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafs', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgcip - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgcip', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgrib2 - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib2', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsgrib20p25 - if do_wafs in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if006' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if012' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if015' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if018' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if021' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if024' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if027' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if030' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if033' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.wafs.grb2if036' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('wafsgrib20p25', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsblending - if do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib2'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # wafsblending0p25 - if do_wafs in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}wafsgrib20p25'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('wafsblending0p25', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - #postsnd - if do_bufrsnd in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # awips - if do_awips in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # gempak - if do_gempak in ['Y', 'YES']: - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # vrfy - if do_vrfy in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # metp - if do_metp in ['Y', 'YES']: - deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - sdate_gfs = rocoto.create_envar(name='SDATE_GFS', value='&SDATE;') - metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') - metpenvars = envars + [sdate_gfs] + [metpcase] - varname1 = 'metpcase' - varval1 = 'g2g1 g2o1 pcp1' - task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) - tasks.append(task) - tasks.append('\n') - - # arch - deps = [] - dep_dict = {'type':'metatask', 'name':f'{cdump}post'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_vrfy in ['Y', 'YES']: - dep_dict = {'type':'task', 'name':f'{cdump}vrfy'} - deps.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gfs'] and do_metp in ['Y', 'YES']: - dep_dict = {'type':'metatask', 'name':f'{cdump}metp'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':f'{hpssarch}'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostsbs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_wave_bnd in ['YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}wavepostbndpnt'} - deps.append(rocoto.add_dependency(dep_dict)) - if do_ocean in ['Y', 'YES']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}ocnpost'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) - tasks.append(task) - tasks.append('\n') - - return ''.join(tasks) - - -def get_workflow_body(dict_configs, cdump='gdas'): - ''' - Create the workflow body - ''' - - strings = [] - - strings.append('\n') - strings.append(']>\n') - strings.append('\n') - strings.append('\n') - strings.append('\n') - strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') - strings.append('\n') - strings.append('\t\n') - strings.append(f'\t&SDATE; &EDATE; &INTERVAL;\n') - strings.append('\n') - strings.append(get_workflow(dict_configs, cdump=cdump)) - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def create_xml(dict_configs): - ''' - Given an experiment directory containing config files and - XML directory containing XML templates, create the workflow XML - ''' - - - dict_configs['base']['INTERVAL'] = wfu.get_gfs_interval(dict_configs['base']['gfs_cyc']) - base = dict_configs['base'] - - preamble = get_preamble() - definitions = get_definitions(base) - resources = get_resources(dict_configs, cdump=base['CDUMP']) - workflow = get_workflow_body(dict_configs, cdump=base['CDUMP']) - - # Removes &MEMORY_JOB_DUMP post mortem from gdas tasks - temp_workflow = '' - memory_dict = [] - for each_resource_string in re.split(r'(\s+)', resources): - if 'MEMORY' in each_resource_string: - memory_dict.append(each_resource_string) - for each_line in re.split(r'(\s+)', workflow): - if 'MEMORY' not in each_line: - temp_workflow += each_line - else: - if any( substring in each_line for substring in memory_dict): - temp_workflow += each_line - workflow = temp_workflow - - # Start writing the XML file - fh = open(f'{base["EXPDIR"]}/{base["PSLOT"]}.xml', 'w') - - fh.write(preamble) - fh.write(definitions) - fh.write(resources) - fh.write(workflow) - - fh.close() - - return - -if __name__ == '__main__': - main() - sys.exit(0) diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py new file mode 100644 index 00000000000..967298afd33 --- /dev/null +++ b/ush/rocoto/setup_xml.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Entry point for setting up Rocoto XML for all applications in global-workflow +""" + +import workflow_utils as wfu + +from configuration import Configuration +from applications import AppConfig +from workflow_xml import RocotoXML + + +@property +def input_args(): + """ + Method to collect user arguments for `setup_xml.py` + """ + + here = os.path.dirname(__file__) + top = os.path.abspath(os.path.join( + os.path.abspath(here), '../..')) + + description = """ + Sources configuration files based on application and\n + creates "PSLOT.xml" for use with Rocoto.\n + """ + + parser = ArgumentParser(description=description, + formatter_class=ArgumentDefaultsHelpFormatter) + + parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, + required=False, default=os.environ['PWD']) + + # Set up sub-parsers for various modes of experimentation + subparser = parser.add_subparsers(dest='mode') + cycled = subparser.add_parser( + 'cycled', help='arguments for cycled mode') + forecasts = subparser.add_parser( + 'forecast-only', help='arguments for forecast-only mode') + + forecasts.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', + required=False) + + args = parser.parse_args() + + return args + + +def main(): + + user_inputs = input_args + + cfg = Configuration(user_inputs.expdir) + + _base = cfg.parse_config('config.base') + wfu.check_expdir(user_inputs.expdir, _base['EXPDIR']) + + # Configure the application + app_config = AppConfig(user_inputs.mode, cfg) + + # Create Rocoto Tasks and Assemble them into an XML + xml = RocotoXML(app_config) + xml.write() + + # Create crontab for Rocoto # TODO - move to RocotoXML since it is the beneficiary + wfu.create_crontab(_base) diff --git a/ush/rocoto/test_setup_workflow.py b/ush/rocoto/test_setup_workflow.py deleted file mode 100644 index bf37626e355..00000000000 --- a/ush/rocoto/test_setup_workflow.py +++ /dev/null @@ -1,18 +0,0 @@ -import workflow_utils as wfu - -from configuration import Configuration -from applications import AppConfig -from workflow_xml import RocotoXML - -expdir = '/Users/rmahajan/scratch/gwWork/EXPDIR/test-workflow' -#mode = 'forecast-only' -mode = 'cycled' - -cfg = Configuration(expdir) - -_base = cfg.parse_config('config.base') -wfu.check_expdir(expdir, _base['EXPDIR']) - -app_config = AppConfig(mode, cfg) -xml = RocotoXML(app_config) -xml.write('text.xml') \ No newline at end of file diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index 2a93544fef1..d1eaaf1ca02 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -4,43 +4,9 @@ Module containing functions all workflow setups require """ import os -import numpy as np from distutils.spawn import find_executable from datetime import timedelta from typing import Dict, Any -from rocoto import create_task, create_metatask - - -def create_wf_task(task, resources, - cdump='gdas', cycledef=None, envar=None, dependency=None, - metatask=None, varname=None, varval=None, vardict=None, - final=False): - - taskstr = f'{cdump}{task}' - metatask_dict = None - if metatask is not None: - taskstr = f'{taskstr}#{varname}#' - metatask_dict = {'metataskname': f'{cdump}{metatask}', - 'varname': f'{varname}', - 'varval': f'{varval}', - 'vardict': vardict} - - cycledefstr = cdump if cycledef is None else cycledef - - task_dict = {'taskname': f'{taskstr}', - 'cycledef': f'{cycledefstr}', - 'maxtries': '&MAXTRIES;', - 'command': f'&JOBS_DIR;/{task}.sh', - 'jobname': f'&PSLOT;_{taskstr}_@H', - 'resources': resources, - 'log': f'&ROTDIR;/logs/@Y@m@d@H/{taskstr}.log', - 'envars': envar, - 'dependency': dependency, - 'final': final} - - task = create_task(task_dict) if metatask is None else create_metatask(task_dict, metatask_dict) - - return ''.join(task) def check_expdir(cmd_expdir, cfg_expdir): diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index f3a4ab2d959..2e13d8dd78f 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -160,7 +160,10 @@ def assemble_xml(self) -> str: return ''.join(strings) - def write(self, xml_file: str) -> None: + def write(self, xml_file: str = None) -> None: + + if xml_file is None: + xml_file = f"{self._base['EXPDIR']}/{self._base['PSLOT']}.xml" with open(xml_file, 'w') as fh: fh.write(self.xml) From f09e8455669811bc31bb4f7996f7bb976e7814ae Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 15 Jun 2022 22:55:12 -0400 Subject: [PATCH 09/41] All "modes" produce an XML. --- ush/rocoto/applications.py | 29 +++++---- ush/rocoto/configuration.py | 14 ++--- ush/rocoto/rocoto.py | 3 - ush/rocoto/setup_xml.py | 24 +++---- ush/rocoto/test_configuration.py | 2 +- ush/rocoto/workflow_tasks.py | 23 +++---- ush/rocoto/workflow_utils.py | 35 +---------- ush/rocoto/workflow_xml.py | 104 +++++++++++++++++++++---------- 8 files changed, 117 insertions(+), 117 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 0ea5a65e431..e0b90d93c72 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -1,10 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from typing import Dict, Any from configuration import Configuration import workflow_utils as wfu from hosts import Host + class AppConfig: VALID_MODES = ['cycled', 'forecast-only'] @@ -38,10 +39,10 @@ def __init__(self, mode: str, configuration: Configuration) -> None: self.do_hpssarch = _base.get('HPSSARCH', False) - # Get a list of all possible configs that would be part of the application + # Get a list of all possible config_files that would be part of the application self.configs_names = self._get_app_configs() - # Source the configs for the jobs in the application + # Source the config_files for the jobs in the application self.configs = self._source_configs(configuration) # Update the base config dictionary based on application @@ -95,13 +96,16 @@ def _get_app_configs(self): @property def _cycled_configs(self): """ - Returns the configs that are involved in the cycled app_config + Returns the config_files that are involved in the cycled app """ configs = ['prep', - 'anal', 'analdiag', 'analcalc', 'gldas', + 'anal', 'analdiag', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + if self.do_gldas: + configs += ['gldas'] + if self.do_hybvar: configs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] @@ -129,7 +133,7 @@ def _cycled_configs(self): @property def _forecast_only_configs(self): """ - Returns the configs that are involved in the forecast-only app_config + Returns the config_files that are involved in the forecast-only app """ configs = ['fcst', 'post', 'vrfy', 'arch'] @@ -178,6 +182,7 @@ def _forecast_only_upd_base(base_in): base_out = base_in.copy() base_out['INTERVAL'] = wfu.get_gfs_interval(base_in['gfs_cyc']) + base_out['CDUMP'] = 'gfs' # TODO - is there a use case for CDUMP=gdas? return base_out @@ -193,7 +198,7 @@ def _source_configs(self, configuration: Configuration) -> Dict[str, Any]: # Return config.base as well configs['base'] = configuration.parse_config('config.base') - # Source the list of all configs involved in the application + # Source the list of all config_files involved in the application for job in self.configs_names: # All must source config.base first @@ -238,9 +243,10 @@ def _get_cycled_task_names(self): gldas_tasks = ['gldas'] wave_tasks = ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - hybrid_tasks = ['eobs', 'eupd', 'echgres'] - hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] + if self.do_hybvar: + hybrid_tasks = ['eobs', 'eupd', 'echgres'] + hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] # First collect all gdas tasks gdas_tasks = tasks + gdas_only_tasks @@ -325,5 +331,4 @@ def _get_forecast_only_task_names(self): if self.do_metp: tasks += ['metp'] - return {'cdump': tasks} - + return {f"{self._base['CDUMP']}": tasks} diff --git a/ush/rocoto/configuration.py b/ush/rocoto/configuration.py index 5313c99414a..76e34e9cdd2 100644 --- a/ush/rocoto/configuration.py +++ b/ush/rocoto/configuration.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import random @@ -6,7 +6,7 @@ import subprocess from datetime import datetime from pathlib import Path -from typing import Union +from typing import Union, List __all__ = ['Configuration'] @@ -40,17 +40,17 @@ class Configuration: def __init__(self, config_dir: Union[str, Path]): """ Given a directory containing config files (config.XYZ), - return a list of configs minus the ones ending with ".default" + return a list of config_files minus the ones ending with ".default" """ self.config_dir = config_dir - self.configs = self._get_configs + self.config_files = self._get_configs @property - def _get_configs(self) -> list: + def _get_configs(self) -> List[str]: """ Given a directory containing config files (config.XYZ), - return a list of configs minus the ones ending with ".default" + return a list of config_files minus the ones ending with ".default" """ result = list() for config in glob.glob(f'{self.config_dir}/config.*'): @@ -64,7 +64,7 @@ def find_config(self, config_name: str) -> str: Given a config file name, find the full path of the config file """ - for config in self.configs: + for config in self.config_files: if config_name == os.path.basename(config): return config diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py index 9571d7e9fcd..f17caccae2c 100755 --- a/ush/rocoto/rocoto.py +++ b/ush/rocoto/rocoto.py @@ -73,7 +73,6 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: walltime = resources_dict.get('walltime', '00:01:00') native = resources_dict.get('native', None) memory = resources_dict.get('memory', None) - # compute = resources_dict.get('compute', None) # TODO - eliminate nodes = resources_dict.get('nodes', 1) ppn = resources_dict.get('ppn', 1) threads = resources_dict.get('threads', 1) @@ -96,8 +95,6 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: if partition is not None: strings.append(f'\t{partition}\n') strings.append(f'\t{walltime}\n') - # if compute is not None: # TODO - eliminate this block - # strings.append(f'\t{compute}\n') strings.append(f'\t{nodes}:ppn={ppn}:tpp={threads}\n') if memory is not None: strings.append(f'\t{memory}\n') diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index 967298afd33..b10fdd07f35 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -3,6 +3,8 @@ Entry point for setting up Rocoto XML for all applications in global-workflow """ +import os +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter import workflow_utils as wfu from configuration import Configuration @@ -10,16 +12,11 @@ from workflow_xml import RocotoXML -@property def input_args(): """ Method to collect user arguments for `setup_xml.py` """ - here = os.path.dirname(__file__) - top = os.path.abspath(os.path.join( - os.path.abspath(here), '../..')) - description = """ Sources configuration files based on application and\n creates "PSLOT.xml" for use with Rocoto.\n @@ -28,9 +25,6 @@ def input_args(): parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, - required=False, default=os.environ['PWD']) - # Set up sub-parsers for various modes of experimentation subparser = parser.add_subparsers(dest='mode') cycled = subparser.add_parser( @@ -38,6 +32,11 @@ def input_args(): forecasts = subparser.add_parser( 'forecast-only', help='arguments for forecast-only mode') + # Common arguments across all modes + for subp in [cycled, forecasts]: + subp.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, + required=False, default=os.environ['PWD']) + forecasts.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) @@ -46,9 +45,9 @@ def input_args(): return args -def main(): +if __name__ == '__main__': - user_inputs = input_args + user_inputs = input_args() cfg = Configuration(user_inputs.expdir) @@ -60,7 +59,4 @@ def main(): # Create Rocoto Tasks and Assemble them into an XML xml = RocotoXML(app_config) - xml.write() - - # Create crontab for Rocoto # TODO - move to RocotoXML since it is the beneficiary - wfu.create_crontab(_base) + xml.write() \ No newline at end of file diff --git a/ush/rocoto/test_configuration.py b/ush/rocoto/test_configuration.py index 9dbe107d1af..2a64568c608 100644 --- a/ush/rocoto/test_configuration.py +++ b/ush/rocoto/test_configuration.py @@ -9,7 +9,7 @@ print('configuration files ...') line_separator = '\n' # \escapes are not allowed inside f-strings -print(f'{line_separator.join(cfg.configs)}') +print(f'{line_separator.join(cfg.config_files)}') print(f'config.base: {cfg.find_config("config.base")}') diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index dfe23506c0f..1326f2ada94 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import numpy as np import rocoto from applications import AppConfig -from hosts import Host from typing import List @@ -95,8 +94,6 @@ def get_resource(self, task_name): if self.cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_config.keys(): threads = task_config[f'nth_{task_name}_gfs'] - compute = f'{nodes}:ppn={ppn}:tpp={threads}' # TODO - remove dependence on compute - memory = task_config.get(f'memory_{task_name}', None) native = '--export=NONE' if scheduler in ['slurm'] else None @@ -116,7 +113,6 @@ def get_resource(self, task_name): 'cores': cores, 'ppn': ppn, 'threads': threads, - 'compute': compute, # TODO - eliminate dependence on compute 'memory': memory, 'native': native, 'queue': queue, @@ -144,7 +140,7 @@ def coupled_ic(self): # Atm ICs atm_res = self._base.get('CASE', 'C384') - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/&CDUMP;" + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/{self.cdump}" for file in ['gfs_ctrl.nc'] + \ [f'{datatype}_data.tile{tile}.nc' for datatype in ['gfs', 'sfc'] @@ -193,7 +189,7 @@ def getic(self): deps = [] for file in files: - dep_dict = {'type': 'data', 'data': f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/{file}'} + dep_dict = {'type': 'data', 'data': f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/{file}'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps) @@ -212,7 +208,7 @@ def init(self): deps = [] for file in files: - dep_dict = {'type': 'data', 'data': f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/{file}'} + dep_dict = {'type': 'data', 'data': f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/{file}'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -231,6 +227,7 @@ def prep(self): suffix = self._base["SUFFIX"] dump_suffix = self._base["DUMP_SUFFIX"] gfs_cyc = self._base["gfs_cyc"] + dmpdir = self._base["DMPDIR"] gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False deps = [] @@ -239,7 +236,7 @@ def prep(self): data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) - data = f'&DMPDIR;/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -279,7 +276,7 @@ def aerosol_init(self): # Files from current cycle files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] for file in files: - data = f'&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/{file}' + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/INPUT/{file}' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -293,7 +290,7 @@ def aerosol_init(self): [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] for file in files: - data = ['&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RERUN_RESTART/', file] + data = [f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/RERUN_RESTART/', file] offset = [f'-{self._base["INTERVAL"]}', None] dep_dict = {'type': 'data', 'data': data, 'offset': offset} deps.append(rocoto.add_dependency(dep_dict)) @@ -377,10 +374,10 @@ def fcst(self): @property def _fcst_free(self): deps = [] - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' + data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index d1eaaf1ca02..1c7ab6bad8e 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Module containing functions all workflow setups require """ import os -from distutils.spawn import find_executable from datetime import timedelta from typing import Dict, Any @@ -82,35 +81,3 @@ def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: base_out['FHMAX_GFS'] = fhmax_gfs return base_out - - -def create_crontab(base, cronint=5): - """ - Create crontab to execute rocotorun every cronint (5) minutes - """ - - # No point creating a crontab if rocotorun is not available. - rocotoruncmd = find_executable('rocotorun') - if rocotoruncmd is None: - print('Failed to find rocotorun, crontab will not be created') - return - - rocotorunstr = f'''{rocotoruncmd} -d {base['EXPDIR']}/{base['PSLOT']}.db -w {base['EXPDIR']}/{base['PSLOT']}.xml''' - cronintstr = f'*/{cronint} * * * *' - - try: - replyto = os.environ['REPLYTO'] - except KeyError: - replyto = '' - - strings = ['', - f'#################### {base["PSLOT"]} ####################', - f'MAILTO="{replyto}"', - f'{cronintstr} {rocotorunstr}', - '#################################################################', - ''] - - with open(os.path.join(base['EXPDIR'], f'{base["PSLOT"]}.crontab'), 'w') as fh: - fh.write('\n'.join(strings)) - - return diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index 2e13d8dd78f..09aff72c447 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -1,5 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +import os +from distutils.spawn import find_executable from datetime import datetime from collections import OrderedDict import rocoto @@ -11,9 +13,6 @@ class RocotoXML: def __init__(self, app_config: AppConfig) -> None: - # Initialize variables - self.xml = None - self._app_config = app_config self._base = self._app_config.configs['base'] @@ -26,7 +25,7 @@ def __init__(self, app_config: AppConfig) -> None: self.tasks = ''.join(task_list) self.footer = self._get_workflow_footer() - self.xml = self.assemble_xml() + self.xml = self._assemble_xml() @staticmethod def _get_preamble(): @@ -56,29 +55,12 @@ def _get_definitions(self) -> str: entity = OrderedDict() entity['PSLOT'] = self._base['PSLOT'] - entity['SDATE'] = self._base['SDATE'].strftime('%Y%m%d%H%M') - entity['EDATE'] = self._base['EDATE'].strftime('%Y%m%d%H%M') - - if self._app_config.mode in ['cycled']: - entity['INTERVAL'] = self._base.get('INTERVAL', '06:00:00') - if self._base['gfs_cyc'] in [1, 2, 4]: - entity['SDATE_GFS'] = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M') - entity['EDATE_GFS'] = self._base['EDATE_GFS'].strftime('%Y%m%d%H%M') - entity['INTERVAL_GFS'] = self._base['INTERVAL_GFS'] - entity['DMPDIR'] = self._base['DMPDIR'] - - elif self._app_config.mode in ['forecast-only']: - entity['INTERVAL'] = self._base.get('INTERVAL_GFS', '24:00:00') - entity['CDUMP'] = self._base['CDUMP'] - entity['CASE'] = self._base['CASE'] # TODO - is this really used in the XML? + + if self._app_config.mode in ['forecast-only']: entity['ICSDIR'] = self._base['ICSDIR'] - entity['EXPDIR'] = self._base['EXPDIR'] entity['ROTDIR'] = self._base['ROTDIR'] - entity['RUN_ENVIR'] = self._base.get('RUN_ENVIR', 'emc') - entity['HOMEgfs'] = self._base['HOMEgfs'] entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['ARCHIVE_TO_HPSS'] = self._base.get('HPSSARCH', 'NO') entity['MAXTRIES'] = self._base.get('ROCOTO_MAXTRIES', 2) @@ -99,12 +81,14 @@ def _get_workflow_header(self): taskthrottle = self._base.get('ROCOTO_TASKTHROTTLE', 25) verbosity = self._base.get('ROCOTO_VERBOSITY', 10) + expdir = self._base['EXPDIR'] + strings = ['', ']>', '', f'', '', - f'\t&EXPDIR;/logs/@Y@m@d@H.log', + f'\t{expdir}/logs/@Y@m@d@H.log', '', '\t', ''] @@ -126,18 +110,32 @@ def _get_cycledefs(self): return cycledefs def _get_cycledefs_cycled(self): - strings = ['\t&SDATE; &SDATE; &INTERVAL;', - '\t&SDATE; &EDATE; &INTERVAL;', - '\t&SDATE; &EDATE; &INTERVAL;'] + sdate = self._base['SDATE'].strftime('%Y%m%d%H%M') + edate = self._base['EDATE'].strftime('%Y%m%d%H%M') + interval = self._base.get('INTERVAL', '06:00:00') + + strings = [f'\t{sdate} {sdate} {interval}', + f'\t{sdate} {edate} {interval}'] + + if self._app_config.do_hybvar: + strings.append(f'\t{sdate} {edate} {interval}') + if self._app_config.gfs_cyc != 0: - strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;') + sdate_gfs = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M') + edate_gfs = self._base['EDATE_GFS'].strftime('%Y%m%d%H%M') + interval_gfs = self._base['INTERVAL_GFS'] + strings.append(f'\t{sdate_gfs} {edate_gfs} {interval_gfs}') strings.append('') strings.append('') return '\n'.join(strings) def _get_cycledefs_forecast_only(self): - strings = f'\t&SDATE; &EDATE; &INTERVAL;' + sdate = self._base['SDATE'].strftime('%Y%m%d%H%M') + edate = self._base['EDATE'].strftime('%Y%m%d%H%M') + cdump = self._base['CDUMP'] + interval = self._base.get('INTERVAL_GFS', '24:00:00') + strings = f'\t{sdate} {edate} {interval}\n\n' return strings @@ -149,7 +147,7 @@ def _get_workflow_footer(): return '\n\n' - def assemble_xml(self) -> str: + def _assemble_xml(self) -> str: strings = [self.preamble, self.definitions, @@ -160,10 +158,50 @@ def assemble_xml(self) -> str: return ''.join(strings) - def write(self, xml_file: str = None) -> None: + def write(self): + self._write_xml() + self._write_crontab() + + def _write_xml(self, xml_file: str = None) -> None: if xml_file is None: - xml_file = f"{self._base['EXPDIR']}/{self._base['PSLOT']}.xml" + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + xml_file = f"{expdir}/{pslot}.xml" with open(xml_file, 'w') as fh: fh.write(self.xml) + + def _write_crontab(self, cronint: int = 5) -> None: + """ + Create crontab to execute rocotorun every cronint (5) minutes + """ + + # No point creating a crontab if rocotorun is not available. + rocotoruncmd = find_executable('rocotorun') + if rocotoruncmd is None: + print('Failed to find rocotorun, crontab will not be created') + return + + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + + rocotorunstr = f'{rocotoruncmd} -d {expdir}/{pslot}.db -w {expdir}/{pslot}.xml' + cronintstr = f'*/{cronint} * * * *' + + try: + replyto = os.environ['REPLYTO'] + except KeyError: + replyto = '' + + strings = ['', + f'#################### {pslot} ####################', + f'MAILTO="{replyto}"', + f'{cronintstr} {rocotorunstr}', + '#################################################################', + ''] + + with open(os.path.join(expdir, f'{pslot}.crontab'), 'w') as fh: + fh.write('\n'.join(strings)) + + return From 601bc65fa096395c7069cbaf108fc3440709cc29 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 15 Jun 2022 23:25:24 -0400 Subject: [PATCH 10/41] Update `setup_expt.py` to use `hosts.py` --- ush/rocoto/setup_expt.py | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index 8a6f161c790..09937c0212f 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -1,32 +1,32 @@ #!/usr/bin/env python3 -''' +""" Entry point for setting up an experiment in the global-workflow -''' +""" import os import glob import shutil from datetime import datetime from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import workflow_utils as wfu +from hosts import Host def makedirs_if_missing(dirname): - ''' + """ Creates a directory if not already present - ''' + """ if not os.path.exists(dirname): os.makedirs(dirname) def fill_COMROT(host, inputs): - ''' + """ Method to populate the COMROT for supported modes. INPUTS: host: host specific object from class HostInfo in workflow_utils.py inputs: user inputs to setup_expt.py - ''' + """ fill_modes = { 'cycled': fill_COMROT_cycled, @@ -36,7 +36,7 @@ def fill_COMROT(host, inputs): try: fill_modes[inputs.mode](host, inputs) except KeyError: - raise NotImplementedError(f'{mode} is not a supported mode.\n' + + raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + 'Currently supported modes are:\n' + f'{" | ".join(fill_modes.keys())}') @@ -44,9 +44,9 @@ def fill_COMROT(host, inputs): def fill_COMROT_cycled(host, inputs): - ''' + """ Implementation of 'fill_COMROT' for cycled mode - ''' + """ idatestr = inputs.idate.strftime('%Y%m%d%H') comrot = os.path.join(inputs.comrot, inputs.pslot) @@ -75,18 +75,18 @@ def fill_COMROT_cycled(host, inputs): def fill_COMROT_forecasts(host, inputs): - ''' + """ Implementation of 'fill_COMROT' for forecast-only mode - ''' + """ return def fill_EXPDIR(inputs): - ''' + """ Method to copy config files from workflow to experiment directory INPUTS: inputs: user inputs to `setup_expt.py` - ''' + """ configdir = inputs.configdir expdir = os.path.join(inputs.expdir, inputs.pslot) @@ -106,9 +106,9 @@ def fill_EXPDIR(inputs): def edit_baseconfig(host, inputs): - ''' + """ Parses and populates the templated `config.base.emc.dyn` to `config.base` - ''' + """ here = os.path.dirname(__file__) top = os.path.abspath(os.path.join( @@ -147,6 +147,7 @@ def edit_baseconfig(host, inputs): "@APP@": inputs.app, } + extend_dict = dict() if inputs.mode in ['cycled']: extend_dict = { "@CASEENS@": f'C{inputs.resens}', @@ -183,9 +184,9 @@ def edit_baseconfig(host, inputs): def input_args(): - ''' + """ Method to collect user arguments for `setup_expt.py` - ''' + """ here = os.path.dirname(__file__) top = os.path.abspath(os.path.join( @@ -258,9 +259,9 @@ def input_args(): def query_and_clean(dirname): - ''' + """ Method to query if a directory exists and gather user input for further action - ''' + """ create_dir = True if os.path.exists(dirname): @@ -279,7 +280,7 @@ def query_and_clean(dirname): if __name__ == '__main__': user_inputs = input_args() - host=wfu.HostInfo(wfu.detectMachine()) + host = Host() comrot = os.path.join(user_inputs.comrot, user_inputs.pslot) expdir = os.path.join(user_inputs.expdir, user_inputs.pslot) From 2a685ed225d424e79cddac28864f4f14d57d4f65 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 16 Jun 2022 22:29:20 -0400 Subject: [PATCH 11/41] pull out host specific info out of python code and into yamls. --- ush/rocoto/applications.py | 2 +- ush/rocoto/hosts.py | 215 ++++------------------------ ush/rocoto/hosts/hera.yaml | 19 +++ ush/rocoto/hosts/orion.yaml | 19 +++ ush/rocoto/hosts/wcoss_dell_p3.yaml | 19 +++ ush/rocoto/test_hosts.py | 2 +- 6 files changed, 87 insertions(+), 189 deletions(-) create mode 100644 ush/rocoto/hosts/hera.yaml create mode 100644 ush/rocoto/hosts/orion.yaml create mode 100644 ush/rocoto/hosts/wcoss_dell_p3.yaml diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index e0b90d93c72..c64f20682dc 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -19,7 +19,7 @@ def __init__(self, mode: str, configuration: Configuration) -> None: self.mode = mode - self.scheduler = Host.get_scheduler + self.scheduler = Host().scheduler _base = configuration.parse_config('config.base') diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index 296913623d2..dca063c25da 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -1,33 +1,43 @@ #!/usr/bin/env python3 import os +from pathlib import Path +from yaml import load +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + __all__ = ['Host'] +def load_yaml(_path: Path): + with open(_path, "r") as _file: + yaml_dict = load(_file, Loader=Loader) + return yaml_dict + + class Host: """ Gather Host specific information. - Someday the info will be pushed out of the code and into a yaml or something. """ - # SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', - # 'WCOSS_C', 'WCOSS_DELL_P3', 'WCOSS2'] - SUPPORTED_HOSTS = ['HERA', 'ORION', - 'WCOSS_DELL_P3'] # TODO - Remove this and uncomment above line + SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', + 'WCOSS_DELL_P3', 'WCOSS2'] - def __init__(self): + def __init__(self, host=None): - self.machine = self.detect - self.scheduler = self.get_scheduler + self.machine = self.detect if host is None else host self.info = self._get_info + self.scheduler = self.info['scheduler'] @classmethod @property def detect(cls): - # machine = 'NOTFOUND' # TODO - Uncomment this line - machine = 'ORION' # TODO - Remove this line + machine = 'NOTFOUND' # TODO - remove this comment + machine = 'ORION' # TODO - remove this line if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' @@ -49,186 +59,17 @@ def detect(cls): return machine - @classmethod - @property - def get_scheduler(cls) -> str: - - SCHEDULER_MAP = {'HERA': 'slurm', - 'JET': 'slurm', - 'ORION': 'slurm', - 'WCOSS_C': 'lsfcray', - 'WCOSS_DELL_P3': 'lsf', - 'WCOSS_DELL_P3p5': 'lsf', - 'WCOSS2': 'pbspro'} - - machine = Host.detect - try: - return SCHEDULER_MAP[machine.upper()] - except KeyError: - raise NotImplementedError(f'{machine} is not a supported host.\n' + - 'Currently supported hosts are:\n' + - f'{" | ".join(Host.SUPPORTED_HOSTS)}') - @property def _get_info(self) -> dict: - HOST_INFO_MAP = { - 'HERA': self._hera(), - 'ORION': self._orion(), - 'JET': self._jet(), - 'WCOSS_C': self._wcoss_c(), - 'WCOSS_DELL_P3': self._wcoss_dell_p3(), - 'WCOSS_DELL_P3p5': self._wcoss_dell_p3p5(), - 'WCOSS2': self._wcoss2()} - + hostfile = Path(os.path.join(os.path.dirname(__file__), f'hosts/{self.machine.lower()}.yaml')) try: - info = HOST_INFO_MAP[self.machine.upper()] - except KeyError as exc: - raise NotImplementedError(f'{self.machine} is not a supported host.\n' + - 'Currently supported hosts are:\n' + - f'{" | ".join(Host.SUPPORTED_HOSTS)}') - - return info - - @staticmethod - def _hera() -> dict: - - info = { - 'base_git': '/scratch1/NCEPDEV/global/glopara/git', - 'base_svn': '/scratch1/NCEPDEV/global/glopara/svn', - 'dmpdir': '/scratch1/NCEPDEV/global/glopara/dump', - 'nwprod': '/scratch1/NCEPDEV/global/glopara/nwpara', - 'comroot': '/scratch1/NCEPDEV/global/glopara/com', - 'homedir': '/scratch1/NCEPDEV/global/$USER', - 'stmp': '/scratch1/NCEPDEV/stmp2/$USER', - 'ptmp': '/scratch1/NCEPDEV/stmp4/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'fv3-cpu', - 'queue': 'batch', - 'queue_service': 'service', - 'partition_batch': 'hera', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @staticmethod - def _orion() -> dict: - - info = { - 'base_git': '/work/noaa/global/glopara/git', - 'base_svn': '/work/noaa/global/glopara/svn', - 'dmpdir': '/work/noaa/rstprod/dump', - 'nwprod': '/work/noaa/global/glopara/nwpara', - 'comroot': '/work/noaa/global/glopara/com', - 'homedir': '/work/noaa/global/$USER', - 'stmp': '/work/noaa/stmp/$USER', - 'ptmp': '/work/noaa/stmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'fv3-cpu', - 'queue': 'batch', - 'queue_service': 'service', - 'partition_batch': 'orion', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'NO', - 'localarch': 'NO', - 'atardir': '$NOSCRUB/archive_rotdir/$PSLOT', - } - - return info - - @staticmethod - def _jet() -> dict: - - info = {} - - return info - - @staticmethod - def _wcoss_c() -> dict: - - info = { - 'base_git': '/gpfs/hps3/emc/global/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/hps/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/hps/nco/ops/com"}', - 'homedir': '/gpfs/hps3/emc/global/noscrub/$USER', - 'stmp': '/gpfs/hps2/stmp/$USER', - 'ptmp': '/gpfs/hps2/ptmp/$USER', - 'noscrub': '/gpfs/hps3/emc/global/noscrub/$USER', - 'account': 'GFS-DEV', - 'queue': 'dev', - 'queue_service': 'dev_transfer', - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @staticmethod - def _wcoss_dell_p3() -> dict: - - info = { - 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', - 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', - 'stmp': '/gpfs/dell3/stmp/$USER', - 'ptmp': '/gpfs/dell3/ptmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'GFS-DEV', - 'queue': 'dev', - 'queue_service': 'dev_transfer', - 'partition_batch': None, - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @staticmethod - def _wcoss_dell_p3p5() -> dict: - - info = { - 'base_git': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'base_svn': '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git', - 'dmpdir': '/gpfs/dell3/emc/global/dump', - 'nwprod': '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}', - 'comroot': '${COMROOT:-"/gpfs/dell1/nco/ops/com"}', - 'homedir': '/gpfs/dell2/emc/modeling/noscrub/$USER', - 'stmp': '/gpfs/dell3/stmp/$USER', - 'ptmp': '/gpfs/dell3/ptmp/$USER', - 'noscrub': '$HOMEDIR', - 'account': 'GFS-DEV', - 'queue': 'dev2', - 'queue_service': 'dev2_transfer', - 'partition_batch': None, - 'chgrp_rstprod': 'YES', - 'chgrp_cmd': 'chgrp rstprod', - 'hpssarch': 'YES', - 'localarch': 'NO', - 'atardir': '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT', - } - - return info - - @staticmethod - def _wcoss2() -> dict: - - info = {} + info = load_yaml(hostfile) + except FileNotFoundError: + raise FileNotFoundError(f'{hostfile} does not exist!') + except IOError: + raise IOError(f'Unable to read from {hostfile}') + except Exception: + raise Exception(f'unable to get information for {self.machine}') return info diff --git a/ush/rocoto/hosts/hera.yaml b/ush/rocoto/hosts/hera.yaml new file mode 100644 index 00000000000..ad7c3c0ffab --- /dev/null +++ b/ush/rocoto/hosts/hera.yaml @@ -0,0 +1,19 @@ +base_git: '/scratch1/NCEPDEV/global/glopara/git' +base_svn: '/scratch1/NCEPDEV/global/glopara/svn' +dmpdir: '/scratch1/NCEPDEV/global/glopara/dump' +nwprod: '/scratch1/NCEPDEV/global/glopara/nwpara' +comroot: '/scratch1/NCEPDEV/global/glopara/com' +homedir: '/scratch1/NCEPDEV/global/$USER' +stmp: '/scratch1/NCEPDEV/stmp2/$USER' +ptmp: '/scratch1/NCEPDEV/stmp4/$USER' +noscrub': $HOMEDIR +account': fv3-cpu +scheduler: slurm +queue: batch +queue_service: service +partition_batch: hera +chgrp_rstprod: YES +chgrp_cmd: 'chgrp rstprod' +hpssarch: YES +localarch: NO +atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' \ No newline at end of file diff --git a/ush/rocoto/hosts/orion.yaml b/ush/rocoto/hosts/orion.yaml new file mode 100644 index 00000000000..d0837c5c6d0 --- /dev/null +++ b/ush/rocoto/hosts/orion.yaml @@ -0,0 +1,19 @@ +base_git: '/work/noaa/global/glopara/git' +base_svn: '/work/noaa/global/glopara/svn' +dmpdir: '/work/noaa/rstprod/dump' +nwprod: '/work/noaa/global/glopara/nwpara' +comroot: '/work/noaa/global/glopara/com' +homedir: '/work/noaa/global/$USER' +stmp: '/work/noaa/stmp/$USER' +ptmp: '/work/noaa/stmp/$USER' +noscrub: $HOMEDIR +scheduler: slurm +account: fv3-cpu +queue: batch +queue_service: service +partition_batch: orion +chgrp_rstprod: YES +chgrp_cmd: 'chgrp rstprod' +hpssarch: NO +localarch: NO +atardir: '$NOSCRUB/archive_rotdir/$PSLOT' \ No newline at end of file diff --git a/ush/rocoto/hosts/wcoss_dell_p3.yaml b/ush/rocoto/hosts/wcoss_dell_p3.yaml new file mode 100644 index 00000000000..2fed540270e --- /dev/null +++ b/ush/rocoto/hosts/wcoss_dell_p3.yaml @@ -0,0 +1,19 @@ +base_git: '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' +base_svn: '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git' +dmpdir: '/gpfs/dell3/emc/global/dump' +nwprod: '${NWROOT:-"/gpfs/dell1/nco/ops/nwprod"}' +comroot: '${COMROOT:-"/gpfs/dell1/nco/ops/com"}' +homedir: '/gpfs/dell2/emc/modeling/noscrub/$USER' +stmp: '/gpfs/dell3/stmp/$USER' +ptmp: '/gpfs/dell3/ptmp/$USER' +noscrub: $HOMEDIR +account: GFS-DEV +scheduler: lsf +queue: dev +queue_service: dev_transfer +partition_batch: None +chgrp_rstprod: YES +chgrp_cmd: 'chgrp rstprod' +hpssarch: YES +localarch: NO +atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' \ No newline at end of file diff --git a/ush/rocoto/test_hosts.py b/ush/rocoto/test_hosts.py index 89e0daa7e0e..b9fa969e143 100644 --- a/ush/rocoto/test_hosts.py +++ b/ush/rocoto/test_hosts.py @@ -3,7 +3,7 @@ print(f'supported hosts are: {", ".join(Host.SUPPORTED_HOSTS)}') print(f'host detected as: {Host.detect}') -print(f'scheduler on host: {Host.get_scheduler}') +print(f'scheduler on host: {Host().scheduler}') print('initializing host ...') host = Host() From 62c5394c1375c416f17fc3478673d2451e7cbcc6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 16 Jun 2022 23:00:11 -0400 Subject: [PATCH 12/41] workflow_utils.py is no longer needed. --- ush/rocoto/applications.py | 96 ++++++++++++++++++++++++++++++------ ush/rocoto/setup_expt.py | 2 +- ush/rocoto/setup_xml.py | 13 +++-- ush/rocoto/workflow_tasks.py | 2 + ush/rocoto/workflow_utils.py | 83 ------------------------------- ush/rocoto/workflow_xml.py | 2 +- 6 files changed, 96 insertions(+), 102 deletions(-) delete mode 100755 ush/rocoto/workflow_utils.py diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index c64f20682dc..72c7ac6c5e9 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -1,10 +1,78 @@ #!/usr/bin/env python3 from typing import Dict, Any +from datetime import timedelta from configuration import Configuration -import workflow_utils as wfu from hosts import Host +__all__ = ['AppConfig'] + + +def get_gfs_interval(gfs_cyc: int) -> str: + """ + return interval in hours based on gfs_cyc + """ + + gfs_internal_map = {'0': None, '1': '24:00:00', '2': '12:00:00', '4': '06:00:00'} + + try: + return gfs_internal_map[str(gfs_cyc)] + except KeyError: + raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') + + +def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate GFS dates from experiment dates and gfs_cyc choice + """ + + base_out = base.copy() + + gfs_cyc = base['gfs_cyc'] + sdate = base['SDATE'] + edate = base['EDATE'] + base_out['INTERVAL'] = '06:00:00' # Cycled interval is 6 hours + + interval_gfs = get_gfs_interval(gfs_cyc) + + # Set GFS cycling dates + hrinc = 0 + hrdet = 0 + if gfs_cyc == 0: + return base_out + elif gfs_cyc == 1: + hrinc = 24 - sdate.hour + hrdet = edate.hour + elif gfs_cyc == 2: + if sdate.hour in [0, 12]: + hrinc = 12 + elif sdate.hour in [6, 18]: + hrinc = 6 + if edate.hour in [6, 18]: + hrdet = 6 + elif gfs_cyc == 4: + hrinc = 6 + sdate_gfs = sdate + timedelta(hours=hrinc) + edate_gfs = edate - timedelta(hours=hrdet) + if sdate_gfs > edate: + print('W A R N I N G!') + print('Starting date for GFS cycles is after Ending date of experiment') + print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') + print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') + gfs_cyc = 0 + + base_out['gfs_cyc'] = gfs_cyc + base_out['SDATE_GFS'] = sdate_gfs + base_out['EDATE_GFS'] = edate_gfs + base_out['INTERVAL_GFS'] = interval_gfs + + fhmax_gfs = {} + for hh in ['00', '06', '12', '18']: + fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', base.get('FHMAX_GFS_00', 120)) + base_out['FHMAX_GFS'] = fhmax_gfs + + return base_out + class AppConfig: @@ -175,13 +243,13 @@ def _forecast_only_configs(self): @staticmethod def _cycled_upd_base(base_in): - return wfu.get_gfs_cyc_dates(base_in) + return get_gfs_cyc_dates(base_in) @staticmethod def _forecast_only_upd_base(base_in): base_out = base_in.copy() - base_out['INTERVAL'] = wfu.get_gfs_interval(base_in['gfs_cyc']) + base_out['INTERVAL'] = get_gfs_interval(base_in['gfs_cyc']) base_out['CDUMP'] = 'gfs' # TODO - is there a use case for CDUMP=gdas? return base_out @@ -189,8 +257,8 @@ def _forecast_only_upd_base(base_in): def _source_configs(self, configuration: Configuration) -> Dict[str, Any]: """ Given the configuration object and jobs, - source the configurations for each job and return a dictionary - Every job depends on "config.base" + source the configurations for each config and return a dictionary + Every config depends on "config.base" """ configs = dict() @@ -199,24 +267,24 @@ def _source_configs(self, configuration: Configuration) -> Dict[str, Any]: configs['base'] = configuration.parse_config('config.base') # Source the list of all config_files involved in the application - for job in self.configs_names: + for config in self.configs_names: # All must source config.base first files = ['config.base'] - if job in ['eobs', 'eomg']: + if config in ['eobs', 'eomg']: files += ['config.anal', 'config.eobs'] - elif job in ['eupd']: + elif config in ['eupd']: files += ['config.anal', 'config.eupd'] - elif job in ['efcs']: + elif config in ['efcs']: files += ['config.fcst', 'config.efcs'] - elif 'wave' in job: - files += ['config.wave', f'config.{job}'] + elif 'wave' in config: + files += ['config.wave', f'config.{config}'] else: - files += [f'config.{job}'] + files += [f'config.{config}'] - print(f'sourcing config.{job}') - configs[job] = configuration.parse_config(files) + print(f'sourcing config.{config}') + configs[config] = configuration.parse_config(files) return configs diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index 09937c0212f..7480fd5b010 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -24,7 +24,7 @@ def fill_COMROT(host, inputs): """ Method to populate the COMROT for supported modes. INPUTS: - host: host specific object from class HostInfo in workflow_utils.py + host: host object from class Host inputs: user inputs to setup_expt.py """ diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index b10fdd07f35..a3b801c201f 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -5,7 +5,6 @@ import os from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import workflow_utils as wfu from configuration import Configuration from applications import AppConfig @@ -45,14 +44,22 @@ def input_args(): return args +def check_expdir(cmd_expdir, cfg_expdir): + + if not os.path.samefile(cmd_expdir, cfg_expdir): + print('MISMATCH in experiment directories!') + print(f'config.base: EXPDIR = {cfg_expdir}') + print(f' input arg: --expdir = {cmd_expdir}') + raise ValueError('Abort!') + + if __name__ == '__main__': user_inputs = input_args() cfg = Configuration(user_inputs.expdir) - _base = cfg.parse_config('config.base') - wfu.check_expdir(user_inputs.expdir, _base['EXPDIR']) + check_expdir(user_inputs.expdir, cfg.parse_config('config.base')['EXPDIR']) # Configure the application app_config = AppConfig(user_inputs.mode, cfg) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 1326f2ada94..e629138d2c6 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -5,6 +5,8 @@ from applications import AppConfig from typing import List +__all__ = ['Tasks', 'create_wf_task', 'get_wf_tasks'] + class Tasks: SERVICE_TASKS = ['arch', 'earc', 'getic'] diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py deleted file mode 100755 index 1c7ab6bad8e..00000000000 --- a/ush/rocoto/workflow_utils.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 - -""" - Module containing functions all workflow setups require -""" -import os -from datetime import timedelta -from typing import Dict, Any - - -def check_expdir(cmd_expdir, cfg_expdir): - - if not os.path.samefile(cmd_expdir, cfg_expdir): - print('MISMATCH in experiment directories!') - print(f'config.base: EXPDIR = {cfg_expdir}') - print(f'input arg: --expdir = {cmd_expdir}') - raise ValueError('Abort!') - - -def get_gfs_interval(gfs_cyc: int) -> str: - """ - return interval in hours based on gfs_cyc - """ - - gfs_internal_map = {'0': None, '1': '24:00:00', '2': '12:00:00', '4': '06:00:00'} - - try: - return gfs_internal_map[str(gfs_cyc)] - except KeyError: - raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - - -def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: - """ - Generate GFS dates from experiment dates and gfs_cyc choice - """ - - base_out = base.copy() - - gfs_cyc = base['gfs_cyc'] - sdate = base['SDATE'] - edate = base['EDATE'] - base_out['INTERVAL'] = '06:00:00' # Cycled interval is 6 hours - - interval_gfs = get_gfs_interval(gfs_cyc) - - # Set GFS cycling dates - hrinc = 0 - hrdet = 0 - if gfs_cyc == 0: - return base_out - elif gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - fhmax_gfs = {} - for hh in ['00', '06', '12', '18']: - fhmax_gfs[hh] = base.get(f'FHMAX_GFS_{hh}', base.get('FHMAX_GFS_00', 120)) - base_out['FHMAX_GFS'] = fhmax_gfs - - return base_out diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index 09aff72c447..f1629852bae 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -22,7 +22,7 @@ def __init__(self, app_config: AppConfig) -> None: self.header = self._get_workflow_header() self.cycledefs = self._get_cycledefs() task_list = get_wf_tasks(app_config) - self.tasks = ''.join(task_list) + self.tasks = '\n'.join(task_list) self.footer = self._get_workflow_footer() self.xml = self._assemble_xml() From ec5e79c0bbaa92717ee81278ecc31c3116a0e786 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 22 Jun 2022 18:04:12 -0400 Subject: [PATCH 13/41] remove `first` and `enkf` `cycledefs`. --- ush/rocoto/applications.py | 10 +++--- ush/rocoto/setup_xml.py | 2 +- ush/rocoto/workflow_tasks.py | 66 ++++++++++++++++++++---------------- ush/rocoto/workflow_xml.py | 9 ++--- 4 files changed, 45 insertions(+), 42 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 72c7ac6c5e9..a3ef998ff2e 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -304,7 +304,7 @@ def get_task_names(self): def _get_cycled_task_names(self): - tasks = ['prep', 'anal', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + gdas_gfs_common_tasks = ['prep', 'anal', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] gdas_only_tasks = ['analdiag'] @@ -316,8 +316,8 @@ def _get_cycled_task_names(self): hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] - # First collect all gdas tasks - gdas_tasks = tasks + gdas_only_tasks + # Collect all "gdas" cycle tasks + gdas_tasks = gdas_gfs_common_tasks + gdas_only_tasks if self.do_gldas: gdas_tasks += gldas_tasks @@ -330,8 +330,8 @@ def _get_cycled_task_names(self): gdas_tasks += hybrid_tasks gdas_tasks += hybrid_gdas_tasks - # Now collect gfs tasks - gfs_tasks = tasks + # Collect "gfs" cycle tasks + gfs_tasks = gdas_gfs_common_tasks if self.do_hybvar: if 'gfs' in self.eupd_cdumps: diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index a3b801c201f..02c355f64e7 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -66,4 +66,4 @@ def check_expdir(cmd_expdir, cfg_expdir): # Create Rocoto Tasks and Assemble them into an XML xml = RocotoXML(app_config) - xml.write() \ No newline at end of file + xml.write() diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index e629138d2c6..a4f5a7a181b 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -32,9 +32,6 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: self.n_tiles = 6 # TODO - this needs to be elsewhere - if self.app_config.do_hybvar: - self.hyb_cycledef = 'enkf' # TODO - sort this out a bit more - envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'), 'HOMEgfs': self._base.get('HOMEgfs'), 'EXPDIR': self._base.get('EXPDIR'), @@ -256,14 +253,23 @@ def prep(self): def waveinit(self): resources = self.get_resource('waveinit') - task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars) + dependencies = None + if self.app_config.mode in ['cycled']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task def waveprep(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self}waveinit'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -340,7 +346,9 @@ def analdiag(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analdiag') task = create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) @@ -352,7 +360,9 @@ def gldas(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('gldas') task = create_wf_task('gldas', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) @@ -361,7 +371,7 @@ def gldas(self): def fcst(self): - fcst_map = {'forecast-only': self._fcst_free, + fcst_map = {'forecast-only': self._fcst_forecast_only, 'cycled': self._fcst_cycled} try: @@ -374,7 +384,7 @@ def fcst(self): return task @property - def _fcst_free(self): + def _fcst_forecast_only(self): deps = [] data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type': 'data', 'data': data} @@ -416,6 +426,11 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + if self.cdump in ['gdas']: + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + resources = self.get_resource('fcst') task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) @@ -783,8 +798,7 @@ def eobs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('eobs') - task = create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -801,8 +815,7 @@ def eomg(self): resources = self.get_resource('eomg') task = create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + metatask='eomn', varname='grp', varval=groups) return task @@ -813,8 +826,7 @@ def ediag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ediag') - task = create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -828,8 +840,7 @@ def eupd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('eupd') - task = create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -881,8 +892,7 @@ def _get_ecengroups(): resources = self.get_resource('ecen') task = create_wf_task('ecen', resources, cdump=self.cdump, envar=ecenenvars, dependency=dependencies, - metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict, - cycledef=self.hyb_cycledef) + metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict) return task def esfc(self): @@ -899,8 +909,7 @@ def esfc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('esfc') - task = create_wf_task('esfc', resources, cdump='gdas', envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('esfc', resources, cdump='gdas', envar=self.envars, dependency=dependencies) return task @@ -914,6 +923,9 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) efcsenvars = self.envars efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) @@ -922,8 +934,7 @@ def efcs(self): resources = self.get_resource('efcs') task = create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, - metatask='efmn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + metatask='efmn', varname='grp', varval=groups) return task @@ -939,8 +950,7 @@ def echgres(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('echgres') - task = create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=self.hyb_cycledef) + task = create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -984,8 +994,7 @@ def _get_eposgroups(epos): resources = self.get_resource('epos') task = create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, - cycledef=self.hyb_cycledef) + metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) return task @@ -1005,8 +1014,7 @@ def earc(self): resources = self.get_resource('earc') task = create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=groups, - cycledef=self.hyb_cycledef) + metatask='eamn', varname='grp', varval=groups) return task diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index f1629852bae..d7bbd5d418d 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -113,12 +113,7 @@ def _get_cycledefs_cycled(self): sdate = self._base['SDATE'].strftime('%Y%m%d%H%M') edate = self._base['EDATE'].strftime('%Y%m%d%H%M') interval = self._base.get('INTERVAL', '06:00:00') - - strings = [f'\t{sdate} {sdate} {interval}', - f'\t{sdate} {edate} {interval}'] - - if self._app_config.do_hybvar: - strings.append(f'\t{sdate} {edate} {interval}') + strings = [f'\t{sdate} {edate} {interval}'] if self._app_config.gfs_cyc != 0: sdate_gfs = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M') @@ -133,8 +128,8 @@ def _get_cycledefs_cycled(self): def _get_cycledefs_forecast_only(self): sdate = self._base['SDATE'].strftime('%Y%m%d%H%M') edate = self._base['EDATE'].strftime('%Y%m%d%H%M') - cdump = self._base['CDUMP'] interval = self._base.get('INTERVAL_GFS', '24:00:00') + cdump = self._base['CDUMP'] strings = f'\t{sdate} {edate} {interval}\n\n' return strings From bd5a70fcd856020ee6003ae95516023133e32d6e Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 28 Jun 2022 15:15:39 -0400 Subject: [PATCH 14/41] correct order of the tasks for cycled and forecast-only configurations. --- ush/rocoto/applications.py | 86 ++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index a3ef998ff2e..4ba1d8c2980 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -303,42 +303,68 @@ def get_task_names(self): return task_names def _get_cycled_task_names(self): + """ + Get the task names for all the tasks in the cycled application. + Note that the order of the task names matters in the XML. + This is the place where that order is set. + """ - gdas_gfs_common_tasks = ['prep', 'anal', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] - - gdas_only_tasks = ['analdiag'] + gdas_gfs_common_tasks_before_fcst = ['prep', 'anal', 'analcalc'] + gdas_gfs_common_tasks_after_fcst = ['post', 'vrfy'] + gdas_gfs_common_cleanup_tasks = ['arch'] gldas_tasks = ['gldas'] - wave_tasks = ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + wave_prep_tasks = ['waveinit', 'waveprep'] + wave_post_tasks = ['wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + hybrid_gdas_or_gfs_tasks = [] + hybrid_gdas_tasks = [] if self.do_hybvar: - hybrid_tasks = ['eobs', 'eupd', 'echgres'] - hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - hybrid_gdas_tasks = ['ecen', 'esfc', 'efcs', 'epos', 'earc'] + hybrid_gdas_or_gfs_tasks += ['eobs', 'eupd', 'echgres'] + hybrid_gdas_or_gfs_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + hybrid_gdas_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc'] # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks + gdas_only_tasks + gdas_tasks = gdas_gfs_common_tasks_before_fcst + ['analdiag'] if self.do_gldas: gdas_tasks += gldas_tasks if self.do_wave: - gdas_tasks += wave_tasks + gdas_tasks += wave_prep_tasks + + gdas_tasks += ['fcst'] + + gdas_tasks += gdas_gfs_common_tasks_after_fcst if self.do_hybvar: if 'gdas' in self.eupd_cdumps: - gdas_tasks += hybrid_tasks + gdas_tasks += hybrid_gdas_or_gfs_tasks gdas_tasks += hybrid_gdas_tasks + if self.do_wave: + gdas_tasks += wave_post_tasks + + gdas_tasks += gdas_gfs_common_cleanup_tasks + # Collect "gfs" cycle tasks - gfs_tasks = gdas_gfs_common_tasks + gfs_tasks = gdas_gfs_common_tasks_before_fcst - if self.do_hybvar: - if 'gfs' in self.eupd_cdumps: - gfs_tasks += hybrid_tasks + if self.do_wave and 'gfs' in self.wave_cdumps: + gfs_tasks += wave_prep_tasks + + gfs_tasks += ['fcst'] + + gfs_tasks += gdas_gfs_common_tasks_after_fcst + + if self.do_metp: + gfs_tasks += ['metp'] + + if self.do_hybvar and 'gfs' in self.eupd_cdumps: + gfs_tasks += hybrid_gdas_or_gfs_tasks if self.do_wave and 'gfs' in self.wave_cdumps: - gfs_tasks += wave_tasks + gfs_tasks += wave_post_tasks if self.do_gempak: gfs_tasks += ['wavegempak'] if self.do_awips: @@ -356,19 +382,23 @@ def _get_cycled_task_names(self): if self.do_wafs: gfs_tasks += ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25', 'wafsblending', 'wafsblending0p25'] - if self.do_metp: - gfs_tasks += ['metp'] + gfs_tasks += gdas_gfs_common_cleanup_tasks tasks = {'gdas': gdas_tasks, 'gfs': gfs_tasks} return tasks def _get_forecast_only_task_names(self): + """ + Get the task names for all the tasks in the forecast-only application. + Note that the order of the task names matters in the XML. + This is the place where that order is set. + """ - tasks = ['fcst', 'post', 'vrfy', 'arch'] + tasks = [] if 'S2S' in self.model_app: - tasks += ['coupled_ic', 'ocnpost'] + tasks += ['coupled_ic'] else: if self.do_hpssarch: tasks += ['getic'] @@ -378,7 +408,20 @@ def _get_forecast_only_task_names(self): tasks += ['aerosol_init'] if self.do_wave: - tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + tasks += ['waveinit', 'waveprep'] + + tasks += ['fcst'] + + tasks += ['post'] + if 'S2S' in self.model_app: + tasks += ['ocnpost'] + + tasks += ['vrfy'] + if self.do_metp: + tasks += ['metp'] + + if self.do_wave: + tasks += ['wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] if self.do_gempak: tasks += ['wavegempak'] if self.do_awips: @@ -396,7 +439,6 @@ def _get_forecast_only_task_names(self): if self.do_wafs: tasks += ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25', 'wafsblending', 'wafsblending0p25'] - if self.do_metp: - tasks += ['metp'] + tasks += ['arch'] # arch **must** be the last task return {f"{self._base['CDUMP']}": tasks} From f5e42c1c5de1b0701d7fce37acf4af9029c21151 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 30 Jun 2022 17:32:46 -0400 Subject: [PATCH 15/41] address reviewer comments. --- ush/rocoto/configuration.py | 14 +++++++------- ush/rocoto/hosts/hera.yaml | 2 +- ush/rocoto/setup_xml.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ush/rocoto/configuration.py b/ush/rocoto/configuration.py index 76e34e9cdd2..74dc890fdc4 100644 --- a/ush/rocoto/configuration.py +++ b/ush/rocoto/configuration.py @@ -6,7 +6,7 @@ import subprocess from datetime import datetime from pathlib import Path -from typing import Union, List +from typing import Union, List, Dict, Any __all__ = ['Configuration'] @@ -71,7 +71,7 @@ def find_config(self, config_name: str) -> str: raise UnknownConfigError( f'{config_name} does not exist (known: {repr(config_name)}), ABORT!') - def parse_config(self, files: Union[str, bytes, list]) -> dict: + def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]: """ Given the name of config file(s), key-value pair of all variables in the config file(s) are returned as a dictionary @@ -89,7 +89,7 @@ def parse_config(self, files: Union[str, bytes, list]) -> dict: if key in self.DATE_ENV_VARS: # likely a date, convert to datetime varbles[key] = datetime.strptime(value, '%Y%m%d%H') elif value in self.BOOLS: # Likely a boolean, convert to True/False - varbles[key] = self._bool_or_not(value) + varbles[key] = self._true_or_not(value) elif '.' in value: # Likely a number and that too a float varbles[key] = self._cast_or_not(float, value) else: # Still could be a number, may be an integer @@ -97,7 +97,7 @@ def parse_config(self, files: Union[str, bytes, list]) -> dict: return varbles - def print_config(self, files: Union[str, bytes, list]): + def print_config(self, files: Union[str, bytes, list]) -> None: """ Given the name of config file(s), key-value pair of all variables in the config file(s) are printed Same signature as parse_config @@ -110,7 +110,7 @@ def print_config(self, files: Union[str, bytes, list]): print(f'{line_separator.join(f"{key}: {config[key]}" for key in config.keys())}') @classmethod - def _get_script_env(cls, scripts): + def _get_script_env(cls, scripts: List) -> Dict[str, Any]: default_env = cls._get_shell_env([]) and_script_env = cls._get_shell_env(scripts) vars_just_in_script = set(and_script_env) - set(default_env) @@ -119,7 +119,7 @@ def _get_script_env(cls, scripts): return dict([(v, union_env[v]) for v in vars_just_in_script]) @staticmethod - def _get_shell_env(scripts): + def _get_shell_env(scripts: List) -> Dict[str, Any]: varbls = dict() runme = ''.join([f'source {s} ; ' for s in scripts]) magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---' @@ -146,7 +146,7 @@ def _cast_or_not(type, value): return value @staticmethod - def _bool_or_not(value): + def _true_or_not(value): try: return value.lower() in Configuration.TRUTHS except AttributeError: diff --git a/ush/rocoto/hosts/hera.yaml b/ush/rocoto/hosts/hera.yaml index ad7c3c0ffab..1916c5e89c2 100644 --- a/ush/rocoto/hosts/hera.yaml +++ b/ush/rocoto/hosts/hera.yaml @@ -6,7 +6,7 @@ comroot: '/scratch1/NCEPDEV/global/glopara/com' homedir: '/scratch1/NCEPDEV/global/$USER' stmp: '/scratch1/NCEPDEV/stmp2/$USER' ptmp: '/scratch1/NCEPDEV/stmp4/$USER' -noscrub': $HOMEDIR +noscrub: $HOMEDIR account': fv3-cpu scheduler: slurm queue: batch diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index 02c355f64e7..f4d7f67dffb 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -17,8 +17,8 @@ def input_args(): """ description = """ - Sources configuration files based on application and\n - creates "PSLOT.xml" for use with Rocoto.\n + Sources configuration files based on application and + creates "$PSLOT.xml" for use with Rocoto. """ parser = ArgumentParser(description=description, From 6b1fbe6889bd249a901060c4a0acbb3d4313d51c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 1 Jul 2022 09:50:49 -0400 Subject: [PATCH 16/41] Use `pprint` in configuration.py. Use `MODE` from `config.base` instead of user argument in `setup_xml.py` --- ush/rocoto/applications.py | 16 ++++++++-------- ush/rocoto/configuration.py | 4 ++-- ush/rocoto/setup_xml.py | 16 ++++------------ 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 4ba1d8c2980..b38663d52da 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -78,19 +78,19 @@ class AppConfig: VALID_MODES = ['cycled', 'forecast-only'] - def __init__(self, mode: str, configuration: Configuration) -> None: - - if mode not in self.VALID_MODES: - raise NotImplementedError(f'{mode} is not a valid application mode.\n' + - 'Valid application modes are:\n' + - f'{", ".join(self.VALID_MODES)}') - - self.mode = mode + def __init__(self, configuration: Configuration) -> None: self.scheduler = Host().scheduler _base = configuration.parse_config('config.base') + self.mode = _base['MODE'] + + if self.mode not in self.VALID_MODES: + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + + 'Valid application modes are:\n' + + f'{", ".join(self.VALID_MODES)}') + self.model_app = _base.get('APP', 'ATM') self.do_hybvar = _base.get('DOHYBVAR', False) self.do_wave = _base.get('DO_WAVE', False) diff --git a/ush/rocoto/configuration.py b/ush/rocoto/configuration.py index 74dc890fdc4..ba027d47fb2 100644 --- a/ush/rocoto/configuration.py +++ b/ush/rocoto/configuration.py @@ -4,6 +4,7 @@ import random import glob import subprocess +from pprint import pprint from datetime import datetime from pathlib import Path from typing import Union, List, Dict, Any @@ -106,8 +107,7 @@ def print_config(self, files: Union[str, bytes, list]) -> None: :return: None """ config = self.parse_config(files) - line_separator = '\n' - print(f'{line_separator.join(f"{key}: {config[key]}" for key in config.keys())}') + pprint(config, width=4) @classmethod def _get_script_env(cls, scripts: List) -> Dict[str, Any]: diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index f4d7f67dffb..73fb6ff82cb 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -24,19 +24,11 @@ def input_args(): parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) - # Set up sub-parsers for various modes of experimentation - subparser = parser.add_subparsers(dest='mode') - cycled = subparser.add_parser( - 'cycled', help='arguments for cycled mode') - forecasts = subparser.add_parser( - 'forecast-only', help='arguments for forecast-only mode') - # Common arguments across all modes - for subp in [cycled, forecasts]: - subp.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, - required=False, default=os.environ['PWD']) + parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, + required=False, default=os.environ['PWD']) - forecasts.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', + parser.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) args = parser.parse_args() @@ -62,7 +54,7 @@ def check_expdir(cmd_expdir, cfg_expdir): check_expdir(user_inputs.expdir, cfg.parse_config('config.base')['EXPDIR']) # Configure the application - app_config = AppConfig(user_inputs.mode, cfg) + app_config = AppConfig(cfg) # Create Rocoto Tasks and Assemble them into an XML xml = RocotoXML(app_config) From 81cb5f5a47d3de8d0d20c03f9bf283bc9992408c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 1 Jul 2022 16:04:56 -0400 Subject: [PATCH 17/41] Reviewer comments addressed. --- ush/rocoto/applications.py | 6 +++--- ush/rocoto/setup_xml.py | 7 ++----- ush/rocoto/workflow_tasks.py | 14 +++++++++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index b38663d52da..3fa7c197f72 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -249,8 +249,8 @@ def _cycled_upd_base(base_in): def _forecast_only_upd_base(base_in): base_out = base_in.copy() - base_out['INTERVAL'] = get_gfs_interval(base_in['gfs_cyc']) - base_out['CDUMP'] = 'gfs' # TODO - is there a use case for CDUMP=gdas? + base_out['INTERVAL_GFS'] = get_gfs_interval(base_in['gfs_cyc']) + base_out['CDUMP'] = 'gfs' return base_out @@ -404,7 +404,7 @@ def _get_forecast_only_task_names(self): tasks += ['getic'] tasks += ['init'] - if 'AERO' in self.model_app: + if self.do_aero: tasks += ['aerosol_init'] if self.do_wave: diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py index 73fb6ff82cb..74d1b4eed27 100644 --- a/ush/rocoto/setup_xml.py +++ b/ush/rocoto/setup_xml.py @@ -25,11 +25,8 @@ def input_args(): formatter_class=ArgumentDefaultsHelpFormatter) # Common arguments across all modes - parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, - required=False, default=os.environ['PWD']) - - parser.add_argument('--cdump', help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', - required=False) + parser.add_argument('expdir', help='full path to experiment directory containing config files', + type=str, default=os.environ['PWD']) args = parser.parse_args() diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index a4f5a7a181b..abd54c26b8e 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -288,8 +288,16 @@ def aerosol_init(self): dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) + # Calculate offset based on CDUMP = gfs | gdas + interval = None + if self.cdump in ['gfs']: + interval = self._base['INTERVAL_GFS'] + elif self.cdump in ['gdas']: + interval = self._base['INTERVAL'] + offset = f'-{interval}' + # Previous cycle - dep_dict = {'type': 'cycleexist', 'offset': f'-{self._base["INTERVAL"]}'} + dep_dict = {'type': 'cycleexist', 'offset': offset} deps.append(rocoto.add_dependency(dep_dict)) # Files from previous cycle @@ -299,8 +307,8 @@ def aerosol_init(self): for file in files: data = [f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/RERUN_RESTART/', file] - offset = [f'-{self._base["INTERVAL"]}', None] - dep_dict = {'type': 'data', 'data': data, 'offset': offset} + offset_list = [f'{offset}', None] + dep_dict = {'type': 'data', 'data': data, 'offset': [offset, None]} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 4551f853bd4811f0a93e92a2073a7f60a8e9fd8e Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 1 Jul 2022 16:13:12 -0400 Subject: [PATCH 18/41] Remove "Orion" fake and hard-coded path. --- ush/rocoto/hosts.py | 3 +-- ush/rocoto/test_configuration.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index dca063c25da..7a158fd0c10 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -36,8 +36,7 @@ def __init__(self, host=None): @property def detect(cls): - machine = 'NOTFOUND' # TODO - remove this comment - machine = 'ORION' # TODO - remove this line + machine = 'NOTFOUND' if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' diff --git a/ush/rocoto/test_configuration.py b/ush/rocoto/test_configuration.py index 2a64568c608..f210ceefa4e 100644 --- a/ush/rocoto/test_configuration.py +++ b/ush/rocoto/test_configuration.py @@ -1,7 +1,8 @@ +import sys from configuration import Configuration -expdir = '/Users/rmahajan/scratch/gwWork/EXPDIR/test-workflow' +expdir = sys.argv[1] cfg = Configuration(expdir) From d4a4c5e461800f2470c21320f884095f53b4fdc3 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 1 Jul 2022 17:00:53 -0400 Subject: [PATCH 19/41] validate user provided host with detected host. --- ush/rocoto/hosts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index 7a158fd0c10..959cd2f51f1 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -28,7 +28,12 @@ class Host: def __init__(self, host=None): - self.machine = self.detect if host is None else host + detected_host = self.detect + + if host is not None and host != detected_host: + raise ValueError(f'detected host: "{detected_host}" does not match host: "{host}"') + + self.machine = detected_host self.info = self._get_info self.scheduler = self.info['scheduler'] From def8b765cafdf70ce6121a22826e8dfc2fc08735 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 7 Jul 2022 16:00:48 -0400 Subject: [PATCH 20/41] stray mark and method to detect host --- ush/rocoto/hosts.py | 3 +-- ush/rocoto/hosts/hera.yaml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index 959cd2f51f1..22de4cfe0e0 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -28,7 +28,7 @@ class Host: def __init__(self, host=None): - detected_host = self.detect + detected_host = self.detect() if host is not None and host != detected_host: raise ValueError(f'detected host: "{detected_host}" does not match host: "{host}"') @@ -38,7 +38,6 @@ def __init__(self, host=None): self.scheduler = self.info['scheduler'] @classmethod - @property def detect(cls): machine = 'NOTFOUND' diff --git a/ush/rocoto/hosts/hera.yaml b/ush/rocoto/hosts/hera.yaml index 1916c5e89c2..08beeb6bd5a 100644 --- a/ush/rocoto/hosts/hera.yaml +++ b/ush/rocoto/hosts/hera.yaml @@ -7,7 +7,7 @@ homedir: '/scratch1/NCEPDEV/global/$USER' stmp: '/scratch1/NCEPDEV/stmp2/$USER' ptmp: '/scratch1/NCEPDEV/stmp4/$USER' noscrub: $HOMEDIR -account': fv3-cpu +account: fv3-cpu scheduler: slurm queue: batch queue_service: service @@ -16,4 +16,4 @@ chgrp_rstprod: YES chgrp_cmd: 'chgrp rstprod' hpssarch: YES localarch: NO -atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' \ No newline at end of file +atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' From d86b1f5e70d10469fbab53ecb4a01d97c6abfece Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 11 Jul 2022 11:25:27 -0400 Subject: [PATCH 21/41] change perms on py scripts --- ush/rocoto/rocoto.py | 0 ush/rocoto/setup_xml.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 ush/rocoto/rocoto.py mode change 100644 => 100755 ush/rocoto/setup_xml.py diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py old mode 100755 new mode 100644 diff --git a/ush/rocoto/setup_xml.py b/ush/rocoto/setup_xml.py old mode 100644 new mode 100755 From 1a2cf0fd0c060bea1680f61a4e9e6a420754717e Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 11 Jul 2022 11:42:27 -0400 Subject: [PATCH 22/41] add aerosol coupled app --- ush/rocoto/applications.py | 2 +- ush/rocoto/workflow_xml.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 3fa7c197f72..56c499df203 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -206,7 +206,7 @@ def _forecast_only_configs(self): configs = ['fcst', 'post', 'vrfy', 'arch'] - if self.model_app in ['S2S', 'S2SW']: + if self.model_app in ['S2S', 'S2SW', 'S2SWA']: configs += ['coupled_ic'] else: configs += ['init'] diff --git a/ush/rocoto/workflow_xml.py b/ush/rocoto/workflow_xml.py index d7bbd5d418d..eb92b7bde3c 100644 --- a/ush/rocoto/workflow_xml.py +++ b/ush/rocoto/workflow_xml.py @@ -153,21 +153,22 @@ def _assemble_xml(self) -> str: return ''.join(strings) - def write(self): - self._write_xml() - self._write_crontab() + def write(self, xml_file: str = None, crontab_file: str = None): + self._write_xml(xml_file = xml_file) + self._write_crontab(crontab_file = crontab_file) def _write_xml(self, xml_file: str = None) -> None: + expdir = self._base['EXPDIR'] + pslot = self._base['PSLOT'] + if xml_file is None: - expdir = self._base['EXPDIR'] - pslot = self._base['PSLOT'] xml_file = f"{expdir}/{pslot}.xml" with open(xml_file, 'w') as fh: fh.write(self.xml) - def _write_crontab(self, cronint: int = 5) -> None: + def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: """ Create crontab to execute rocotorun every cronint (5) minutes """ @@ -196,7 +197,10 @@ def _write_crontab(self, cronint: int = 5) -> None: '#################################################################', ''] - with open(os.path.join(expdir, f'{pslot}.crontab'), 'w') as fh: + if crontab_file is None: + crontab_file = f"{expdir}/{pslot}.crontab" + + with open(crontab_file, 'w') as fh: fh.write('\n'.join(strings)) return From 3333e7a01ae2992f66b11709a258069350dcc21a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 11 Jul 2022 14:04:10 -0400 Subject: [PATCH 23/41] envars were somehow updated past postenvars. perform deep copy of envars for local use. --- ush/rocoto/workflow_tasks.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index abd54c26b8e..ce7216512e2 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -501,10 +501,9 @@ def _get_postgroups(cdump, config, add_anl=False): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - postenvars = self.envars + postenvars = self.envars.copy() postenvar_dict = {'FHRGRP': '#grp#', - 'FHRLST': '#lst#', - 'ROTDIR': self._base.get('ROTDIR')} + 'FHRLST': '#lst#'} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -711,7 +710,7 @@ def _get_awipsgroups(cdump, config): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - awipsenvars = self.envars + awipsenvars = self.envars.copy() awipsenvar_dict = {'FHRGRP': '#grp#', 'FHRLST': '#lst#', 'ROTDIR': self._base.get('ROTDIR')} @@ -757,7 +756,7 @@ def metp(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - metpenvars = self.envars + metpenvars = self.envars.copy() metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS'), # TODO - in Forecast-only, this is `SDATE` on the RHS 'METPCASE': '#metpcase#'} @@ -816,7 +815,7 @@ def eomg(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - eomgenvars = self.envars + eomgenvars = self.envars.copy() eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['eobs']['NMEM_EOMGGRP']) @@ -888,7 +887,7 @@ def _get_ecengroups(): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - ecenenvars = self.envars + ecenenvars = self.envars.copy() ecenenvar_dict = {'FHRGRP': '#grp#', 'FHRLST': '#lst#'} for key, value in ecenenvar_dict.items(): @@ -935,7 +934,7 @@ def efcs(self): dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) - efcsenvars = self.envars + efcsenvars = self.envars.copy() efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['efcs']['NMEM_EFCSGRP']) @@ -990,7 +989,7 @@ def _get_eposgroups(epos): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - eposenvars = self.envars + eposenvars = self.envars.copy() eposenvar_dict = {'FHRGRP': '#grp#', 'FHRLST': '#lst#'} for key, value in eposenvar_dict.items(): @@ -1015,7 +1014,7 @@ def earc(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - earcenvars = self.envars + earcenvars = self.envars.copy() earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['earc']['NMEM_EARCGRP']) From d4be7f706f3c85c1556f2ddb98b6a2187eb5fea0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 12 Jul 2022 09:35:17 -0400 Subject: [PATCH 24/41] fix eamn grp variable in metatask. --- ush/rocoto/workflow_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index ce7216512e2..449efd1ce0b 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -51,9 +51,9 @@ def _set_envars(envar_dict) -> list: return envars @staticmethod - def _get_hybgroups(nens: int, nmem_per_group: int): + def _get_hybgroups(nens: int, nmem_per_group: int, start_index: int = 1): ngrps = nens / nmem_per_group - groups = ' '.join([f'{x:02d}' for x in range(1, int(ngrps) + 1)]) + groups = ' '.join([f'{x:02d}' for x in range(start_index, int(ngrps) + 1)]) return groups @staticmethod @@ -458,7 +458,7 @@ def _post_task(self, task_name, add_anl_to_post=False): if task_name not in ['post', 'ocnpost']: raise KeyError(f'Invalid post-processing task: {task_name}') - if task_name in 'ocnpost': + if task_name in ['ocnpost']: add_anl_to_post = False def _get_postgroups(cdump, config, add_anl=False): @@ -1017,7 +1017,7 @@ def earc(self): earcenvars = self.envars.copy() earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['earc']['NMEM_EARCGRP']) + groups = self._get_hybgroups(self._base['NMEM_ENKF'], self._configs['earc']['NMEM_EARCGRP'], start_index=0) resources = self.get_resource('earc') task = create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, From a7bc67d70689ab225aee4bdf89cb2f9a69d42e19 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 14 Jul 2022 23:04:59 -0400 Subject: [PATCH 25/41] update fcst_forecast_only and aerosol_init dependencies --- ush/rocoto/workflow_tasks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 449efd1ce0b..3d781ee6cb1 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -403,12 +403,16 @@ def _fcst_forecast_only(self): dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: - dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} + wave_job = 'waveprep' if self.app_config.model_app in ['ATMW'] else 'waveinit' + dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero: # TODO - check dependency on cycleexist of previous cycle? + if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} dependencies.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) From 58b0aa614dd513aba57e5719fba8cd266e3a81e7 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 14 Jul 2022 23:35:33 -0400 Subject: [PATCH 26/41] Fix do_ocean variable logical in the applications.py --- ush/rocoto/applications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 56c499df203..673fa26f880 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -94,7 +94,7 @@ def __init__(self, configuration: Configuration) -> None: self.model_app = _base.get('APP', 'ATM') self.do_hybvar = _base.get('DOHYBVAR', False) self.do_wave = _base.get('DO_WAVE', False) - self.do_ocean = _base.get('DO_OCEAN', False) + self.do_ocean = _base.get('DO_OCN', False) self.do_ice = _base.get('DO_ICE', False) self.do_aero = _base.get('DO_AERO', False) self.do_gldas = _base.get('DO_GLDAS', False) From ca88c5f5b4ed164d2e7e252e2678cf97638471e8 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 12:20:16 -0400 Subject: [PATCH 27/41] add `sfcanl` task. --- ush/rocoto/applications.py | 4 ++-- ush/rocoto/workflow_tasks.py | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 673fa26f880..fdca3661fff 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -168,7 +168,7 @@ def _cycled_configs(self): """ configs = ['prep', - 'anal', 'analdiag', 'analcalc', + 'anal', 'sfcanl', 'analdiag', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] if self.do_gldas: @@ -309,7 +309,7 @@ def _get_cycled_task_names(self): This is the place where that order is set. """ - gdas_gfs_common_tasks_before_fcst = ['prep', 'anal', 'analcalc'] + gdas_gfs_common_tasks_before_fcst = ['prep', 'anal', 'sfcanl', 'analcalc'] gdas_gfs_common_tasks_after_fcst = ['post', 'vrfy'] gdas_gfs_common_cleanup_tasks = ['arch'] diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 3d781ee6cb1..101f7117509 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -11,7 +11,7 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc', 'getic'] VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init', - 'prep', 'anal', 'analcalc', 'analdiag', 'gldas', 'arch', + 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'gldas', 'arch', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', @@ -307,7 +307,6 @@ def aerosol_init(self): for file in files: data = [f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/RERUN_RESTART/', file] - offset_list = [f'{offset}', None] dep_dict = {'type': 'data', 'data': data, 'offset': [offset, None]} deps.append(rocoto.add_dependency(dep_dict)) @@ -334,11 +333,25 @@ def anal(self): return task + def sfcanl(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('sfcanl') + task = create_wf_task('sfcanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + def analcalc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} + deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -366,7 +379,7 @@ def analdiag(self): def gldas(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -424,7 +437,7 @@ def _fcst_forecast_only(self): @property def _fcst_cycled(self): - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} dep = rocoto.add_dependency(dep_dict) dependencies = rocoto.create_dependency(dep=dep) From d91f9634b5ac3c16a24308ef672f40ff3a022da0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 12:29:30 -0400 Subject: [PATCH 28/41] fix type mismatch and over-indented block. --- ush/rocoto/hosts.py | 2 +- ush/rocoto/workflow_tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/rocoto/hosts.py b/ush/rocoto/hosts.py index 22de4cfe0e0..c54112dce10 100644 --- a/ush/rocoto/hosts.py +++ b/ush/rocoto/hosts.py @@ -31,7 +31,7 @@ def __init__(self, host=None): detected_host = self.detect() if host is not None and host != detected_host: - raise ValueError(f'detected host: "{detected_host}" does not match host: "{host}"') + raise ValueError(f'detected host: "{detected_host}" does not match host: "{host}"') self.machine = detected_host self.info = self._get_info diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 101f7117509..5b92cd04423 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -706,8 +706,8 @@ def _get_awipsgroups(cdump, config): fhmax = 240 if fhmax_hf > 240: fhmax_hf = 240 - fhrs_hf = range(fhmin, fhmax_hf + fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout) + fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) + fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) nawipsgrp = config['NAWIPSGRP'] ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) From cfd8f48a60d53026ba2c795380f5696678c9ff5c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 12:32:35 -0400 Subject: [PATCH 29/41] remove obsolete fv3gfs_workflow.sh --- ush/rocoto/fv3gfs_workflow.sh | 49 ----------------------------------- 1 file changed, 49 deletions(-) delete mode 100755 ush/rocoto/fv3gfs_workflow.sh diff --git a/ush/rocoto/fv3gfs_workflow.sh b/ush/rocoto/fv3gfs_workflow.sh deleted file mode 100755 index 7331b646f77..00000000000 --- a/ush/rocoto/fv3gfs_workflow.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# Checkout, build, setup and execute the workflow - -set -ex - -fv3gfs_tag="https://svnemc.ncep.noaa.gov/projects/fv3gfs/trunk" - -pslot="fv3test" -expdir="/path/to/expdir" -comrot="/path/to/comrot" -fv3gfs="/path/to/fv3gfs_tag/checkout" -idate="2017073118" -edate="2017080112" - -###################################### -# USER NEED NOT MODIFY BELOW THIS LINE -###################################### - -if [ -d /gpfs/hps3 ]; then - machine="cray" - icsdir="/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS" -else - echo "Unknown machine $machine, ABORT!" - exit -1 -fi - -[[ -d $expdir/$pslot ]] && rm -rf $expdir/$pslot -[[ -d $comrot/$pslot ]] && rm -rf $comrot/$pslot -[[ -d $fv3gfs/$pslot ]] && rm -rf $fv3gfs/$pslot - -gfs_ver=v15.0.0 -mkdir -p $fv3gfs -cd $fv3gfs -git clone --recursive gerrit:fv3gfs gfs.${gfs_ver} - -cd $fv3gfs/gfs.${gfs_ver}/sorc -sh checkout.sh -sh build_all.sh $machine -sh link_fv3gfs.sh emc $machine - -cd $fv3gfs/gfs.${gfs_ver}/ush/rocoto -python setup_expt.py --pslot $pslot --comrot $comrot --expdir $expdir --idate $idate --edate $edate --icsdir $icsdir --configdir ../parm/config -python setup_workflow.py --expdir $expdir/$pslot - -cd $expdir/$pslot -crontab $pslot.crontab - -exit From 07e8da091cdd587053d01158c2690c51efd10dd7 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 13:45:30 -0400 Subject: [PATCH 30/41] move DOBNDPNT_WAVE toggle to config.base --- parm/config/config.base.emc.dyn | 3 ++- parm/config/config.base.nco.static | 3 ++- parm/config/config.wave | 11 +++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index f44f4ecee39..c48260abdd1 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -159,6 +159,7 @@ export DO_ICE="NO" export DO_AERO="NO" export CCPP_SUITE="FV3_GFS_v17_p8" export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both +export DOBNDPNT_WAVE="NO" export cplwav2atm=".false." case "${APP}" in @@ -190,7 +191,7 @@ case "${APP}" in if [[ "$APP" =~ ^S2SW ]]; then export DO_WAVE="YES" - export WAVE_CDUMP="both" + export WAVE_CDUMP="both" export cplwav2atm=".true." export confignamevarfornems="${confignamevarfornems}_wave" fi diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 432e95cc327..4980ecf7527 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -161,7 +161,7 @@ export FHMAX_HF_GFS=120 export FHOUT_HF_GFS=1 export ILPOST=1 # gempak output frequency up to F120 -# frequency for saving restart files. set to 6,12,24,48 etc +# frequency for saving restart files. set to 6,12,24,48 etc export restart_interval_gfs=12 # I/O QUILTING, true--use Write Component; false--use GFDL FMS @@ -209,6 +209,7 @@ export gldas_cyc=00 # run wave component export DO_WAVE=YES export WAVE_CDUMP="both" +export DOBNDPNT_WAVE="YES" # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL export imp_physics=8 diff --git a/parm/config/config.wave b/parm/config/config.wave index 40809b30c7e..a3b06a8041b 100755 --- a/parm/config/config.wave +++ b/parm/config/config.wave @@ -30,7 +30,6 @@ export waveGRD=${waveGRD:-'gnh_10m aoc_9km gsh_15m'} export waveGRDN=${waveGRDN:-'1 2 3'} # gridnumber for ww3_multi export waveGRDG=${waveGRDG:-'10 20 30'} # gridgroup for ww3_multi export USE_WAV_RMP=${USE_WAV_RMP:-'YES'} #yes/no rmp grid remapping pre-processed coefficients -export DOBNDPNT_WAVE=${DOBNDPNT_WAVE:-'YES'} export waveMULTIGRID=${waveMULTIGRID:-'.true.'} export MESH_WAV=${MESH_WAV:-'mesh.gwes_30m.nc'} @@ -84,20 +83,20 @@ fi # Restart timing business export RSTTYPE_WAV='T' # generate second tier of restart files -if [ "${CDUMP}" != gfs ]; then # Setting is valid for GDAS and GEFS +if [ "${CDUMP}" != gfs ]; then # Setting is valid for GDAS and GEFS export DT_1_RST_WAV=10800 # time between restart files, set to DTRST=1 for a single restart file export DT_2_RST_WAV=43200 # restart stride for checkpointing restart export RSTIOFF_WAV=0 # first restart file offset relative to model start else # This is a GFS run rst_dt_gfs=$(( restart_interval_gfs * 3600 )) - if [ $rst_dt_gfs -gt 0 ]; then + if [ $rst_dt_gfs -gt 0 ]; then export DT_1_RST_WAV=${rst_dt_gfs:-0} # time between restart files, set to DTRST=1 for a single restart file export DT_2_RST_WAV=${rst_dt_gfs:-0} # restart stride for checkpointing restart - else + else rst_dt_fhmax=$(( FHMAX_WAV * 3600 )) export DT_1_RST_WAV=0 # time between restart files, set to DTRST=1 for a single restart file - export DT_2_RST_WAV=${rst_dt_fhmax:-0} # use checkpoint restart file name for creating restart at end of run - fi + export DT_2_RST_WAV=${rst_dt_fhmax:-0} # use checkpoint restart file name for creating restart at end of run + fi export RSTIOFF_WAV=0 # first restart file offset relative to model start fi # From 50ab300f4dd6088d70f0d571c2cdc9b4be8f6bf6 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 14:39:47 -0400 Subject: [PATCH 31/41] wavebndpnt toggle has changed in config.base. Handle that. --- ush/rocoto/applications.py | 25 +++++++++++++++++++------ ush/rocoto/workflow_tasks.py | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index fdca3661fff..accc7306ee3 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -129,6 +129,8 @@ def __init__(self, configuration: Configuration) -> None: # Get more configuration options into the class attributes self.gfs_cyc = self._base.get('gfs_cyc') + self.lobsdiag_forenkf = False + self.eupd_cdumps = None if self.do_hybvar: self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) eupd_cdump = self._base.get('EUPD_CYC', 'gdas').lower() @@ -137,13 +139,15 @@ def __init__(self, configuration: Configuration) -> None: elif eupd_cdump in ['gfs', 'gdas']: self.eupd_cdumps = [eupd_cdump] + self.do_wave_bnd = False + self.wave_cdumps = None if self.do_wave: wave_cdump = self._base.get('WAVE_CDUMP', 'BOTH').lower() if wave_cdump in ['both']: self.wave_cdumps = ['gfs', 'gdas'] elif wave_cdump in ['gfs', 'gdas']: self.wave_cdumps = [wave_cdump] - self.do_wave_bnd = self.configs['wavepostsbs'].get('DOBNDPNT_WAVE', False) + self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) # Finally get task names for the application self.task_names = self.get_task_names() @@ -187,7 +191,9 @@ def _cycled_configs(self): configs += ['awips'] if self.do_wave: - configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostpnt'] + if self.do_wave_bnd: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] if self.do_gempak: configs += ['wavegempak'] if self.do_awips: @@ -229,7 +235,9 @@ def _forecast_only_configs(self): configs += ['awips'] if self.do_wave: - configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostpnt'] + if self.do_wave_bnd: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] if self.do_gempak: configs += ['wavegempak'] if self.do_awips: @@ -315,7 +323,8 @@ def _get_cycled_task_names(self): gldas_tasks = ['gldas'] wave_prep_tasks = ['waveinit', 'waveprep'] - wave_post_tasks = ['wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] + wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] hybrid_gdas_or_gfs_tasks = [] hybrid_gdas_tasks = [] @@ -330,7 +339,7 @@ def _get_cycled_task_names(self): if self.do_gldas: gdas_tasks += gldas_tasks - if self.do_wave: + if self.do_wave and 'gdas' in self.wave_cdumps: gdas_tasks += wave_prep_tasks gdas_tasks += ['fcst'] @@ -342,7 +351,9 @@ def _get_cycled_task_names(self): gdas_tasks += hybrid_gdas_or_gfs_tasks gdas_tasks += hybrid_gdas_tasks - if self.do_wave: + if self.do_wave and 'gdas' in self.wave_cdumps: + if self.do_wave_bnd: + gdas_tasks += wave_bndpnt_tasks gdas_tasks += wave_post_tasks gdas_tasks += gdas_gfs_common_cleanup_tasks @@ -364,6 +375,8 @@ def _get_cycled_task_names(self): gfs_tasks += hybrid_gdas_or_gfs_tasks if self.do_wave and 'gfs' in self.wave_cdumps: + if self.do_wave_bnd: + gfs_tasks += wave_bndpnt_tasks gfs_tasks += wave_post_tasks if self.do_gempak: gfs_tasks += ['wavegempak'] diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 5b92cd04423..2002d9b34b7 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -802,7 +802,7 @@ def arch(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_wave_bnd and self.cdump in ['gfs']: + if self.app_config.do_wave_bnd: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 6fcd71eabb57e42c8f4d32ccbd96c38e262b97e0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 15:18:32 -0400 Subject: [PATCH 32/41] move a section in applications constructor where it makes sense. --- ush/rocoto/applications.py | 39 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index accc7306ee3..d4331bc6651 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -94,6 +94,7 @@ def __init__(self, configuration: Configuration) -> None: self.model_app = _base.get('APP', 'ATM') self.do_hybvar = _base.get('DOHYBVAR', False) self.do_wave = _base.get('DO_WAVE', False) + self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) self.do_ocean = _base.get('DO_OCN', False) self.do_ice = _base.get('DO_ICE', False) self.do_aero = _base.get('DO_AERO', False) @@ -107,6 +108,24 @@ def __init__(self, configuration: Configuration) -> None: self.do_hpssarch = _base.get('HPSSARCH', False) + self.wave_cdumps = None + if self.do_wave: + wave_cdump = _base.get('WAVE_CDUMP', 'BOTH').lower() + if wave_cdump in ['both']: + self.wave_cdumps = ['gfs', 'gdas'] + elif wave_cdump in ['gfs', 'gdas']: + self.wave_cdumps = [wave_cdump] + + self.lobsdiag_forenkf = False + self.eupd_cdumps = None + if self.do_hybvar: + self.lobsdiag_forenkf = _base.get('lobsdiag_forenkf', False) + eupd_cdump = _base.get('EUPD_CYC', 'gdas').lower() + if eupd_cdump in ['both']: + self.eupd_cdumps = ['gfs', 'gdas'] + elif eupd_cdump in ['gfs', 'gdas']: + self.eupd_cdumps = [eupd_cdump] + # Get a list of all possible config_files that would be part of the application self.configs_names = self._get_app_configs() @@ -129,26 +148,6 @@ def __init__(self, configuration: Configuration) -> None: # Get more configuration options into the class attributes self.gfs_cyc = self._base.get('gfs_cyc') - self.lobsdiag_forenkf = False - self.eupd_cdumps = None - if self.do_hybvar: - self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) - eupd_cdump = self._base.get('EUPD_CYC', 'gdas').lower() - if eupd_cdump in ['both']: - self.eupd_cdumps = ['gfs', 'gdas'] - elif eupd_cdump in ['gfs', 'gdas']: - self.eupd_cdumps = [eupd_cdump] - - self.do_wave_bnd = False - self.wave_cdumps = None - if self.do_wave: - wave_cdump = self._base.get('WAVE_CDUMP', 'BOTH').lower() - if wave_cdump in ['both']: - self.wave_cdumps = ['gfs', 'gdas'] - elif wave_cdump in ['gfs', 'gdas']: - self.wave_cdumps = [wave_cdump] - self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) - # Finally get task names for the application self.task_names = self.get_task_names() From e5697ff08fe027fd3e6b686929fd8d1ec3c7db83 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 15:29:44 -0400 Subject: [PATCH 33/41] update forecast only tasks after do_wave_bnd change. --- ush/rocoto/applications.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index d4331bc6651..2250314ffb0 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -433,7 +433,9 @@ def _get_forecast_only_task_names(self): tasks += ['metp'] if self.do_wave: - tasks += ['wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] + if self.do_wave_bnd: + tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] + tasks += ['wavepostsbs', 'wavepostpnt'] if self.do_gempak: tasks += ['wavegempak'] if self.do_awips: From b3993d801ac9071adcff8a1f714834f88c95f418 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 17:57:08 -0400 Subject: [PATCH 34/41] fix forecast only dependencies for fcst task. --- ush/rocoto/workflow_tasks.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 2002d9b34b7..1faa413ae34 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -406,6 +406,7 @@ def fcst(self): @property def _fcst_forecast_only(self): + dependencies = [] deps = [] data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc' dep_dict = {'type': 'data', 'data': data} @@ -413,7 +414,7 @@ def _fcst_forecast_only(self): data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + dependencies.append(rocoto.create_dependency(dep_condition='or', dep=deps)) if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: wave_job = 'waveprep' if self.app_config.model_app in ['ATMW'] else 'waveinit' @@ -421,11 +422,19 @@ def _fcst_forecast_only(self): dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: + # Calculate offset based on CDUMP = gfs | gdas + interval = None + if self.cdump in ['gfs']: + interval = self._base['INTERVAL_GFS'] + elif self.cdump in ['gdas']: + interval = self._base['INTERVAL'] + offset = f'-{interval}' + deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} - dependencies.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies.append(rocoto.create_dependency(dep_condition='or', dep=deps)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) From 9ef325b70bc8c54a1ae1fe7625c7ad90a145c971 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 18:19:28 -0400 Subject: [PATCH 35/41] remove waveprep in forecast-only applicaiton. verify later. --- ush/rocoto/applications.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/rocoto/applications.py b/ush/rocoto/applications.py index 2250314ffb0..e91475e36f9 100644 --- a/ush/rocoto/applications.py +++ b/ush/rocoto/applications.py @@ -420,7 +420,8 @@ def _get_forecast_only_task_names(self): tasks += ['aerosol_init'] if self.do_wave: - tasks += ['waveinit', 'waveprep'] + tasks += ['waveinit'] + # tasks += ['waveprep'] # TODO - verify if waveprep is executed in forecast-only mode when APP=ATMW|S2SW tasks += ['fcst'] From e7b8e751c893c1f6fb3006242d1e1779616bbe58 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 15 Jul 2022 22:26:40 -0400 Subject: [PATCH 36/41] yaml turns YES/NO into True/False unless they are put in '' e.g. 'YES' will remail YES. --- ush/rocoto/hosts/hera.yaml | 6 +++--- ush/rocoto/hosts/orion.yaml | 6 +++--- ush/rocoto/hosts/wcoss_dell_p3.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ush/rocoto/hosts/hera.yaml b/ush/rocoto/hosts/hera.yaml index 08beeb6bd5a..84debf1b9aa 100644 --- a/ush/rocoto/hosts/hera.yaml +++ b/ush/rocoto/hosts/hera.yaml @@ -12,8 +12,8 @@ scheduler: slurm queue: batch queue_service: service partition_batch: hera -chgrp_rstprod: YES +chgrp_rstprod: 'YES' chgrp_cmd: 'chgrp rstprod' -hpssarch: YES -localarch: NO +hpssarch: 'YES' +localarch: 'NO' atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' diff --git a/ush/rocoto/hosts/orion.yaml b/ush/rocoto/hosts/orion.yaml index d0837c5c6d0..bf59ea3e1d6 100644 --- a/ush/rocoto/hosts/orion.yaml +++ b/ush/rocoto/hosts/orion.yaml @@ -12,8 +12,8 @@ account: fv3-cpu queue: batch queue_service: service partition_batch: orion -chgrp_rstprod: YES +chgrp_rstprod: 'YES' chgrp_cmd: 'chgrp rstprod' -hpssarch: NO -localarch: NO +hpssarch: 'NO' +localarch: 'NO' atardir: '$NOSCRUB/archive_rotdir/$PSLOT' \ No newline at end of file diff --git a/ush/rocoto/hosts/wcoss_dell_p3.yaml b/ush/rocoto/hosts/wcoss_dell_p3.yaml index 2fed540270e..daa9595c310 100644 --- a/ush/rocoto/hosts/wcoss_dell_p3.yaml +++ b/ush/rocoto/hosts/wcoss_dell_p3.yaml @@ -12,8 +12,8 @@ scheduler: lsf queue: dev queue_service: dev_transfer partition_batch: None -chgrp_rstprod: YES +chgrp_rstprod: 'YES' chgrp_cmd: 'chgrp rstprod' -hpssarch: YES -localarch: NO +hpssarch: 'YES' +localarch: 'NO' atardir: '/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT' \ No newline at end of file From 892bbe99399ff6da0e1588702343ebfed39c4585 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 16 Jul 2022 01:22:00 -0400 Subject: [PATCH 37/41] postanl dependency is never going to get satisfied with loganl --- ush/rocoto/workflow_tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 1faa413ae34..9a008a7652c 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -525,7 +525,9 @@ def _get_postgroups(cdump, config, add_anl=False): data = f'&ROTDIR;/{self.cdump}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.log#dep#.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) postenvars = self.envars.copy() postenvar_dict = {'FHRGRP': '#grp#', From bdd6477d27247cb05485f89213a8446d7aac80d9 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 16 Jul 2022 02:02:33 -0400 Subject: [PATCH 38/41] C48 timestep of 1200 is too big for some members. cut it down to 450 (same as C96). All members run w/ success --- parm/config/config.fv3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index e7f521440c1..8c4b74ee87f 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -41,7 +41,7 @@ fi # (Standard) Model resolution dependent variables case $case_in in "C48") - export DELTIM=1200 + export DELTIM=450 export layout_x=3 export layout_y=2 export layout_x_gfs=3 From 703d09efcfa40cdd76c211251de09a4f72730da1 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sat, 16 Jul 2022 13:45:41 -0400 Subject: [PATCH 39/41] eobs timed out with 15 mins on Orion. increase time --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 4a84518996a..47f9ee5bca4 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -428,7 +428,7 @@ elif [ $step = "coupled_ic" ]; then elif [ $step = "eobs" -o $step = "eomg" ]; then - export wtime_eobs="00:15:00" + export wtime_eobs="00:45:00" export wtime_eomg="01:00:00" if [ $CASE = "C768" ]; then export npe_eobs=200 From 89ff9c325d7d000b2b2bb8afc383dcb062b24633 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sun, 17 Jul 2022 09:27:47 -0400 Subject: [PATCH 40/41] add ROTDIR back as an envar set in xml for post.sh --- ush/rocoto/workflow_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/rocoto/workflow_tasks.py b/ush/rocoto/workflow_tasks.py index 9a008a7652c..b455ceb8871 100644 --- a/ush/rocoto/workflow_tasks.py +++ b/ush/rocoto/workflow_tasks.py @@ -531,7 +531,8 @@ def _get_postgroups(cdump, config, add_anl=False): postenvars = self.envars.copy() postenvar_dict = {'FHRGRP': '#grp#', - 'FHRLST': '#lst#'} + 'FHRLST': '#lst#', + 'ROTDIR': self._base.get('ROTDIR')} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) From 252062bffb4322b3dbbbe7af8c07e93cf6c2a042 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 18 Jul 2022 12:26:09 -0400 Subject: [PATCH 41/41] update GSI-utils hash to include bugfix for processing surface ensemble members with 3D fields --- sorc/checkout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index cdd63aa0ff5..978d932ba70 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -132,7 +132,7 @@ mkdir -p ${logdir} errs=0 checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-b97375c}" ; errs=$((errs + $?)) checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4" ; errs=$((errs + $?)) -checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "e0dc960" ; errs=$((errs + $?)) +checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b" ; errs=$((errs + $?)) checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870" ; errs=$((errs + $?)) checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62" ; errs=$((errs + $?)) checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?))