From 8328f1c4700f67b57f48e2a62b0f4441a8ceff70 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Mar 2023 22:29:27 -0500 Subject: [PATCH 1/5] add rocoto XML options --- workflow/rocoto/workflow_xml.py | 13 ++++++++----- workflow/setup_xml.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index 4e43ff8ec17..4ce005c17fc 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -5,16 +5,19 @@ from datetime import datetime from pygw.timetools import to_timedelta from collections import OrderedDict +from typing import Dict from applications import AppConfig from rocoto.workflow_tasks import get_wf_tasks import rocoto.rocoto as rocoto + class RocotoXML: - def __init__(self, app_config: AppConfig) -> None: + def __init__(self, app_config: AppConfig, rocoto: Dict) -> None: self._app_config = app_config + self.rocoto = rocoto self._base = self._app_config.configs['base'] @@ -60,7 +63,7 @@ def _get_definitions(self) -> str: entity['ROTDIR'] = self._base['ROTDIR'] entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['MAXTRIES'] = self._base.get('ROCOTO_MAXTRIES', 2) + entity['MAXTRIES'] = self.rocoto['maxtries'] # Put them all in an XML key-value syntax strings = [] @@ -75,9 +78,9 @@ def _get_workflow_header(self): """ scheduler = self._app_config.scheduler - cyclethrottle = self._base.get('ROCOTO_CYCLETHROTTLE', 3) - taskthrottle = self._base.get('ROCOTO_TASKTHROTTLE', 25) - verbosity = self._base.get('ROCOTO_VERBOSITY', 10) + cyclethrottle = self.rocoto['cyclethrottle'] + taskthrottle = self.rocoto['taskthrottle'] + verbosity = self.rocoto['verbosity'] expdir = self._base['EXPDIR'] diff --git a/workflow/setup_xml.py b/workflow/setup_xml.py index 09212b59d96..8ecc4552092 100755 --- a/workflow/setup_xml.py +++ b/workflow/setup_xml.py @@ -28,6 +28,15 @@ def input_args(): parser.add_argument('expdir', help='full path to experiment directory containing config files', type=str, default=os.environ['PWD']) + parser.add_argument('--maxtries', help='maximum number of retries', type=int, + default=2, required=False) + parser.add_argument('--cyclethrottle', help='maximum number of concurrent cycles', type=int, + default=3, required=False) + parser.add_argument('--taskthrottle', help='maximum number of concurrent tasks', type=int, + default=25, required=False) + parser.add_argument('--verbosity', help='verbosity level of Rocoto', type=int, + default=10, required=False) + args = parser.parse_args() return args @@ -45,6 +54,10 @@ def check_expdir(cmd_expdir, cfg_expdir): if __name__ == '__main__': user_inputs = input_args() + rocoto_param_dict = {'maxtries': user_inputs['maxtries'], + 'cyclethrottle': user_inputs['cyclethrottle'], + 'taskthrottle': user_inputs['taskthrottle'], + 'verbosity': user_inputs['verbosity']} cfg = Configuration(user_inputs.expdir) @@ -54,5 +67,5 @@ def check_expdir(cmd_expdir, cfg_expdir): app_config = AppConfig(cfg) # Create Rocoto Tasks and Assemble them into an XML - xml = RocotoXML(app_config) + xml = RocotoXML(app_config, rocoto_param_dict) xml.write() From 897c712dca4c48bbb10dbce8f93281051faf49f1 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Mar 2023 22:43:46 -0500 Subject: [PATCH 2/5] update documentation --- docs/source/setup.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 19e3f9ac3fe..a4e70fbfcb7 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -156,6 +156,8 @@ Example: ./setup_xml.py /some_safe_disk_area/Joe.Schmo/expdir/test +Additional options for setting up Rocoto are available with `setup_xml.py -h` that allow users to change the number of failed tries, number of concurrent cycles and tasks as well as Rocoto's verbosity levels. + **************************************** Step 4: Confirm files from setup scripts **************************************** From de85f5da3d07d63dc20b1efcf07c0e3a6663b7ac Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Mar 2023 22:47:03 -0500 Subject: [PATCH 3/5] norm fail check --- workflow/rocoto/workflow_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index 4ce005c17fc..a3231daa688 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -79,7 +79,7 @@ def _get_workflow_header(self): scheduler = self._app_config.scheduler cyclethrottle = self.rocoto['cyclethrottle'] - taskthrottle = self.rocoto['taskthrottle'] + taskthrottle = self.rocoto['taskthrottle'] verbosity = self.rocoto['verbosity'] expdir = self._base['EXPDIR'] From 815dcdf65dfa76d533c8984bc7e97d767c174fba Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 1 Mar 2023 22:50:19 -0500 Subject: [PATCH 4/5] deconflict w/ the rocoto module --- workflow/rocoto/workflow_xml.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index a3231daa688..856cade2b94 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -11,13 +11,12 @@ import rocoto.rocoto as rocoto - class RocotoXML: - def __init__(self, app_config: AppConfig, rocoto: Dict) -> None: + def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: self._app_config = app_config - self.rocoto = rocoto + self.rocoto_config = rocoto_config self._base = self._app_config.configs['base'] @@ -63,7 +62,7 @@ def _get_definitions(self) -> str: entity['ROTDIR'] = self._base['ROTDIR'] entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['MAXTRIES'] = self.rocoto['maxtries'] + entity['MAXTRIES'] = self.rocoto_config['maxtries'] # Put them all in an XML key-value syntax strings = [] @@ -78,9 +77,9 @@ def _get_workflow_header(self): """ scheduler = self._app_config.scheduler - cyclethrottle = self.rocoto['cyclethrottle'] - taskthrottle = self.rocoto['taskthrottle'] - verbosity = self.rocoto['verbosity'] + cyclethrottle = self.rocoto_config['cyclethrottle'] + taskthrottle = self.rocoto_config['taskthrottle'] + verbosity = self.rocoto_config['verbosity'] expdir = self._base['EXPDIR'] From 00272a0e4deecc3f68ee4e42fc0a9546c8526b32 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 2 Mar 2023 11:14:01 -0500 Subject: [PATCH 5/5] argparse arguments are not your typical dictionary. they are a collection --- workflow/setup_xml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/setup_xml.py b/workflow/setup_xml.py index 8ecc4552092..d43efe21e1f 100755 --- a/workflow/setup_xml.py +++ b/workflow/setup_xml.py @@ -54,10 +54,10 @@ def check_expdir(cmd_expdir, cfg_expdir): if __name__ == '__main__': user_inputs = input_args() - rocoto_param_dict = {'maxtries': user_inputs['maxtries'], - 'cyclethrottle': user_inputs['cyclethrottle'], - 'taskthrottle': user_inputs['taskthrottle'], - 'verbosity': user_inputs['verbosity']} + rocoto_param_dict = {'maxtries': user_inputs.maxtries, + 'cyclethrottle': user_inputs.cyclethrottle, + 'taskthrottle': user_inputs.taskthrottle, + 'verbosity': user_inputs.verbosity} cfg = Configuration(user_inputs.expdir)