From b009106e74514a123c973f65be3b702ed7d670a6 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 22 Dec 2023 16:43:24 -0600 Subject: [PATCH 01/10] Refactor rocot task XML creation Refactors the rocoto task generation to be recursive. This will allow nested metatasks to loop over multiple variables, which is needed for GEFS product generation. As part of this refactor, there is no longer separate arguments to designate metatasks. Instead, task dicts can include a nested 'task_dict' as well as a 'var_dict' containing the variables to loop over. The nested task dict can then either have another layer, or be the innermost task. To accomodate the new recursive nature, some defaults that were previously defined in `create_wf_task()` had to be pushed down into the function that creates the innermost task. Also, former keywords have been absorbed by the task dict. Refs #823 Refs #827 --- workflow/rocoto/gefs_tasks.py | 42 +- workflow/rocoto/gfs_tasks.py | 902 ++++++++++++++++++++++++++++------ workflow/rocoto/rocoto.py | 126 +++-- workflow/rocoto/tasks.py | 60 ++- 4 files changed, 919 insertions(+), 211 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 680c7d8686f..230e753e832 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -57,14 +57,25 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') - task = create_wf_task('stage_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'stage_ic', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task def waveinit(self): resources = self.get_resource('waveinit') - task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=None) + task_dict = {'task_name': 'waveinit', + 'cdump': self.cdump, + 'resources': resources, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -81,7 +92,13 @@ def fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -100,9 +117,24 @@ def efcs(self): efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['efcs']['NMEM_EFCSGRP']) + var_dict = {'grp': groups} resources = self.get_resource('efcs') - task = create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, - metatask='efmn', varname='grp', varval=groups, cycledef='gefs') + + task_dict = {'task_name': 'efcs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': efcsenvars, + 'cycledef': 'gefs' + } + + metatask_dict = {'task_name': 'efmn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = create_wf_task(metatask_dict) return task diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 18208983b87..539ac147b57 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -71,7 +71,14 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') - task = create_wf_task('stage_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'stage_ic', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -102,8 +109,15 @@ def prep(self): cycledef = 'gdas' resources = self.get_resource('prep') - task = create_wf_task('prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'prep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + + task = create_wf_task(task_dict) return task @@ -121,7 +135,15 @@ def waveinit(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump - task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) + + task_dict = {'task_name': 'waveinit', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -133,7 +155,14 @@ def waveprep(self): dependencies = rocoto.create_dependency(dep=deps) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('waveprep') - task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) + task_dict = {'task_name': 'waveprep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -172,8 +201,14 @@ def aerosol_init(self): cycledef = 'gfs_seq' resources = self.get_resource('aerosol_init') - task = create_wf_task('aerosol_init', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'aerosol_init', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -189,7 +224,13 @@ def anal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('anal') - task = create_wf_task('anal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'anal', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -209,7 +250,13 @@ def sfcanl(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('sfcanl') - task = create_wf_task('sfcanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'sfcanl', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -229,7 +276,13 @@ def analcalc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analcalc') - task = create_wf_task('analcalc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'analcalc', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -241,7 +294,13 @@ def analdiag(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analdiag') - task = create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'analdiag', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -253,7 +312,13 @@ def prepatmiodaobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('prepatmiodaobs') - task = create_wf_task('prepatmiodaobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'prepatmiodaobs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -277,8 +342,14 @@ def atmanlinit(self): cycledef = 'gdas' resources = self.get_resource('atmanlinit') - task = create_wf_task('atmanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'atmanlinit', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -290,7 +361,13 @@ def atmanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmanlrun') - task = create_wf_task('atmanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'atmanlrun', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -302,7 +379,13 @@ def atmanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmanlfinal') - task = create_wf_task('atmanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'atmanlfinal', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -314,7 +397,13 @@ def aeroanlinit(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') - task = create_wf_task('aeroanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'aeroanlinit', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task def aeroanlrun(self): @@ -325,7 +414,13 @@ def aeroanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('aeroanlrun') - task = create_wf_task('aeroanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'aeroanlrun', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -337,7 +432,13 @@ def aeroanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlfinal') - task = create_wf_task('aeroanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'aeroanlfinal', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -349,7 +450,13 @@ def preplandobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('preplandobs') - task = create_wf_task('preplandobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'preplandobs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -361,7 +468,13 @@ def landanl(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('landanl') - task = create_wf_task('landanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'landanl', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task def ocnanalprep(self): @@ -375,11 +488,13 @@ def ocnanalprep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalprep') - task = create_wf_task('ocnanalprep', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalprep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -391,11 +506,13 @@ def ocnanalbmat(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalbmat') - task = create_wf_task('ocnanalbmat', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalbmat', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -407,11 +524,13 @@ def ocnanalrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalrun') - task = create_wf_task('ocnanalrun', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalrun', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -427,11 +546,13 @@ def ocnanalchkpt(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalchkpt') - task = create_wf_task('ocnanalchkpt', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalchkpt', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -443,11 +564,13 @@ def ocnanalpost(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalpost') - task = create_wf_task('ocnanalpost', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalpost', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -459,11 +582,13 @@ def ocnanalvrfy(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalvrfy') - task = create_wf_task('ocnanalvrfy', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'ocnanalvrfy', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -510,7 +635,13 @@ def _fcst_forecast_only(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -547,8 +678,14 @@ def _fcst_cycled(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('fcst') - task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -572,8 +709,15 @@ def atmanlupp(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('upp') - task = create_wf_task('atmanlupp', resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - cycledef=self.cdump, command='&JOBS_DIR;/upp.sh') + task_dict = {'task_name': 'atmanlupp', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump, + 'command': '&JOBS_DIR;/upp.sh' + } + task = create_wf_task(task_dict) return task @@ -590,8 +734,15 @@ def atmanlprod(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmos_products') - task = create_wf_task('atmanlprod', resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - cycledef=self.cdump, command='&JOBS_DIR;/atmos_products.sh') + task_dict = {'task_name': 'atmanlprod', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump, + 'command': '&JOBS_DIR;/atmos_products.sh' + } + task = create_wf_task(task_dict) return task @@ -632,7 +783,7 @@ def atmupp(self): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = self._get_ufs_postproc_grps(self.cdump, self._configs['upp']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} postenvars = self.envars.copy() postenvar_dict = {'FHRLST': '#lst#', @@ -654,9 +805,23 @@ def atmupp(self): dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('upp') - task = create_wf_task('atmupp', resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - metatask='atmupp', varname=varname1, varval=varval1, vardict=vardict, cycledef=cycledef, - command='&JOBS_DIR;/upp.sh') + + task_dict = {'task_name': f'atmupp#{varname1}#', + 'cdump': self.cdump, + 'cycledef': cycledef, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'command': '&JOBS_DIR;/upp.sh' + } + + metatask_dict = {'task_name': 'atmupp', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -664,7 +829,7 @@ def atmprod(self): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = self._get_ufs_postproc_grps(self.cdump, self._configs['atmos_products']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} postenvars = self.envars.copy() postenvar_dict = {'FHRLST': '#lst#'} @@ -679,9 +844,23 @@ def atmprod(self): dependencies = rocoto.create_dependency(dep=deps) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('atmos_products') - task = create_wf_task('atmprod', resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - metatask='atmprod', varname=varname1, varval=varval1, vardict=vardict, cycledef=cycledef, - command='&JOBS_DIR;/atmos_products.sh') + + task_dict = {'task_name': f'atmprod#{varname1}#', + 'cdump': self.cdump, + 'cycledef': cycledef, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'command': '&JOBS_DIR;/atmos_products.sh' + } + + metatask_dict = {'task_name': 'atmprod', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -689,7 +868,7 @@ def ocnpost(self): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = self._get_ufs_postproc_grps(self.cdump, self._configs['ocnpost']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} postenvars = self.envars.copy() postenvar_dict = {'FHRLST': '#lst#', @@ -707,8 +886,23 @@ def ocnpost(self): dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('ocnpost') - task = create_wf_task('ocnpost', resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, - metatask='ocnpost', varname=varname1, varval=varval1, vardict=vardict, cycledef=cycledef) + + task_dict = {'task_name': f'ocnpost#{varname1}#', + 'cdump': self.cdump, + 'cycledef': cycledef, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'command': '&JOBS_DIR;/atmos_products.sh' + } + + metatask_dict = {'task_name': 'ocnpost', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -722,7 +916,14 @@ def wavepostsbs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostsbs') - task = create_wf_task('wavepostsbs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'wavepostsbs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -733,7 +934,14 @@ def wavepostbndpnt(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpnt') - task = create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'wavepostbndpnt', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -746,8 +954,14 @@ def wavepostbndpntbll(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpntbll') - task = create_wf_task('wavepostbndpntbll', resources, cdump=self.cdump, envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'wavepostbndpntbll', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -761,7 +975,14 @@ def wavepostpnt(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostpnt') - task = create_wf_task('wavepostpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'wavepostpnt', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -772,7 +993,14 @@ def wavegempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavegempak') - task = create_wf_task('wavegempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'wavegempak', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -785,7 +1013,14 @@ def waveawipsbulls(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('waveawipsbulls') - task = create_wf_task('waveawipsbulls', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'waveawipsbulls', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -796,8 +1031,14 @@ def waveawipsgridded(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('waveawipsgridded') - task = create_wf_task('waveawipsgridded', resources, cdump=self.cdump, envar=self.envars, - dependency=dependencies) + task_dict = {'task_name': 'waveawipsgridded', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -808,7 +1049,14 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('postsnd') - task = create_wf_task('postsnd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'postsnd', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -828,13 +1076,19 @@ def fbwind(self): dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('awips') - # TODO: It would be better to use task dependencies on the # individual post jobs rather than data dependencies to avoid # prematurely starting with partial files. Unfortunately, the # ability to "group" post would make this more convoluted than # it should be and not worth the complexity. - task = create_wf_task('fbwind', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'fbwind', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -891,11 +1145,24 @@ def awips_20km_1p0deg(self): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips_20km_1p0deg', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips_20km_1p0deg', varname=varname1, varval=varval1, vardict=vardict) + + task_dict = {'task_name': f'awips_20km_1p0deg#{varname1}#', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': awipsenvars + } + + metatask_dict = {'task_name': 'awips_20km_1p0deg', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -915,11 +1182,24 @@ def awips_g2(self): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips_g2', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips_g2', varname=varname1, varval=varval1, vardict=vardict) + + task_dict = {'task_name': f'awips_g2#{varname1}#', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': awipsenvars + } + + metatask_dict = {'task_name': 'awips_g2', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -931,7 +1211,14 @@ def gempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'gempak', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -942,7 +1229,14 @@ def gempakmeta(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempakmeta', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'gempakmeta', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -953,7 +1247,14 @@ def gempakmetancdc(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempakmetancdc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'gempakmetancdc', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -964,7 +1265,14 @@ def gempakncdcupapgif(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempakncdcupapgif', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'gempakncdcupapgif', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -974,7 +1282,14 @@ def gempakpgrb2spec(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempakpgrb2spec', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'gempakpgrb2spec', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -986,7 +1301,14 @@ def npoess_pgrb2_0p5deg(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('npoess') - task = create_wf_task('npoess_pgrb2_0p5deg', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'npoess_pgrb2_0p5deg', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -997,7 +1319,14 @@ def verfozn(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('verfozn') - task = create_wf_task('verfozn', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'verfozn', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1008,7 +1337,14 @@ def verfrad(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('verfrad') - task = create_wf_task('verfrad', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'verfrad', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1019,7 +1355,14 @@ def vminmon(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('vminmon') - task = create_wf_task('vminmon', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'vminmon', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1030,7 +1373,14 @@ def tracker(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('tracker') - task = create_wf_task('tracker', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'tracker', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1041,7 +1391,14 @@ def genesis(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('genesis') - task = create_wf_task('genesis', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'genesis', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1052,7 +1409,14 @@ def genesis_fsu(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('genesis_fsu') - task = create_wf_task('genesis_fsu', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'genesis_fsu', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1063,7 +1427,14 @@ def fit2obs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('fit2obs') - task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'fit2obs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1084,10 +1455,24 @@ def metp(self): varname1 = 'metpcase' varval1 = 'g2g1 g2o1 pcp1' + var_dict = {varname1: varval1} resources = self.get_resource('metp') - task = create_wf_task('metp', resources, cdump=self.cdump, envar=metpenvars, dependency=dependencies, - metatask='metp', varname=varname1, varval=varval1) + + task_dict = {'task_name': f'metp#{varname1}#', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': metpenvars + } + + metatask_dict = {'task_name': 'metp', + 'cdump': self.cdump, + 'task_dict': task_dict, + 'var_dict': var_dict + } + + task = create_wf_task(metatask_dict) return task @@ -1098,7 +1483,14 @@ def mos_stn_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_prep') - task = create_wf_task('mos_stn_prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_stn_prep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1109,7 +1501,14 @@ def mos_grd_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_grd_prep') - task = create_wf_task('mos_grd_prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_grd_prep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1120,7 +1519,14 @@ def mos_ext_stn_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_ext_stn_prep') - task = create_wf_task('mos_ext_stn_prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_stn_prep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1131,7 +1537,14 @@ def mos_ext_grd_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_ext_grd_prep') - task = create_wf_task('mos_ext_grd_prep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_grd_prep', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1142,7 +1555,14 @@ def mos_stn_fcst(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_fcst') - task = create_wf_task('mos_stn_fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_stn_fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1156,7 +1576,14 @@ def mos_grd_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_grd_fcst') - task = create_wf_task('mos_grd_fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_grd_fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1170,7 +1597,14 @@ def mos_ext_stn_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_stn_fcst') - task = create_wf_task('mos_ext_stn_fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_stn_fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1187,7 +1621,14 @@ def mos_ext_grd_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_grd_fcst') - task = create_wf_task('mos_ext_grd_fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_grd_fcst', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1198,7 +1639,14 @@ def mos_stn_prdgen(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_prdgen') - task = create_wf_task('mos_stn_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_stn_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1212,7 +1660,14 @@ def mos_grd_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_grd_prdgen') - task = create_wf_task('mos_grd_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_grd_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1226,7 +1681,14 @@ def mos_ext_stn_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_stn_prdgen') - task = create_wf_task('mos_ext_stn_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_stn_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1243,7 +1705,14 @@ def mos_ext_grd_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_grd_prdgen') - task = create_wf_task('mos_ext_grd_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_ext_grd_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1254,7 +1723,14 @@ def mos_wx_prdgen(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_wx_prdgen') - task = create_wf_task('mos_wx_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_wx_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1268,7 +1744,14 @@ def mos_wx_ext_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_wx_ext_prdgen') - task = create_wf_task('mos_wx_ext_prdgen', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'mos_wx_ext_prdgen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1341,8 +1824,15 @@ def arch(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('arch') - task = create_wf_task('arch', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'arch', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + + task = create_wf_task(task_dict) return task @@ -1359,7 +1849,14 @@ def cleanup(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('cleanup') - task = create_wf_task('cleanup', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'cleanup', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1373,7 +1870,14 @@ def eobs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('eobs') - task = create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'eobs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + + task = create_wf_task(task_dict) return task @@ -1388,9 +1892,23 @@ def eomg(self): groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['eobs']['NMEM_EOMGGRP']) + var_dict = {'grp': groups} + resources = self.get_resource('eomg') - task = create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=groups) + task_dict = {'task_name': 'eomg', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': eomgenvars + } + + metatask_dict = {'task_name': 'eomn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict, + } + + task = create_wf_task(metatask_dict) return task @@ -1401,7 +1919,13 @@ def ediag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ediag') - task = create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'ediag', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -1415,7 +1939,13 @@ def eupd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('eupd') - task = create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'eupd', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -1429,8 +1959,14 @@ def atmensanlinit(self): cycledef = "gdas" resources = self.get_resource('atmensanlinit') - task = create_wf_task('atmensanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'atmensanlinit', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + task = create_wf_task(task_dict) return task @@ -1444,7 +1980,13 @@ def atmensanlrun(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmensanlrun') - task = create_wf_task('atmensanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'atmensanlrun', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -1456,7 +1998,13 @@ def atmensanlfinal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmensanlfinal') - task = create_wf_task('atmensanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'atmensanlfinal', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -1503,11 +2051,24 @@ def _get_ecengroups(): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = _get_ecengroups() - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} resources = self.get_resource('ecen') - task = create_wf_task('ecen', resources, cdump=self.cdump, envar=ecenenvars, dependency=dependencies, - metatask='ecmn', varname=varname1, varval=varval1, vardict=vardict) + + task_dict = {'task_name': 'ecen', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': ecenenvars + } + + metatask_dict = {'task_name': 'ecmn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = create_wf_task(metatask_dict) return task def esfc(self): @@ -1525,7 +2086,13 @@ def esfc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('esfc') - task = create_wf_task('esfc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task_dict = {'task_name': 'esfc', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars + } + task = create_wf_task(task_dict) return task @@ -1550,8 +2117,24 @@ def efcs(self): groups = self._get_hybgroups(self._base['NMEM_ENS_GFS'], self._configs['efcs']['NMEM_EFCSGRP_GFS']) cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('efcs') - task = create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, - metatask='efmn', varname='grp', varval=groups, cycledef=cycledef) + + var_dict = {'grp': groups} + + task_dict = {'task_name': 'efcs', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': efcsenvars, + 'cycledef': cycledef + } + + metatask_dict = {'task_name': 'efmn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = create_wf_task(metatask_dict) return task @@ -1569,8 +2152,15 @@ def echgres(self): cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump resources = self.get_resource('echgres') - task = create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task_dict = {'task_name': 'echgres', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': cycledef + } + + task = create_wf_task(task_dict) return task @@ -1611,13 +2201,27 @@ def _get_eposgroups(epos): varname1, varname2, varname3 = 'grp', 'dep', 'lst' varval1, varval2, varval3 = _get_eposgroups(self._configs['epos']) - vardict = {varname2: varval2, varname3: varval3} + var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('epos') - task = create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict, cycledef=cycledef) + + task_dict = {'task_name': 'epos', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': eposenvars, + 'cycledef': cycledef + } + + metatask_dict = {'task_name': 'epmn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = create_wf_task(metatask_dict) return task @@ -1636,7 +2240,23 @@ def earc(self): cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('earc') - task = create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=groups, cycledef=cycledef) + + var_dict = {'grp': groups} + + task_dict = {'task_name': 'earc', + 'cdump': self.cdump, + 'resources': resources, + 'dependency': dependencies, + 'envars': earcenvars, + 'cycledef': cycledef + } + + metatask_dict = {'task_name': 'eamn', + 'cdump': self.cdump, + 'var_dict': var_dict, + 'task_dict': task_dict + } + + task = create_wf_task(metatask_dict) return task diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index c4240622d4a..8d019564d91 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -10,62 +10,112 @@ Helper module to create tasks, metatasks, and dependencies for Rocoto ''' -__all__ = ['create_task', 'create_metatask', +__all__ = ['create_task', 'add_dependency', 'create_dependency', 'create_envar', 'create_entity', 'create_cycledef'] -def create_metatask(task_dict: Dict[str, Any], metatask_dict: Dict[str, Any]) -> List[str]: +def create_task(task_dict: Dict[str, Any]) -> List[str]: """ - create a Rocoto metatask given a dictionary containing task and metatask information - :param metatask_dict: metatask key-value parameters - :type metatask_dict: dict - :param task_dict: task key-value parameters - :type task_dict: dict - :return: Rocoto metatask - :rtype: list + Create XML for a rocoto task or metatask + + Creates the XML required to define a task and returns the lines + as a list of strings. Tasks can be nested to create metatasks by + defining a key 'task_dict' within the task_dict. When including + a nested task, you also need to provide a 'var_dict' key that + contains a dictionary of variables to loop over. + + All task dicts must include a 'task_name' and a 'cdump'. + + Innermost tasks (regular tasks) additionally require a 'resources' + key containing a dict of values defining the HPC settings. + + Parameters + ---------- + task_dict: dict + Dictionary of task definitions + + Returns + ------- + List[str] + List of strings containing the XML code defining the task + + Raises + ------ + KeyError + If a required key is missing + """ - # Grab metatask info from the metatask_dict - metataskname = metatask_dict.get('metataskname', 'demometatask') - varname = metatask_dict.get('varname', 'demovar') - varval = metatask_dict.get('varval', 1) - vardict = metatask_dict.get('vardict', None) + inner_task_dict = task_dict.pop('task_dict', None) - strings = [f'\n', - '\n', - f'\t{str(varval)}\n'] + if inner_task_dict is None: + strings = _create_innermost_task(task_dict) + + else: + # There is a nested task_dict, so this is a metatask + metataskname = f"{task_dict.get('cdump','gdas')}{task_dict.get('task_name', 'demometatask')}" + var_dict = task_dict.get('var_dict', None) + + strings = [f'\n', + '\n'] + + if var_dict is None: + msg = f'Task {metataskname} has a nested task dict, but has no var_dict' + raise KeyError(msg) - if vardict is not None: - for key in vardict.keys(): - value = str(vardict[key]) + for key in var_dict.keys(): + value = str(var_dict[key]) strings.append(f'\t{value}\n') - strings.append('\n') - tasklines = create_task(task_dict) - for tl in tasklines: - strings.append(f'{tl}') if tl == '\n' else strings.append(f'\t{tl}') - strings.append('\n') - strings.append('\n') + + strings.append('\n') + task_dict.update(inner_task_dict) + tasklines = create_task(task_dict) + for tl in tasklines: + strings.append(f'{tl}') if tl == '\n' else strings.append(f'\t{tl}') + strings.append('\n') + strings.append('\n') return strings -def create_task(task_dict: Dict[str, Any]) -> List[str]: +def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: """ - create a Rocoto task given a dictionary containing task information - :param task_dict: task key-value parameters - :type task_dict: dict - :return: Rocoto task - :rtype: list + Create XML for a regular rocoto task + + Creates the XML required to define a task and returns the lines + as a list of strings. + + All task dicts must include a 'task_name', 'cdump', and a 'resources' + key containing a dict of values defining the HPC settings. + + Parameters + ---------- + task_dict: dict + Dictionary of task definitions + + Returns + ------- + List[str] + List of strings containing the XML code defining the task + + Raises + ------ + KeyError + If a required key is missing + """ # Grab task info from the task_names - taskname = task_dict.get('taskname', 'demotask') - cycledef = task_dict.get('cycledef', 'democycle') - maxtries = task_dict.get('maxtries', 3) + cdump = task_dict.get('cdump', 'gdas') + base_taskname = task_dict.get('task_name', 'demotask') + + taskname = f'{cdump}{base_taskname}' + cycledef = task_dict.get('cycledef', cdump.replace('enkf', '')) + maxtries = task_dict.get('maxtries', '&MAXTRIES;') final = task_dict.get('final', False) - command = task_dict.get('command', 'sleep 10') - jobname = task_dict.get('jobname', 'demojob') + command = task_dict.get('command', f'&JOBS_DIR;/{base_taskname}.sh') + jobname = task_dict.get('jobname', f'&PSLOT;_{taskname}_@H') resources_dict = task_dict['resources'] account = resources_dict.get('account', 'batch') queue = resources_dict.get('queue', 'debug') @@ -76,7 +126,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: nodes = resources_dict.get('nodes', 1) ppn = resources_dict.get('ppn', 1) threads = resources_dict.get('threads', 1) - log = task_dict.get('log', 'demo.log') + log = task_dict.get('log', f'&ROTDIR;/logs/@Y@m@d@H/{taskname}.log') envar = task_dict.get('envars', None) dependency = task_dict.get('dependency', []) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 2d44c00d4d4..7103e59a087 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -199,32 +199,38 @@ def get_task(self, task_name, *args, **kwargs): f'{", ".join(Tasks.VALID_TASKS)}') -def create_wf_task(task_name, resources, - cdump='gdas', cycledef=None, envar=None, dependency=None, - metatask=None, varname=None, varval=None, vardict=None, - final=False, command=None): - tasknamestr = f'{cdump}{task_name}' - metatask_dict = None - if metatask is not None: - tasknamestr = f'{tasknamestr}#{varname}#' - metatask_dict = {'metataskname': f'{cdump}{metatask}', - 'varname': f'{varname}', - 'varval': f'{varval}', - 'vardict': vardict} - - cycledefstr = cdump.replace('enkf', '') if cycledef is None else cycledef - - task_dict = {'taskname': f'{tasknamestr}', - 'cycledef': f'{cycledefstr}', - 'maxtries': '&MAXTRIES;', - 'command': f'&JOBS_DIR;/{task_name}.sh' if command is None else command, - 'jobname': f'&PSLOT;_{tasknamestr}_@H', - 'resources': resources, - 'log': f'&ROTDIR;/logs/@Y@m@d@H/{tasknamestr}.log', - 'envars': envar, - 'dependency': dependency, - 'final': final} - - task = rocoto.create_task(task_dict) if metatask is None else rocoto.create_metatask(task_dict, metatask_dict) +def create_wf_task(task_dict): + """ + Create XML for a rocoto task or metatask + + Creates the XML required to define a task and returns the lines + as a list of strings. Tasks can be nested to create metatasks by + defining a key 'task_dict' within the task_dict. When including + a nested task, you also need to provide a 'var_dict' key that + contains a dictionary of variables to loop over. + + All task dicts must include a 'task_name' and a 'cdump'. + + Innermost tasks (regular tasks) additionally require a 'resources' + key containing a dict of values defining the HPC settings. + + Parameters + ---------- + task_dict: dict + Dictionary of task definitions + + Returns + ------- + str + The XML code defining the task + + Raises + ------ + KeyError + If a required key is missing + + """ + + task = rocoto.create_task(task_dict) return ''.join(task) From 685de59883220973ca5b882bff7308faeaef9a61 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 22 Dec 2023 21:19:00 -0600 Subject: [PATCH 02/10] Add missing loop variables to task names --- workflow/rocoto/gefs_tasks.py | 2 +- workflow/rocoto/gfs_tasks.py | 13 ++++++------- workflow/rocoto/rocoto.py | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 230e753e832..e9071ce784b 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -121,7 +121,7 @@ def efcs(self): resources = self.get_resource('efcs') - task_dict = {'task_name': 'efcs', + task_dict = {'task_name': 'efcs#grp#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 539ac147b57..9c27e36541b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -125,7 +125,7 @@ def waveinit(self): resources = self.get_resource('waveinit') dependencies = None - cycledef = None + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump if self.app_config.mode in ['cycled']: deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} @@ -134,7 +134,6 @@ def waveinit(self): dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump task_dict = {'task_name': 'waveinit', 'cdump': self.cdump, @@ -1894,7 +1893,7 @@ def eomg(self): var_dict = {'grp': groups} - resources = self.get_resource('eomg') + resources = self.get_resource(f'eomg#grp#') task_dict = {'task_name': 'eomg', 'cdump': self.cdump, 'resources': resources, @@ -2055,7 +2054,7 @@ def _get_ecengroups(): resources = self.get_resource('ecen') - task_dict = {'task_name': 'ecen', + task_dict = {'task_name': f'ecen#{varname1}#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, @@ -2120,7 +2119,7 @@ def efcs(self): var_dict = {'grp': groups} - task_dict = {'task_name': 'efcs', + task_dict = {'task_name': f'efcs#grp#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, @@ -2207,7 +2206,7 @@ def _get_eposgroups(epos): resources = self.get_resource('epos') - task_dict = {'task_name': 'epos', + task_dict = {'task_name': f'epos#{varname1}#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, @@ -2243,7 +2242,7 @@ def earc(self): var_dict = {'grp': groups} - task_dict = {'task_name': 'earc', + task_dict = {'task_name': f'earc#grp#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 8d019564d91..ed1829b246b 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from typing import Union, List, Dict, Any +import re ''' MODULE: @@ -114,7 +115,7 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: cycledef = task_dict.get('cycledef', cdump.replace('enkf', '')) maxtries = task_dict.get('maxtries', '&MAXTRIES;') final = task_dict.get('final', False) - command = task_dict.get('command', f'&JOBS_DIR;/{base_taskname}.sh') + command = task_dict.get('command', f'&JOBS_DIR;/{re.sub("#.+?#", "", base_taskname)}.sh') jobname = task_dict.get('jobname', f'&PSLOT;_{taskname}_@H') resources_dict = task_dict['resources'] account = resources_dict.get('account', 'batch') From 3be6c2fcbd5599767849f04f5a0b32638afe12a7 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 23 Dec 2023 00:49:21 -0600 Subject: [PATCH 03/10] Fix envars in new task definitions --- workflow/rocoto/gfs_tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9c27e36541b..db0e8384067 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -712,7 +712,7 @@ def atmanlupp(self): 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': postenvars, 'cycledef': self.cdump, 'command': '&JOBS_DIR;/upp.sh' } @@ -737,7 +737,7 @@ def atmanlprod(self): 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': postenvars, 'cycledef': self.cdump, 'command': '&JOBS_DIR;/atmos_products.sh' } @@ -891,8 +891,7 @@ def ocnpost(self): 'cycledef': cycledef, 'resources': resources, 'dependency': dependencies, - 'envars': postenvars, - 'command': '&JOBS_DIR;/atmos_products.sh' + 'envars': postenvars } metatask_dict = {'task_name': 'ocnpost', From 8b8594b6fd3e3c170f11495732efb5b62f20c21c Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 23 Dec 2023 01:15:56 -0600 Subject: [PATCH 04/10] Fix typo in gempak meta jjob header call --- jobs/JGFS_ATMOS_GEMPAK_META | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGFS_ATMOS_GEMPAK_META b/jobs/JGFS_ATMOS_GEMPAK_META index b7786b1f49b..8e1c05763f4 100755 --- a/jobs/JGFS_ATMOS_GEMPAK_META +++ b/jobs/JGFS_ATMOS_GEMPAK_META @@ -6,7 +6,7 @@ # GFS GEMPAK META PRODUCT GENERATION ############################################ source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -e "base" +source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base" ############################################### From 5cb3cf34e3e643f4305ea15b6185f053e309788e Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 23 Dec 2023 01:45:34 -0600 Subject: [PATCH 05/10] Fix a couple minor errors in new task dicts --- workflow/rocoto/gfs_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index db0e8384067..9a270b7ad65 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -810,7 +810,7 @@ def atmupp(self): 'cycledef': cycledef, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': postenvars, 'command': '&JOBS_DIR;/upp.sh' } @@ -1892,8 +1892,8 @@ def eomg(self): var_dict = {'grp': groups} - resources = self.get_resource(f'eomg#grp#') - task_dict = {'task_name': 'eomg', + resources = self.get_resource('eomg') + task_dict = {'task_name': f'eomg#grp#', 'cdump': self.cdump, 'resources': resources, 'dependency': dependencies, From f16502c95cf1378e800013461575d95aaf7d0e07 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 5 Jan 2024 03:36:02 -0600 Subject: [PATCH 06/10] Remove global-specific variable from rocoto.py Returns `rocoto.py` to a workflow-agnostic state without needing `cdump` or any rocoto entities. The middle-man `create_wf_task` in task.py is also removed by explicitly setting several keys in the task dict. --- workflow/rocoto/gefs_tasks.py | 55 +- workflow/rocoto/gfs_tasks.py | 1163 ++++++++++++++++++++++----------- workflow/rocoto/rocoto.py | 25 +- workflow/rocoto/tasks.py | 41 +- 4 files changed, 846 insertions(+), 438 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index e9071ce784b..5df7515218a 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -1,5 +1,5 @@ from applications.applications import AppConfig -from rocoto.tasks import Tasks, create_wf_task +from rocoto.tasks import Tasks import rocoto.rocoto as rocoto @@ -57,29 +57,40 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') - task_dict = {'task_name': 'stage_ic', - 'cdump': self.cdump, + task_name = f'stage_ic' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task def waveinit(self): resources = self.get_resource('waveinit') - task_dict = {'task_name': 'waveinit', - 'cdump': self.cdump, + task_name = f'waveinit' + task_dict = {'task_name': task_name, 'resources': resources, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task def fcst(self): + # TODO: Add real dependencies dependencies = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} @@ -92,13 +103,18 @@ def fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task_dict = {'task_name': 'fcst', - 'cdump': self.cdump, + task_name = f'fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -121,20 +137,23 @@ def efcs(self): resources = self.get_resource('efcs') - task_dict = {'task_name': 'efcs#grp#', - 'cdump': self.cdump, + task_name = f'efcs#grp#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': efcsenvars, - 'cycledef': 'gefs' + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/efcs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } metatask_dict = {'task_name': 'efmn', - 'cdump': self.cdump, 'var_dict': var_dict, 'task_dict': task_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9a270b7ad65..00be66314bb 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1,5 +1,5 @@ from applications.applications import AppConfig -from rocoto.tasks import Tasks, create_wf_task +from rocoto.tasks import Tasks from wxflow import timedelta_to_HMS import rocoto.rocoto as rocoto import numpy as np @@ -71,14 +71,19 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') - task_dict = {'task_name': 'stage_ic', - 'cdump': self.cdump, + task_name = f'{self.cdump}stage_ic' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump, + 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -109,15 +114,19 @@ def prep(self): cycledef = 'gdas' resources = self.get_resource('prep') - task_dict = {'task_name': 'prep', - 'cdump': self.cdump, + task_name = f'{self.cdump}prep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/prep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -135,14 +144,19 @@ def waveinit(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - task_dict = {'task_name': 'waveinit', - 'cdump': self.cdump, + task_name = f'{self.cdump}waveinit' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -154,14 +168,19 @@ def waveprep(self): dependencies = rocoto.create_dependency(dep=deps) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('waveprep') - task_dict = {'task_name': 'waveprep', - 'cdump': self.cdump, + task_name = f'{self.cdump}waveprep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveprep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -200,14 +219,19 @@ def aerosol_init(self): cycledef = 'gfs_seq' resources = self.get_resource('aerosol_init') - task_dict = {'task_name': 'aerosol_init', - 'cdump': self.cdump, + task_name = f'{self.cdump}aerosol_init' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/aerosol_init.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -223,13 +247,19 @@ def anal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('anal') - task_dict = {'task_name': 'anal', - 'cdump': self.cdump, + task_name = f'{self.cdump}anal' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/anal.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -249,13 +279,19 @@ def sfcanl(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('sfcanl') - task_dict = {'task_name': 'sfcanl', - 'cdump': self.cdump, + task_name = f'{self.cdump}sfcanl' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/sfcanl.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -275,13 +311,19 @@ def analcalc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analcalc') - task_dict = {'task_name': 'analcalc', - 'cdump': self.cdump, + task_name = f'{self.cdump}analcalc' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/analcalc.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -293,13 +335,19 @@ def analdiag(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analdiag') - task_dict = {'task_name': 'analdiag', - 'cdump': self.cdump, + task_name = f'{self.cdump}analdiag' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/analdiag.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -311,13 +359,19 @@ def prepatmiodaobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('prepatmiodaobs') - task_dict = {'task_name': 'prepatmiodaobs', - 'cdump': self.cdump, + task_name = f'{self.cdump}prepatmiodaobs' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/prepatmiodaobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -341,14 +395,19 @@ def atmanlinit(self): cycledef = 'gdas' resources = self.get_resource('atmanlinit') - task_dict = {'task_name': 'atmanlinit', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmanlinit' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -360,13 +419,19 @@ def atmanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmanlrun') - task_dict = {'task_name': 'atmanlrun', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmanlrun' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlrun.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -378,13 +443,19 @@ def atmanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmanlfinal') - task_dict = {'task_name': 'atmanlfinal', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmanlfinal' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlfinal.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -396,13 +467,20 @@ def aeroanlinit(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') - task_dict = {'task_name': 'aeroanlinit', - 'cdump': self.cdump, + task_name = f'{self.cdump}aeroanlinit' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) + return task def aeroanlrun(self): @@ -413,13 +491,19 @@ def aeroanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('aeroanlrun') - task_dict = {'task_name': 'aeroanlrun', - 'cdump': self.cdump, + task_name = f'{self.cdump}aeroanlrun' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -431,13 +515,19 @@ def aeroanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlfinal') - task_dict = {'task_name': 'aeroanlfinal', - 'cdump': self.cdump, + task_name = f'{self.cdump}aeroanlfinal' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlfinal.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -449,13 +539,19 @@ def preplandobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('preplandobs') - task_dict = {'task_name': 'preplandobs', - 'cdump': self.cdump, + task_name = f'{self.cdump}preplandobs' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/preplandobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -467,13 +563,19 @@ def landanl(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('landanl') - task_dict = {'task_name': 'landanl', - 'cdump': self.cdump, + task_name = f'{self.cdump}landanl' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/landanl.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task def ocnanalprep(self): @@ -487,13 +589,19 @@ def ocnanalprep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalprep') - task_dict = {'task_name': 'ocnanalprep', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanalprep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalprep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -505,13 +613,19 @@ def ocnanalbmat(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalbmat') - task_dict = {'task_name': 'ocnanalbmat', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanalbmat' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalbmat.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -523,13 +637,19 @@ def ocnanalrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalrun') - task_dict = {'task_name': 'ocnanalrun', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanlrun' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanlrun.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -545,13 +665,19 @@ def ocnanalchkpt(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalchkpt') - task_dict = {'task_name': 'ocnanalchkpt', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanalchkpt' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalchkpt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -563,13 +689,19 @@ def ocnanalpost(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalpost') - task_dict = {'task_name': 'ocnanalpost', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanalpost' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalpost.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -581,13 +713,19 @@ def ocnanalvrfy(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('ocnanalvrfy') - task_dict = {'task_name': 'ocnanalvrfy', - 'cdump': self.cdump, + task_name = f'{self.cdump}ocnanalvrfy' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalvrfy.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -634,13 +772,19 @@ def _fcst_forecast_only(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task_dict = {'task_name': 'fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -677,14 +821,19 @@ def _fcst_cycled(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('fcst') - task_dict = {'task_name': 'fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -708,15 +857,19 @@ def atmanlupp(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('upp') - task_dict = {'task_name': 'atmanlupp', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmanlupp' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'cycledef': self.cdump, - 'command': '&JOBS_DIR;/upp.sh' + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/upp.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -733,15 +886,19 @@ def atmanlprod(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmos_products') - task_dict = {'task_name': 'atmanlprod', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmanlprod' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'cycledef': self.cdump, - 'command': '&JOBS_DIR;/atmos_products.sh' + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_products.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -805,22 +962,24 @@ def atmupp(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('upp') - task_dict = {'task_name': f'atmupp#{varname1}#', - 'cdump': self.cdump, - 'cycledef': cycledef, + task_name = f'{self.cdump}atmupp#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'command': '&JOBS_DIR;/upp.sh' + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/upp.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'atmupp', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}atmupp', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -844,22 +1003,24 @@ def atmprod(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('atmos_products') - task_dict = {'task_name': f'atmprod#{varname1}#', - 'cdump': self.cdump, - 'cycledef': cycledef, + task_name = f'{self.cdump}atmprod#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'command': '&JOBS_DIR;/atmos_products.sh' + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_products.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'atmprod', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}atmprod', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -886,21 +1047,24 @@ def ocnpost(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('ocnpost') - task_dict = {'task_name': f'ocnpost#{varname1}#', - 'cdump': self.cdump, - 'cycledef': cycledef, + task_name = f'{self.cdump}ocnpost#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': postenvars + 'envars': postenvars, + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnpost.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'ocnpost', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}ocnpost', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -914,14 +1078,19 @@ def wavepostsbs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostsbs') - task_dict = {'task_name': 'wavepostsbs', - 'cdump': self.cdump, + task_name = f'{self.cdump}wavepostsbs' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -932,14 +1101,19 @@ def wavepostbndpnt(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpnt') - task_dict = {'task_name': 'wavepostbndpnt', - 'cdump': self.cdump, + task_name = f'{self.cdump}wavepostbndpnt' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -952,14 +1126,19 @@ def wavepostbndpntbll(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpntbll') - task_dict = {'task_name': 'wavepostbndpntbll', - 'cdump': self.cdump, + task_name = f'{self.cdump}wavepostbndpntbll' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -973,14 +1152,19 @@ def wavepostpnt(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostpnt') - task_dict = {'task_name': 'wavepostpnt', - 'cdump': self.cdump, + task_name = f'{self.cdump}wavepostpnt' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -991,14 +1175,19 @@ def wavegempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavegempak') - task_dict = {'task_name': 'wavegempak', - 'cdump': self.cdump, + task_name = f'{self.cdump}wavegempak' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavegempak.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1011,14 +1200,19 @@ def waveawipsbulls(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('waveawipsbulls') - task_dict = {'task_name': 'waveawipsbulls', - 'cdump': self.cdump, + task_name = f'{self.cdump}waveawipsbulls' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveawipsbulls.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1029,14 +1223,19 @@ def waveawipsgridded(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('waveawipsgridded') - task_dict = {'task_name': 'waveawipsgridded', - 'cdump': self.cdump, + task_name = f'{self.cdump}waveawipsgridded' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveawipsgridded.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1047,14 +1246,19 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('postsnd') - task_dict = {'task_name': 'postsnd', - 'cdump': self.cdump, + task_name = f'{self.cdump}postsnd' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1079,14 +1283,19 @@ def fbwind(self): # prematurely starting with partial files. Unfortunately, the # ability to "group" post would make this more convoluted than # it should be and not worth the complexity. - task_dict = {'task_name': 'fbwind', - 'cdump': self.cdump, + task_name = f'{self.cdump}fbwind' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/fbwind.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1147,20 +1356,24 @@ def awips_20km_1p0deg(self): resources = self.get_resource('awips') - task_dict = {'task_name': f'awips_20km_1p0deg#{varname1}#', - 'cdump': self.cdump, + task_name = f'{self.cdump}awips_20km_1p0deg#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': awipsenvars + 'envars': awipsenvars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_20km_1p0deg.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'awips_20km_1p0deg', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}awips_20km_1p0deg', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -1184,20 +1397,24 @@ def awips_g2(self): resources = self.get_resource('awips') - task_dict = {'task_name': f'awips_g2#{varname1}#', - 'cdump': self.cdump, + task_name = f'{self.cdump}awips_g2#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': awipsenvars + 'envars': awipsenvars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_g2.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'awips_g2', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}awips_g2', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -1209,14 +1426,19 @@ def gempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task_dict = {'task_name': 'gempak', - 'cdump': self.cdump, + task_name = f'{self.cdump}gempak' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1227,14 +1449,19 @@ def gempakmeta(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task_dict = {'task_name': 'gempakmeta', - 'cdump': self.cdump, + task_name = f'{self.cdump}gempakmeta' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakmeta.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1245,14 +1472,19 @@ def gempakmetancdc(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task_dict = {'task_name': 'gempakmetancdc', - 'cdump': self.cdump, + task_name = f'{self.cdump}gempakmetancdc' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakmetancdc.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1263,31 +1495,42 @@ def gempakncdcupapgif(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task_dict = {'task_name': 'gempakncdcupapgif', - 'cdump': self.cdump, + task_name = f'{self.cdump}gempakncdcupapgif' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakncdcupapgif.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task def gempakpgrb2spec(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}npoess_pgrb2_0p5deg'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task_dict = {'task_name': 'gempakpgrb2spec', - 'cdump': self.cdump, + task_name = f'{self.cdump}gempakgrb2spec' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakgrb2spec.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1299,14 +1542,19 @@ def npoess_pgrb2_0p5deg(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('npoess') - task_dict = {'task_name': 'npoess_pgrb2_0p5deg', - 'cdump': self.cdump, + task_name = f'{self.cdump}npoess_pgrb2_0p5deg' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/npoess_pgrb2_0p5deg.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1317,14 +1565,19 @@ def verfozn(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('verfozn') - task_dict = {'task_name': 'verfozn', - 'cdump': self.cdump, + task_name = f'{self.cdump}verfozn' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/verfozn.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1335,14 +1588,19 @@ def verfrad(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('verfrad') - task_dict = {'task_name': 'verfrad', - 'cdump': self.cdump, + task_name = f'{self.cdump}verfrad' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/verfrad.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1353,14 +1611,19 @@ def vminmon(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('vminmon') - task_dict = {'task_name': 'vminmon', - 'cdump': self.cdump, + task_name = f'{self.cdump}vminmon' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/vminmon.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1371,14 +1634,19 @@ def tracker(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('tracker') - task_dict = {'task_name': 'tracker', - 'cdump': self.cdump, + task_name = f'{self.cdump}tracker' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/tracker.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1389,14 +1657,19 @@ def genesis(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('genesis') - task_dict = {'task_name': 'genesis', - 'cdump': self.cdump, + task_name = f'{self.cdump}genesis' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/genesis.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1407,14 +1680,19 @@ def genesis_fsu(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('genesis_fsu') - task_dict = {'task_name': 'genesis_fsu', - 'cdump': self.cdump, + task_name = f'{self.cdump}genesis_fsu' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/genesis_fsu.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1425,14 +1703,19 @@ def fit2obs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('fit2obs') - task_dict = {'task_name': 'fit2obs', - 'cdump': self.cdump, + task_name = f'{self.cdump}fit2obs' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/fit2obs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1457,20 +1740,24 @@ def metp(self): resources = self.get_resource('metp') - task_dict = {'task_name': f'metp#{varname1}#', - 'cdump': self.cdump, + task_name = f'{self.cdump}metp#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': metpenvars + 'envars': metpenvars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'metp', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}metp', 'task_dict': task_dict, 'var_dict': var_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -1481,14 +1768,19 @@ def mos_stn_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_prep') - task_dict = {'task_name': 'mos_stn_prep', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_stn_prep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_prep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1499,14 +1791,19 @@ def mos_grd_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_grd_prep') - task_dict = {'task_name': 'mos_grd_prep', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_grd_prep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_prep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1517,14 +1814,19 @@ def mos_ext_stn_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_ext_stn_prep') - task_dict = {'task_name': 'mos_ext_stn_prep', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_stn_prep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_prep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1535,14 +1837,19 @@ def mos_ext_grd_prep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_ext_grd_prep') - task_dict = {'task_name': 'mos_ext_grd_prep', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_grd_prep' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_prep.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1553,14 +1860,19 @@ def mos_stn_fcst(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_fcst') - task_dict = {'task_name': 'mos_stn_fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_stn_fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1574,14 +1886,19 @@ def mos_grd_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_grd_fcst') - task_dict = {'task_name': 'mos_grd_fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_grd_fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1595,14 +1912,19 @@ def mos_ext_stn_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_stn_fcst') - task_dict = {'task_name': 'mos_ext_stn_fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_stn_fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1619,14 +1941,19 @@ def mos_ext_grd_fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_grd_fcst') - task_dict = {'task_name': 'mos_ext_grd_fcst', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_grd_fcst' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1637,14 +1964,19 @@ def mos_stn_prdgen(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_stn_prdgen') - task_dict = {'task_name': 'mos_stn_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_stn_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1658,14 +1990,19 @@ def mos_grd_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_grd_prdgen') - task_dict = {'task_name': 'mos_grd_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_grd_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1679,14 +2016,19 @@ def mos_ext_stn_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_stn_prdgen') - task_dict = {'task_name': 'mos_ext_stn_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_stn_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1703,14 +2045,19 @@ def mos_ext_grd_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_ext_grd_prdgen') - task_dict = {'task_name': 'mos_ext_grd_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_ext_grd_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1721,14 +2068,19 @@ def mos_wx_prdgen(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('mos_wx_prdgen') - task_dict = {'task_name': 'mos_wx_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_wx_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_wx_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1742,14 +2094,19 @@ def mos_wx_ext_prdgen(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('mos_wx_ext_prdgen') - task_dict = {'task_name': 'mos_wx_ext_prdgen', - 'cdump': self.cdump, + task_name = f'{self.cdump}mos_wx_ext_prdgen' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_wx_ext_prdgen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1822,15 +2179,19 @@ def arch(self): cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('arch') - task_dict = {'task_name': 'arch', - 'cdump': self.cdump, + task_name = f'{self.cdump}arch' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1847,14 +2208,19 @@ def cleanup(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('cleanup') - task_dict = {'task_name': 'cleanup', - 'cdump': self.cdump, + task_name = f'{self.cdump}cleanup' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1868,14 +2234,19 @@ def eobs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('eobs') - task_dict = {'task_name': 'eobs', - 'cdump': self.cdump, + task_name = f'{self.cdump}eobs' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/eobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -1893,20 +2264,24 @@ def eomg(self): var_dict = {'grp': groups} resources = self.get_resource('eomg') - task_dict = {'task_name': f'eomg#grp#', - 'cdump': self.cdump, + task_name = f'{self.cdump}eomg#grp#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': eomgenvars + 'envars': eomgenvars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/eomg.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'eomn', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}eomn', 'var_dict': var_dict, 'task_dict': task_dict, } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -1917,13 +2292,19 @@ def ediag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ediag') - task_dict = {'task_name': 'ediag', - 'cdump': self.cdump, + task_name = f'{self.cdump}ediag' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ediag.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -1937,13 +2318,19 @@ def eupd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('eupd') - task_dict = {'task_name': 'eupd', - 'cdump': self.cdump, + task_name = f'{self.cdump}eupd' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/eupd.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -1957,14 +2344,19 @@ def atmensanlinit(self): cycledef = "gdas" resources = self.get_resource('atmensanlinit') - task_dict = {'task_name': 'atmensanlinit', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmensanlinit' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -1978,13 +2370,19 @@ def atmensanlrun(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmensanlrun') - task_dict = {'task_name': 'atmensanlrun', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmensanlrun' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlrun.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -1996,13 +2394,19 @@ def atmensanlfinal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmensanlfinal') - task_dict = {'task_name': 'atmensanlfinal', - 'cdump': self.cdump, + task_name = f'{self.cdump}atmensanlfinal' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlfinal.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -2053,20 +2457,24 @@ def _get_ecengroups(): resources = self.get_resource('ecen') - task_dict = {'task_name': f'ecen#{varname1}#', - 'cdump': self.cdump, + task_name = f'{self.cdump}ecen#{varname1}' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': ecenenvars + 'envars': ecenenvars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/ecen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'ecmn', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}ecmn', 'var_dict': var_dict, 'task_dict': task_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task def esfc(self): @@ -2084,13 +2492,19 @@ def esfc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('esfc') - task_dict = {'task_name': 'esfc', - 'cdump': self.cdump, + task_name = f'{self.cdump}esfc' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/esfc.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + + task = rocoto.create_task(task_dict) return task @@ -2118,21 +2532,24 @@ def efcs(self): var_dict = {'grp': groups} - task_dict = {'task_name': f'efcs#grp#', - 'cdump': self.cdump, + task_name = f'{self.cdump}efcs#grp#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': efcsenvars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/efcs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'efmn', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}efmn', 'var_dict': var_dict, 'task_dict': task_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -2150,15 +2567,19 @@ def echgres(self): cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump resources = self.get_resource('echgres') - task_dict = {'task_name': 'echgres', - 'cdump': self.cdump, + task_name = f'{self.cdump}echgres' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/echgres.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = create_wf_task(task_dict) + task = rocoto.create_task(task_dict) return task @@ -2205,21 +2626,24 @@ def _get_eposgroups(epos): resources = self.get_resource('epos') - task_dict = {'task_name': f'epos#{varname1}#', - 'cdump': self.cdump, + task_name = f'{self.cdump}epos#{varname1}#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': eposenvars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/epos.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'epmn', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}epmn', 'var_dict': var_dict, 'task_dict': task_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task @@ -2241,20 +2665,23 @@ def earc(self): var_dict = {'grp': groups} - task_dict = {'task_name': f'earc#grp#', - 'cdump': self.cdump, + task_name = f'{self.cdump}earc#grp#' + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': earcenvars, - 'cycledef': cycledef + 'cycledef': cycledef, + 'command': f'{self.HOMEgfs}/jobs/rocoto/earc.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - metatask_dict = {'task_name': 'eamn', - 'cdump': self.cdump, + metatask_dict = {'task_name': f'{self.cdump}eamn', 'var_dict': var_dict, 'task_dict': task_dict } - task = create_wf_task(metatask_dict) + task = rocoto.create_task(metatask_dict) return task diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index ed1829b246b..3aebade27aa 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -26,7 +26,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: a nested task, you also need to provide a 'var_dict' key that contains a dictionary of variables to loop over. - All task dicts must include a 'task_name' and a 'cdump'. + All task dicts must include a 'task_name'. Innermost tasks (regular tasks) additionally require a 'resources' key containing a dict of values defining the HPC settings. @@ -38,8 +38,8 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: Returns ------- - List[str] - List of strings containing the XML code defining the task + str + Strings containing the XML code defining the task Raises ------ @@ -55,7 +55,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: else: # There is a nested task_dict, so this is a metatask - metataskname = f"{task_dict.get('cdump','gdas')}{task_dict.get('task_name', 'demometatask')}" + metataskname = f"{task_dict.get('task_name', 'demometatask')}" var_dict = task_dict.get('var_dict', None) strings = [f'\n', @@ -77,7 +77,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: strings.append('\n') strings.append('\n') - return strings + return ''.join(strings) def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: @@ -87,7 +87,7 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: Creates the XML required to define a task and returns the lines as a list of strings. - All task dicts must include a 'task_name', 'cdump', and a 'resources' + All task dicts must include a 'task_name' and a 'resources' key containing a dict of values defining the HPC settings. Parameters @@ -108,15 +108,12 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: """ # Grab task info from the task_names - cdump = task_dict.get('cdump', 'gdas') - base_taskname = task_dict.get('task_name', 'demotask') - - taskname = f'{cdump}{base_taskname}' - cycledef = task_dict.get('cycledef', cdump.replace('enkf', '')) - maxtries = task_dict.get('maxtries', '&MAXTRIES;') + taskname = task_dict.get('task_name', 'demotask') + cycledef = task_dict.get('cycledef', 'democycle') + maxtries = task_dict.get('maxtries', 3) final = task_dict.get('final', False) - command = task_dict.get('command', f'&JOBS_DIR;/{re.sub("#.+?#", "", base_taskname)}.sh') - jobname = task_dict.get('jobname', f'&PSLOT;_{taskname}_@H') + command = task_dict.get('command', 'sleep 10') + jobname = task_dict.get('job_name', 'demojob') resources_dict = task_dict['resources'] account = resources_dict.get('account', 'batch') queue = resources_dict.get('queue', 'debug') diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 7103e59a087..de4062f2765 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -42,12 +42,14 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: # Save dict_configs and base in the internal state (never know where it may be needed) self._configs = self.app_config.configs self._base = self._configs['base'] + self.HOMEgfs = self._base['HOMEgfs'] + self.pslot = self._base['PSLOT'] self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'), - 'HOMEgfs': self._base.get('HOMEgfs'), + 'HOMEgfs': self.HOMEgfs, 'EXPDIR': self._base.get('EXPDIR'), 'NET': self._base.get('NET'), 'CDUMP': self.cdump, @@ -197,40 +199,3 @@ def get_task(self, task_name, *args, **kwargs): raise AttributeError(f'"{task_name}" is not a valid task.\n' + 'Valid tasks are:\n' + f'{", ".join(Tasks.VALID_TASKS)}') - - -def create_wf_task(task_dict): - """ - Create XML for a rocoto task or metatask - - Creates the XML required to define a task and returns the lines - as a list of strings. Tasks can be nested to create metatasks by - defining a key 'task_dict' within the task_dict. When including - a nested task, you also need to provide a 'var_dict' key that - contains a dictionary of variables to loop over. - - All task dicts must include a 'task_name' and a 'cdump'. - - Innermost tasks (regular tasks) additionally require a 'resources' - key containing a dict of values defining the HPC settings. - - Parameters - ---------- - task_dict: dict - Dictionary of task definitions - - Returns - ------- - str - The XML code defining the task - - Raises - ------ - KeyError - If a required key is missing - - """ - - task = rocoto.create_task(task_dict) - - return ''.join(task) From d95c718c9efd79ecaf4483c198b424600da142f6 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 5 Jan 2024 05:28:38 -0600 Subject: [PATCH 07/10] Define rotdir as Task field --- workflow/rocoto/gfs_tasks.py | 6 +++--- workflow/rocoto/rocoto.py | 3 +-- workflow/rocoto/tasks.py | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 00be66314bb..017a5324a44 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1032,7 +1032,7 @@ def ocnpost(self): postenvars = self.envars.copy() postenvar_dict = {'FHRLST': '#lst#', - 'ROTDIR': self._base.get('ROTDIR')} + 'ROTDIR': self.rotdir} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1346,7 +1346,7 @@ def awips_20km_1p0deg(self): awipsenvars = self.envars.copy() awipsenvar_dict = {'FHRGRP': '#grp#', 'FHRLST': '#lst#', - 'ROTDIR': self._base.get('ROTDIR')} + 'ROTDIR': self.rotdir} for key, value in awipsenvar_dict.items(): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1387,7 +1387,7 @@ def awips_g2(self): awipsenvars = self.envars.copy() awipsenvar_dict = {'FHRGRP': '#grp#', 'FHRLST': '#lst#', - 'ROTDIR': self._base.get('ROTDIR')} + 'ROTDIR': self.rotdir} for key, value in awipsenvar_dict.items(): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 3aebade27aa..e315058dbe7 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 from typing import Union, List, Dict, Any -import re ''' MODULE: @@ -71,7 +70,7 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: strings.append('\n') task_dict.update(inner_task_dict) - tasklines = create_task(task_dict) + tasklines = create_task(task_dict).splitlines(True) for tl in tasklines: strings.append(f'{tl}') if tl == '\n' else strings.append(f'\t{tl}') strings.append('\n') diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index de4062f2765..dee86d0d046 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -5,7 +5,7 @@ import rocoto.rocoto as rocoto from wxflow import Template, TemplateConstants, to_timedelta -__all__ = ['Tasks', 'create_wf_task'] +__all__ = ['Tasks'] class Tasks: @@ -43,6 +43,7 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: self._configs = self.app_config.configs self._base = self._configs['base'] self.HOMEgfs = self._base['HOMEgfs'] + self.rotdir = self._base['ROTDIR'] self.pslot = self._base['PSLOT'] self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') From c2189a8660accea8a532d4ae2f43f0360f3c90ba Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 5 Jan 2024 23:08:52 -0600 Subject: [PATCH 08/10] Remove missed entity --- workflow/rocoto/rocoto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index e315058dbe7..679c0952ede 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -123,7 +123,7 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]: nodes = resources_dict.get('nodes', 1) ppn = resources_dict.get('ppn', 1) threads = resources_dict.get('threads', 1) - log = task_dict.get('log', f'&ROTDIR;/logs/@Y@m@d@H/{taskname}.log') + log = task_dict.get('log', 'demo.log') envar = task_dict.get('envars', None) dependency = task_dict.get('dependency', []) From 80a65457028fb4094d7d1c67a675a93fe9d0100b Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 5 Jan 2024 23:09:35 -0600 Subject: [PATCH 09/10] Fix GEFS dependencies --- workflow/rocoto/gefs_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 5df7515218a..c46d9ad4522 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -93,11 +93,11 @@ def fcst(self): # TODO: Add real dependencies dependencies = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} + dep_dict = {'type': 'task', 'name': f'stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'} + dep_dict = {'type': 'task', 'name': f'waveinit'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -120,11 +120,11 @@ def fcst(self): def efcs(self): dependencies = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} + dep_dict = {'type': 'task', 'name': f'stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'} + dep_dict = {'type': 'task', 'name': f'waveinit'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) From 2c44986bacade84ce44f67c665190c5dcf4d0e2f Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Sat, 6 Jan 2024 02:41:37 -0600 Subject: [PATCH 10/10] Fix missing # for ecen task --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 017a5324a44..ba27dff2076 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2457,7 +2457,7 @@ def _get_ecengroups(): resources = self.get_resource('ecen') - task_name = f'{self.cdump}ecen#{varname1}' + task_name = f'{self.cdump}ecen#{varname1}#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies,