From 4386983da92e0dcd4693be41b32ce9c359f26f48 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 10 Mar 2025 17:10:24 +0000 Subject: [PATCH 01/70] Add GEFS awips job --- parm/config/gefs/config.awips | 17 +++++++++++ parm/config/gefs/config.resources | 8 +++++ workflow/applications/gefs.py | 6 ++++ workflow/rocoto/gefs_tasks.py | 51 +++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 parm/config/gefs/config.awips diff --git a/parm/config/gefs/config.awips b/parm/config/gefs/config.awips new file mode 100644 index 00000000000..eda08330233 --- /dev/null +++ b/parm/config/gefs/config.awips @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +########## config.awips ########## +# GFS awips step specific + +echo "BEGIN: config.awips" + +# Maximum number of rocoto tasks per member +export MAX_TASKS=25 + +# Get task specific resources +. "${EXPDIR}/config.resources" awips + +# No. of concurrent awips jobs +export NAWIPSGRP=42 + +echo "END: config.awips" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 54a78e009c3..ab1591235f4 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -259,6 +259,14 @@ case ${step} in export is_exclusive=True ;; + "awips") + export walltime="03:30:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="3GB" + ;; + "oceanice_products") # Walltime is per forecast hour; will be multipled by group size export walltime="00:15:00" diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 9650912459c..2736d426b85 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -44,6 +44,9 @@ def _get_app_configs(self, run): if options['do_aero_fcst']: configs += ['prep_emissions'] + if options['do_awips']: + configs += ['awips'] + if options['do_extractvars']: configs += ['extractvars'] @@ -83,6 +86,9 @@ def get_task_names(self): if options['nens'] > 0: tasks += ['atmos_ensstat'] + if options['do_awips']: + tasks += ['awips'] + if options['do_ocean']: tasks += ['ocean_prod'] diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index dc4578687c7..f1e9813788a 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -314,6 +314,57 @@ def atmos_ensstat(self): return task + def awips(self): + + resources = self.get_resource('awips') + + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + fhrs = self._get_forecast_hours(self.run, self._configs['awips']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['awips']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + + print(self._configs['awips']) + max_tasks = self._configs['awips']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() + postenvar_dict = {'FHR_LIST': '#fhr_list#'} + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_awips_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'cycledef': self.run, + 'command': f'{self.HOMEgfs}/jobs/rocoto/awips.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;'} + + fhr_metatask_dict = {'task_name': f'{self.run}_awips', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + task = rocoto.create_task(fhr_metatask_dict) + + return task + + def wavepostsbs(self): wave_grid = self._configs['base']['waveGRD'] From a503aff087d95ad4f5e51e4d1248dd99ad3d122d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 11 Mar 2025 15:25:18 +0000 Subject: [PATCH 02/70] Add postsnd rocoto tasks --- parm/config/gefs/config.postsnd | 17 +++++++++ parm/config/gefs/config.resources | 27 +++++++++++++ workflow/applications/gefs.py | 6 +++ workflow/rocoto/gefs_tasks.py | 63 ++++++++++++++++++++++++++++++- 4 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 parm/config/gefs/config.postsnd diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd new file mode 100644 index 00000000000..1a417f4c742 --- /dev/null +++ b/parm/config/gefs/config.postsnd @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +########## config.postsnd ########## +# GFS bufr sounding step specific + +echo "BEGIN: config.postsnd" + +# Maximum number of rocoto tasks +export MAX_TASKS=25 + +# Get task specific resources +. $EXPDIR/config.resources postsnd + +export ENDHOUR=180 +if [[ "$FHMAX_GFS" -lt "$ENDHOUR" ]] ; then export ENDHOUR=$FHMAX_GFS ; fi + +echo "END: config.postsnd" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index ab1591235f4..9e8a3ee8e7f 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -250,6 +250,33 @@ case ${step} in export is_exclusive=True ;; + "postsnd") + walltime="02:00:00" + export ntasks=141 + export ntasks_postsndcfp=9 + case "${CASE}" in + "C768") + tasks_per_node=21 + threads_per_task=6 + ;; + "C1152") + tasks_per_node=9 + threads_per_task=14 + ;; + *) + tasks_per_node=21 + threads_per_task=6 + ;; + esac + export tasks_per_node_postsndcfp=1 + postsnd_req_cores=$(( tasks_per_node * threads_per_task )) + if (( postsnd_req_cores > max_tasks_per_node )); then + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + fi + export is_exclusive=True + ;; + + "atmos_ensstat") # Walltime is per forecast hour; will be multipled by group size export walltime="00:15:00" diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 2736d426b85..1f9551a5565 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -30,6 +30,9 @@ def _get_app_configs(self, run): options = self.run_options[run] configs = ['stage_ic', 'fcst', 'atmos_products'] + if options['do_bufrsnd']: + configs += ['postsnd'] + if options['nens'] > 0: configs += ['efcs', 'atmos_ensstat'] @@ -83,6 +86,9 @@ def get_task_names(self): tasks += ['atmos_prod'] + if options['do_bufrsnd']: + tasks += ['postsnd'] + if options['nens'] > 0: tasks += ['atmos_ensstat'] diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f1e9813788a..b84ea603aaa 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -264,6 +264,67 @@ def _atmosoceaniceprod(self, component: str): return task + + def postsnd(self): + + resources = self.get_resource('postsnd') + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_mem#member#_#fhr_label#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['postsnd']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + max_tasks = self._configs['postsnd']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + 'FHR_LIST': '#fhr_list#', + } + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_atmos_prod_mem#member#_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'cycledef': self.run, + 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_mem#member#', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_postsnd', + 'task_dict': fhr_metatask_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + + def atmos_ensstat(self): resources = self.get_resource('atmos_ensstat') @@ -331,8 +392,6 @@ def awips(self): if is_replay and 0 in fhrs: fhrs.remove(0) - - print(self._configs['awips']) max_tasks = self._configs['awips']['MAX_TASKS'] fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) From 2f6c239ab1c3e08a639be7b32e14611e9c92e758 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 13:41:08 -0400 Subject: [PATCH 03/70] added gempak job --- parm/config/gefs/config.gempak | 1 + parm/config/gefs/config.resources | 8 +++++ workflow/applications/gefs.py | 6 ++++ workflow/rocoto/gefs_tasks.py | 58 +++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 parm/config/gefs/config.gempak diff --git a/parm/config/gefs/config.gempak b/parm/config/gefs/config.gempak new file mode 100644 index 00000000000..668de5ba0fb --- /dev/null +++ b/parm/config/gefs/config.gempak @@ -0,0 +1 @@ +../gfs/config.gempak \ No newline at end of file diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 9e8a3ee8e7f..75eb7851cd9 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -354,6 +354,14 @@ case ${step} in export NTASKS=${ntasks} ;; + "gempak") + walltime="00:30:00" + ntasks_gdas=28 + tasks_per_node=28 + threads_per_task=1 + memory="2GB" + ;; + "extractvars") export walltime="00:30:00" export ntasks=1 diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 1f9551a5565..a4cebb65873 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -50,6 +50,9 @@ def _get_app_configs(self, run): if options['do_awips']: configs += ['awips'] + if options['do_gempak']: + configs += ['gempak'] + if options['do_extractvars']: configs += ['extractvars'] @@ -95,6 +98,9 @@ def get_task_names(self): if options['do_awips']: tasks += ['awips'] + if options['do_gempak']: + tasks += ['gempak'] + if options['do_ocean']: tasks += ['ocean_prod'] diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index b84ea603aaa..d53268dda38 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -324,6 +324,64 @@ def postsnd(self): return task + def gempak(self): + + resources = self.get_resource('gempak') + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_mem#member#_#fhr_label#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + fhrs = self._get_forecast_hours(self.run, self._configs['gempak']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['gempak']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + max_tasks = self._configs['gempak']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + 'FHR_LIST': '#fhr_list#', + } + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_atmos_prod_mem#member#_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'cycledef': self.run, + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + fhr_metatask_dict = {'task_name': f'{self.run}_gempak_mem#member#', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_gempak', + 'task_dict': fhr_metatask_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + def atmos_ensstat(self): From d7b7d6f4ca30fd2a6f397ec8cdd9a7f55c1bcd3f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 13:44:31 -0400 Subject: [PATCH 04/70] updated config files --- parm/config/gefs/config.cleanup | 2 +- parm/config/gefs/config.gempak | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.cleanup b/parm/config/gefs/config.cleanup index 839ed6c194f..57b314afc91 120000 --- a/parm/config/gefs/config.cleanup +++ b/parm/config/gefs/config.cleanup @@ -1 +1 @@ -../gfs/config.cleanup \ No newline at end of file +../gfs/config.cleanup diff --git a/parm/config/gefs/config.gempak b/parm/config/gefs/config.gempak index 668de5ba0fb..61e722cd064 100644 --- a/parm/config/gefs/config.gempak +++ b/parm/config/gefs/config.gempak @@ -1 +1 @@ -../gfs/config.gempak \ No newline at end of file +../gfs/config.gempak From 1b8371331c576073afb4d7194526459826707733 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 13:52:06 -0400 Subject: [PATCH 05/70] modified parm\config\gefs\config.resources --- parm/config/gefs/config.resources | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 54699deed7e..6955d8d7c8f 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -355,11 +355,11 @@ case ${step} in ;; "gempak") - walltime="00:30:00" - ntasks_gdas=28 - tasks_per_node=28 - threads_per_task=1 - memory="2GB" + export walltime="00:30:00" + export ntasks_gdas=28 + export tasks_per_node=28 + export threads_per_task=1 + export memory="2GB" ;; "extractvars") From b9151d4b701e09a2212bd3fd5c8130bcbcdfa1c5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 13:54:37 -0400 Subject: [PATCH 06/70] modified config and tasks.py --- parm/config/gefs/config.resources | 3 +-- workflow/rocoto/gefs_tasks.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 6955d8d7c8f..f8b8cc580fc 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -271,12 +271,11 @@ case ${step} in export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) if (( postsnd_req_cores > max_tasks_per_node )); then - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) fi export is_exclusive=True ;; - "atmos_ensstat") # Walltime is per forecast hour; will be multipled by group size export walltime="00:15:00" diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index baf1b3e45dc..13996381275 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -323,7 +323,6 @@ def postsnd(self): return task - def gempak(self): resources = self.get_resource('gempak') @@ -382,7 +381,6 @@ def gempak(self): return task - def atmos_ensstat(self): resources = self.get_resource('atmos_ensstat') @@ -437,7 +435,6 @@ def awips(self): resources = self.get_resource('awips') - deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} deps.append(rocoto.add_dependency(dep_dict)) @@ -481,7 +478,6 @@ def awips(self): return task - def wavepostsbs(self): wave_grid = self._configs['base']['waveGRD'] From 62e48ccd7e48b5cc0bd55e034af4614897c03a7a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 13:58:21 -0400 Subject: [PATCH 07/70] modified gefs_tasts.py --- workflow/rocoto/gefs_tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 13996381275..a7934df0d97 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -264,7 +264,6 @@ def _atmosoceaniceprod(self, component: str): return task - def postsnd(self): resources = self.get_resource('postsnd') @@ -310,8 +309,8 @@ def postsnd(self): fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_mem#member#', 'task_dict': task_dict, - 'var_dict': fhr_var_dict} - + 'var_dict': fhr_var_dict + } member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} member_metatask_dict = {'task_name': f'{self.run}_postsnd', From ba70a86145e9be7978096d16112b626f7cd4e09e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 17:48:29 -0400 Subject: [PATCH 08/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index a7934df0d97..2328fc4e043 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -369,7 +369,6 @@ def gempak(self): 'task_dict': task_dict, 'var_dict': fhr_var_dict} - member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} member_metatask_dict = {'task_name': f'{self.run}_gempak', 'task_dict': fhr_metatask_dict, From f3a8976c55a12ef3e9bd54c60b32298c4ec5220d Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 20:21:40 -0400 Subject: [PATCH 09/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 2328fc4e043..e47df82fd1f 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -290,7 +290,7 @@ def postsnd(self): postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHR_LIST': '#fhr_list#', + 'FHR_LIST': '#fhr_list#' } for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) From 7e1b8aa7a1d216d41e0cb4f3c45c692e4ef3d2d6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 20:23:51 -0400 Subject: [PATCH 10/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index e47df82fd1f..d187a34d2d1 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -290,8 +290,7 @@ def postsnd(self): postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHR_LIST': '#fhr_list#' - } + 'FHR_LIST': '#fhr_list#'} for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -348,8 +347,8 @@ def gempak(self): postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHR_LIST': '#fhr_list#', - } + 'FHR_LIST': '#fhr_list#'} + for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) From dae2e242bb33cd48368f5c635116147a45492c3b Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Thu, 13 Mar 2025 00:44:46 +0000 Subject: [PATCH 11/70] add config.gempak --- parm/config/gefs/config.gempak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 120000 parm/config/gefs/config.gempak diff --git a/parm/config/gefs/config.gempak b/parm/config/gefs/config.gempak deleted file mode 100644 index 61e722cd064..00000000000 --- a/parm/config/gefs/config.gempak +++ /dev/null @@ -1 +0,0 @@ -../gfs/config.gempak diff --git a/parm/config/gefs/config.gempak b/parm/config/gefs/config.gempak new file mode 120000 index 00000000000..668de5ba0fb --- /dev/null +++ b/parm/config/gefs/config.gempak @@ -0,0 +1 @@ +../gfs/config.gempak \ No newline at end of file From f37f6a8c456a992056e889c5630e3ebedccf3f41 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 12 Mar 2025 20:57:14 -0400 Subject: [PATCH 12/70] Revert "updated config files" This reverts commit d7b7d6f4ca30fd2a6f397ec8cdd9a7f55c1bcd3f. change config files --- parm/config/gefs/config.cleanup | 2 +- parm/config/gefs/config.gempak | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.cleanup b/parm/config/gefs/config.cleanup index 57b314afc91..839ed6c194f 120000 --- a/parm/config/gefs/config.cleanup +++ b/parm/config/gefs/config.cleanup @@ -1 +1 @@ -../gfs/config.cleanup +../gfs/config.cleanup \ No newline at end of file diff --git a/parm/config/gefs/config.gempak b/parm/config/gefs/config.gempak index 61e722cd064..668de5ba0fb 100644 --- a/parm/config/gefs/config.gempak +++ b/parm/config/gefs/config.gempak @@ -1 +1 @@ -../gfs/config.gempak +../gfs/config.gempak \ No newline at end of file From b3ffb1632c4e7278363c8f76218842ade508540a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 12:55:44 -0400 Subject: [PATCH 13/70] modified ci\cases\yamls\gefs_defaults_ci.yaml --- ci/cases/yamls/gefs_defaults_ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cases/yamls/gefs_defaults_ci.yaml b/ci/cases/yamls/gefs_defaults_ci.yaml index a19be7713f1..7e57e198737 100644 --- a/ci/cases/yamls/gefs_defaults_ci.yaml +++ b/ci/cases/yamls/gefs_defaults_ci.yaml @@ -6,6 +6,9 @@ base: FHOUT_GFS: 6 USE_OCN_ENS_PERTURB_FILES: "NO" USE_ATM_ENS_PERTURB_FILES: "NO" + DO_BUFRSND: "YES" + DO_GEMPAK: "YES" + DO_AWIPS: "YES" ocn: MOM6_INTERP_ICS: "NO" From 1693e25865bbf6a2bba9d80f209e1ee4b0987a34 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 13:07:26 -0400 Subject: [PATCH 14/70] modify parm\config\gefs\config.resources --- parm/config/gefs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index f8b8cc580fc..ec0bb2fe8e8 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -355,8 +355,8 @@ case ${step} in "gempak") export walltime="00:30:00" - export ntasks_gdas=28 - export tasks_per_node=28 + export ntasks=1 + export tasks_per_node=1 export threads_per_task=1 export memory="2GB" ;; From bfcad4b0ee82d554632996b83140dfc8c8adb91e Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 13:27:12 -0400 Subject: [PATCH 15/70] modify workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index d187a34d2d1..439354b75cb 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -352,7 +352,7 @@ def gempak(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_atmos_prod_mem#member#_#fhr_label#' + task_name = f'{self.run}_gempak_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, From 98563cd095e068fa69fd3461ba770a274866cbff Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Thu, 13 Mar 2025 18:30:13 +0000 Subject: [PATCH 16/70] add .sh files --- jobs/rocoto/awips_test.sh | 24 ++++++++++++++++++++++++ jobs/rocoto/gempak_test.sh | 24 ++++++++++++++++++++++++ jobs/rocoto/postsnd_test.sh | 24 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100755 jobs/rocoto/awips_test.sh create mode 100755 jobs/rocoto/gempak_test.sh create mode 100755 jobs/rocoto/postsnd_test.sh diff --git a/jobs/rocoto/awips_test.sh b/jobs/rocoto/awips_test.sh new file mode 100755 index 00000000000..8e7ba1547e1 --- /dev/null +++ b/jobs/rocoto/awips_test.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source FV3GFS workflow modules + +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +if [[ ${status} -ne 0 ]]; then + exit "${status}" +fi + +export job="postsnd" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +status=$? + + +exit "${status}" + diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh new file mode 100755 index 00000000000..8e7ba1547e1 --- /dev/null +++ b/jobs/rocoto/gempak_test.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source FV3GFS workflow modules + +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +if [[ ${status} -ne 0 ]]; then + exit "${status}" +fi + +export job="postsnd" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +status=$? + + +exit "${status}" + diff --git a/jobs/rocoto/postsnd_test.sh b/jobs/rocoto/postsnd_test.sh new file mode 100755 index 00000000000..8e7ba1547e1 --- /dev/null +++ b/jobs/rocoto/postsnd_test.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source FV3GFS workflow modules + +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +if [[ ${status} -ne 0 ]]; then + exit "${status}" +fi + +export job="postsnd" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +status=$? + + +exit "${status}" + From 20706a9ed9efaf9eaa0df2961d877df4a98d14b6 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 14:42:17 -0400 Subject: [PATCH 17/70] modified sh scripts --- jobs/rocoto/awips_test.sh | 23 ++--------------------- jobs/rocoto/gempak_test.sh | 23 ++--------------------- jobs/rocoto/postsnd_test.sh | 22 ++-------------------- 3 files changed, 6 insertions(+), 62 deletions(-) diff --git a/jobs/rocoto/awips_test.sh b/jobs/rocoto/awips_test.sh index 8e7ba1547e1..310ce906c75 100755 --- a/jobs/rocoto/awips_test.sh +++ b/jobs/rocoto/awips_test.sh @@ -1,24 +1,5 @@ #! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - ############################################################### -# Source FV3GFS workflow modules - -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -status=$? -if [[ ${status} -ne 0 ]]; then - exit "${status}" -fi - -export job="postsnd" -export jobid="${job}.$$" - +echo "awips job passed." ############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND -status=$? - - -exit "${status}" - +exit 0 diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh index 8e7ba1547e1..2dbf2385001 100755 --- a/jobs/rocoto/gempak_test.sh +++ b/jobs/rocoto/gempak_test.sh @@ -1,24 +1,5 @@ #! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - ############################################################### -# Source FV3GFS workflow modules - -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -status=$? -if [[ ${status} -ne 0 ]]; then - exit "${status}" -fi - -export job="postsnd" -export jobid="${job}.$$" - +echo "gempak job passed." ############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND -status=$? - - -exit "${status}" - +exit 0 diff --git a/jobs/rocoto/postsnd_test.sh b/jobs/rocoto/postsnd_test.sh index 8e7ba1547e1..44c6c1edf73 100755 --- a/jobs/rocoto/postsnd_test.sh +++ b/jobs/rocoto/postsnd_test.sh @@ -1,24 +1,6 @@ #! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - ############################################################### -# Source FV3GFS workflow modules - -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -status=$? -if [[ ${status} -ne 0 ]]; then - exit "${status}" -fi - -export job="postsnd" -export jobid="${job}.$$" - +echo "postsnd job passed." ############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND -status=$? - - -exit "${status}" +exit 0 From cdaf2de0df7c00fa0ca0ffbdb27a92fad96f89a8 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 14:45:42 -0400 Subject: [PATCH 18/70] modify workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 439354b75cb..6188718b2de 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -300,7 +300,7 @@ def postsnd(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -358,7 +358,7 @@ def gempak(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -462,7 +462,7 @@ def awips(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/awips.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} From dbde82e3f492dabac928eb6dcba7dde56b9fdeed Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 16:11:01 -0400 Subject: [PATCH 19/70] modified workflow\rocoto\gefs_tasks.py --- parm/config/gefs/config.postsnd | 2 +- workflow/rocoto/gefs_tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd index 1a417f4c742..45ca37fa08f 100644 --- a/parm/config/gefs/config.postsnd +++ b/parm/config/gefs/config.postsnd @@ -9,7 +9,7 @@ echo "BEGIN: config.postsnd" export MAX_TASKS=25 # Get task specific resources -. $EXPDIR/config.resources postsnd +. "$EXPDIR/config.resources" postsnd export ENDHOUR=180 if [[ "$FHMAX_GFS" -lt "$ENDHOUR" ]] ; then export ENDHOUR=$FHMAX_GFS ; fi diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6188718b2de..bb2ba4614fa 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -352,7 +352,7 @@ def gempak(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_gempak_mem#member#_#fhr_label#' + task_name = f'{self.run}_atmos_prod_#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, From ea9016d51948354bcc9ad102c3682f6600e1e81a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 13 Mar 2025 16:16:25 -0400 Subject: [PATCH 20/70] modify workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index bb2ba4614fa..d8b65705a9f 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -352,7 +352,7 @@ def gempak(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_atmos_prod_#member#_#fhr_label#' + task_name = f'{self.run}_gempak_#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, From 81a906cc3c10a02ca65fa274e95fe53bf105030a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 14 Mar 2025 12:15:38 -0400 Subject: [PATCH 21/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index d8b65705a9f..64908a1dcdf 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -326,9 +326,12 @@ def gempak(self): resources = self.get_resource('gempak') deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_mem#member#_#fhr_label#'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + for member in range(0, self.nmem + 1): + task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' + dep_dict = {'type': 'task', 'name': task} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) fhrs = self._get_forecast_hours(self.run, self._configs['gempak']) From 18809ba681e7f8018c4fa4bc8f8c95a39765cbea Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 14 Mar 2025 12:33:44 -0400 Subject: [PATCH 22/70] update workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 64908a1dcdf..00102e418a5 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -269,9 +269,12 @@ def postsnd(self): resources = self.get_resource('postsnd') deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_mem#member#_#fhr_label#'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + for member in range(0, self.nmem + 1): + task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' + dep_dict = {'type': 'task', 'name': task} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) @@ -294,7 +297,7 @@ def postsnd(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_atmos_prod_mem#member#_#fhr_label#' + task_name = f'{self.run}_postsnd_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -306,7 +309,7 @@ def postsnd(self): 'maxtries': '&MAXTRIES;' } - fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_mem#member#', + fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_#member#', 'task_dict': task_dict, 'var_dict': fhr_var_dict } @@ -355,7 +358,7 @@ def gempak(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_gempak_#member#_#fhr_label#' + task_name = f'{self.run}_gempak_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -367,7 +370,7 @@ def gempak(self): 'maxtries': '&MAXTRIES;' } - fhr_metatask_dict = {'task_name': f'{self.run}_gempak_mem#member#', + fhr_metatask_dict = {'task_name': f'{self.run}_gempak_#member#', 'task_dict': task_dict, 'var_dict': fhr_var_dict} From 28b32248e889065ad67a350934574af43de9f320 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 14 Mar 2025 12:41:55 -0400 Subject: [PATCH 23/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 00102e418a5..1b6541bb558 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -439,7 +439,7 @@ def awips(self): resources = self.get_resource('awips') deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} + dep_dict = {'type': 'task', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From 6c15d1733aef7fcd9e55bdadeff13203575d7785 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 14 Mar 2025 12:49:16 -0400 Subject: [PATCH 24/70] modified gefs.py --- workflow/applications/gefs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 2b5d85e65f8..004d7e42bd4 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -33,9 +33,15 @@ def _get_app_configs(self, run): if options['do_bufrsnd']: configs += ['postsnd'] + if options['do_gempak']: + configs += ['gempak'] + if options['nens'] > 0: configs += ['efcs', 'atmos_ensstat'] + if options['do_awips']: + configs += ['awips'] + if options['do_wave']: configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] if options['do_wave_bnd']: @@ -47,12 +53,6 @@ def _get_app_configs(self, run): if options['do_aero_fcst']: configs += ['prep_emissions'] - if options['do_awips']: - configs += ['awips'] - - if options['do_gempak']: - configs += ['gempak'] - if options['do_extractvars']: configs += ['extractvars'] @@ -94,15 +94,15 @@ def get_task_names(self): if options['do_bufrsnd']: tasks += ['postsnd'] + if options['do_gempak']: + tasks += ['gempak'] + if options['nens'] > 0: tasks += ['atmos_ensstat'] if options['do_awips']: tasks += ['awips'] - if options['do_gempak']: - tasks += ['gempak'] - if options['do_ocean']: tasks += ['ocean_prod'] From f9ac8f1ba9a60b03f5575a24f41ab84aced3b1c9 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Sat, 15 Mar 2025 12:50:09 -0400 Subject: [PATCH 25/70] modified workflow\applications\gefs.py --- workflow/applications/gefs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 004d7e42bd4..c82d6939950 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -118,7 +118,6 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] - tasks += ['cleanup'] if options['do_archcom']: tasks += ['arch_tars'] if options['do_globusarch']: From d94946a6835367e94b8dc0149fc33756103ef8d1 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 18 Mar 2025 11:26:52 -0400 Subject: [PATCH 26/70] modify parm\config\gefs\config.postsnd --- parm/config/gefs/config.postsnd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd index 45ca37fa08f..4518a0f8834 100644 --- a/parm/config/gefs/config.postsnd +++ b/parm/config/gefs/config.postsnd @@ -9,9 +9,9 @@ echo "BEGIN: config.postsnd" export MAX_TASKS=25 # Get task specific resources -. "$EXPDIR/config.resources" postsnd +. "$EXPDIR/config.resources" "postsnd" export ENDHOUR=180 -if [[ "$FHMAX_GFS" -lt "$ENDHOUR" ]] ; then export ENDHOUR=$FHMAX_GFS ; fi +if [[ "${FHMAX_GFS}" -lt "${ENDHOUR}" ]] ; then export ENDHOUR=${FHMAX_GFS} ; fi echo "END: config.postsnd" From 3c2ce6fa8674a423820fca8a2e775989276056d2 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 18 Mar 2025 11:28:22 -0400 Subject: [PATCH 27/70] mdify parm\config\gefs\config.postsnd --- parm/config/gefs/config.postsnd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd index 4518a0f8834..7fd7ff43cb5 100644 --- a/parm/config/gefs/config.postsnd +++ b/parm/config/gefs/config.postsnd @@ -9,7 +9,7 @@ echo "BEGIN: config.postsnd" export MAX_TASKS=25 # Get task specific resources -. "$EXPDIR/config.resources" "postsnd" +. "${EXPDIR}/config.resources" postsnd export ENDHOUR=180 if [[ "${FHMAX_GFS}" -lt "${ENDHOUR}" ]] ; then export ENDHOUR=${FHMAX_GFS} ; fi From 529acf38e5adec01732d3133ffa0f14e62ce84ee Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:24:05 -0400 Subject: [PATCH 28/70] Update workflow/rocoto/gefs_tasks.py Co-authored-by: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 1b6541bb558..b70530e1ff9 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -270,7 +270,7 @@ def postsnd(self): deps = [] for member in range(0, self.nmem + 1): - task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' + task = f'{self.run}_fcst_mem#member#_#seg_dep#' dep_dict = {'type': 'task', 'name': task} deps.append(rocoto.add_dependency(dep_dict)) From 4ba60714806dac8ae4c4061f87d094f2172c7062 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 21 Mar 2025 13:25:42 -0400 Subject: [PATCH 29/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index b70530e1ff9..99177e2bc29 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -269,10 +269,9 @@ def postsnd(self): resources = self.get_resource('postsnd') deps = [] - for member in range(0, self.nmem + 1): - task = f'{self.run}_fcst_mem#member#_#seg_dep#' - dep_dict = {'type': 'task', 'name': task} - deps.append(rocoto.add_dependency(dep_dict)) + task = f'{self.run}_fcst_mem#member#_#seg_dep#' + dep_dict = {'type': 'task', 'name': task} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From f5753b10c8c7fd6e8ddb49a8b25e57686ab2faa7 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:54:28 -0400 Subject: [PATCH 30/70] Update workflow/rocoto/gefs_tasks.py Co-authored-by: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 99177e2bc29..7589bdefcf5 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -273,7 +273,7 @@ def postsnd(self): dep_dict = {'type': 'task', 'name': task} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency(dep=deps) fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) From f4fedbf6e8898b6558636264efc4318c8f09319f Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:55:20 -0400 Subject: [PATCH 31/70] Update workflow/rocoto/gefs_tasks.py Co-authored-by: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> --- workflow/rocoto/gefs_tasks.py | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 7589bdefcf5..c3770223904 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -438,45 +438,22 @@ def awips(self): resources = self.get_resource('awips') deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - - fhrs = self._get_forecast_hours(self.run, self._configs['awips']) - - # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs - is_replay = self._configs['awips']['REPLAY_ICS'] - if is_replay and 0 in fhrs: - fhrs.remove(0) - - max_tasks = self._configs['awips']['MAX_TASKS'] - fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) - - # Adjust walltime based on the largest group - largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) - resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) - - postenvars = self.envars.copy() - postenvar_dict = {'FHR_LIST': '#fhr_list#'} - for key, value in postenvar_dict.items(): - postenvars.append(rocoto.create_envar(name=key, value=str(value))) - - task_name = f'{self.run}_awips_#fhr_label#' + + task_name = f'{self.run}_awips' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': postenvars, + 'envars': self.envars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} - fhr_metatask_dict = {'task_name': f'{self.run}_awips', - 'task_dict': task_dict, - 'var_dict': fhr_var_dict} - - task = rocoto.create_task(fhr_metatask_dict) + task = rocoto.create_task(task_dict) return task From 571d1743a8724a30674fa2781f362e75436723f5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 24 Mar 2025 12:07:25 -0400 Subject: [PATCH 32/70] remove white space --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index c3770223904..29f8bb10622 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -441,7 +441,7 @@ def awips(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - + task_name = f'{self.run}_awips' task_dict = {'task_name': task_name, 'resources': resources, From 89ce4004128f37d72912d1be0d5b6b72f03956a4 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Tue, 25 Mar 2025 12:25:50 -0400 Subject: [PATCH 33/70] Update workflow/rocoto/gefs_tasks.py Co-authored-by: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> --- workflow/rocoto/gefs_tasks.py | 37 ++++++++++------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 29f8bb10622..da815b9beb4 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -269,38 +269,25 @@ def postsnd(self): resources = self.get_resource('postsnd') deps = [] - task = f'{self.run}_fcst_mem#member#_#seg_dep#' - dep_dict = {'type': 'task', 'name': task} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) - - # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs - is_replay = self._configs['postsnd']['REPLAY_ICS'] - if is_replay and 0 in fhrs: - fhrs.remove(0) - - max_tasks = self._configs['postsnd']['MAX_TASKS'] - fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) - - # Adjust walltime based on the largest group - largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) - resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) - - postenvars = self.envars.copy() + postsnd_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHR_LIST': '#fhr_list#'} + } + for key, value in postenvar_dict.items(): - postenvars.append(rocoto.create_envar(name=key, value=str(value))) + postsnd_envars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_postsnd_mem#member#_#fhr_label#' + resources = self.get_resource('postsnd') + task_name = f'{self.run}_postsnd_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': postenvars, + 'envars': postsnd_envars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -308,14 +295,9 @@ def postsnd(self): 'maxtries': '&MAXTRIES;' } - fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_#member#', - 'task_dict': task_dict, - 'var_dict': fhr_var_dict - } - member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} member_metatask_dict = {'task_name': f'{self.run}_postsnd', - 'task_dict': fhr_metatask_dict, + 'task_dict': task_dict, 'var_dict': member_var_dict } @@ -323,6 +305,7 @@ def postsnd(self): return task + def gempak(self): resources = self.get_resource('gempak') From c6f25598a6012504548f4de76369aa983f28985b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 25 Mar 2025 12:28:48 -0400 Subject: [PATCH 34/70] modify workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index da815b9beb4..23539cddc3b 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -276,8 +276,7 @@ def postsnd(self): postsnd_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#', - } + 'MEMDIR': 'mem#member#'} for key, value in postenvar_dict.items(): postsnd_envars.append(rocoto.create_envar(name=key, value=str(value))) @@ -305,7 +304,6 @@ def postsnd(self): return task - def gempak(self): resources = self.get_resource('gempak') From 136982ebbd586de3aa95b212c9ed0ac083f6f518 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 27 Mar 2025 10:46:41 -0400 Subject: [PATCH 35/70] update jobs\rocoto\gempak_test.sh --- jobs/rocoto/gempak_test.sh | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh index 2dbf2385001..83fb77e87c3 100755 --- a/jobs/rocoto/gempak_test.sh +++ b/jobs/rocoto/gempak_test.sh @@ -1,5 +1,25 @@ #! /usr/bin/env bash ############################################################### -echo "gempak job passed." -############################################################### +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +if (( status != 0 )); then exit "${status}"; fi + +export job="gempak" + +# shellcheck disable=SC2153 +IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" + +export FHR3 jobid +for fhr in "${fhr_list[@]}"; do + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + ############################################################### + # Execute the JJOB + ############################################################### + echo "add J-script here." + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" +done + exit 0 From c8e0a758de868ac2fc3de1544de2d94d2206a9fe Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 27 Mar 2025 10:49:46 -0400 Subject: [PATCH 36/70] modified jobs\rocoto\gempak_test.sh --- jobs/rocoto/gempak_test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh index 83fb77e87c3..d59e182bd39 100755 --- a/jobs/rocoto/gempak_test.sh +++ b/jobs/rocoto/gempak_test.sh @@ -18,7 +18,9 @@ for fhr in "${fhr_list[@]}"; do # Execute the JJOB ############################################################### echo "add J-script here." - status=$? + # Save the status of the previous command to avoid overwriting + prev_status=$? + status=$prev_status [[ ${status} -ne 0 ]] && exit "${status}" done From cca67b1bbb2200ff55f41e16134d0fe645780914 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 27 Mar 2025 10:54:22 -0400 Subject: [PATCH 37/70] modified jobs\rocoto\gempak_test.sh --- jobs/rocoto/gempak_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh index d59e182bd39..04d2e660402 100755 --- a/jobs/rocoto/gempak_test.sh +++ b/jobs/rocoto/gempak_test.sh @@ -20,7 +20,7 @@ for fhr in "${fhr_list[@]}"; do echo "add J-script here." # Save the status of the previous command to avoid overwriting prev_status=$? - status=$prev_status + status=${prev_status} [[ ${status} -ne 0 ]] && exit "${status}" done From 07073c4c2fc340bdb2e646467f88b2b74e52cb52 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 27 Mar 2025 11:12:55 -0400 Subject: [PATCH 38/70] updated jobs\rocoto\gempak_test.sh --- jobs/rocoto/gempak_test.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh index 04d2e660402..3908a100484 100755 --- a/jobs/rocoto/gempak_test.sh +++ b/jobs/rocoto/gempak_test.sh @@ -18,10 +18,8 @@ for fhr in "${fhr_list[@]}"; do # Execute the JJOB ############################################################### echo "add J-script here." - # Save the status of the previous command to avoid overwriting - prev_status=$? - status=${prev_status} - [[ ${status} -ne 0 ]] && exit "${status}" +# status=$? +# [[ ${status} -ne 0 ]] && exit "${status}" done exit 0 From 4e8b265c4e7d59f0a71f0d029a2188f16424a2ed Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 28 Mar 2025 11:45:47 -0400 Subject: [PATCH 39/70] modify parm\config\gfs\config.gempak --- parm/config/gfs/config.gempak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.gempak b/parm/config/gfs/config.gempak index db5e85af3f7..aca124236ec 100644 --- a/parm/config/gfs/config.gempak +++ b/parm/config/gfs/config.gempak @@ -6,7 +6,7 @@ echo "BEGIN: config.gempak" # Maximum number of rocoto tasks per member -export MAX_TASKS=25 +export MAX_TASKS=5 # Get task specific resources source "${EXPDIR}/config.resources" gempak From 2ba8dde51958d0b2d6bd98ba70bcad9a48f3e597 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Fri, 28 Mar 2025 13:30:58 -0400 Subject: [PATCH 40/70] modified parm\config\gfs\config.gempak --- parm/config/gfs/config.gempak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.gempak b/parm/config/gfs/config.gempak index aca124236ec..db5e85af3f7 100644 --- a/parm/config/gfs/config.gempak +++ b/parm/config/gfs/config.gempak @@ -6,7 +6,7 @@ echo "BEGIN: config.gempak" # Maximum number of rocoto tasks per member -export MAX_TASKS=5 +export MAX_TASKS=25 # Get task specific resources source "${EXPDIR}/config.resources" gempak From fb862077a7ad163c0357e44722111b6718f3fc8f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 31 Mar 2025 16:11:27 -0400 Subject: [PATCH 41/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 23539cddc3b..db67146aa53 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -419,22 +419,45 @@ def awips(self): resources = self.get_resource('awips') deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} + dep_dict = {'type': 'task', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - task_name = f'{self.run}_awips' + fhrs = self._get_forecast_hours(self.run, self._configs['awips']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['awips']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + max_tasks = self._configs['awips']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() + postenvar_dict = {'FHR_LIST': '#fhr_list#'} + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_awips_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': postenvars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_test.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} - task = rocoto.create_task(task_dict) + fhr_metatask_dict = {'task_name': f'{self.run}_awips', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + task = rocoto.create_task(fhr_metatask_dict) return task From 52c175bdc8d653f2bdaa71cc1040416a94f8dae3 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 31 Mar 2025 16:18:30 -0400 Subject: [PATCH 42/70] modfied awips job and added shell scripts --- jobs/rocoto/{awips_test.sh => awips.sh} | 0 jobs/rocoto/gempak.sh | 17 +++++++++-------- jobs/rocoto/gempak_test.sh | 25 ------------------------- jobs/rocoto/postsnd.sh | 3 ++- jobs/rocoto/postsnd_test.sh | 6 ------ workflow/rocoto/gefs_tasks.py | 6 +++--- 6 files changed, 14 insertions(+), 43 deletions(-) rename jobs/rocoto/{awips_test.sh => awips.sh} (100%) mode change 100755 => 100644 delete mode 100755 jobs/rocoto/gempak_test.sh delete mode 100755 jobs/rocoto/postsnd_test.sh diff --git a/jobs/rocoto/awips_test.sh b/jobs/rocoto/awips.sh old mode 100755 new mode 100644 similarity index 100% rename from jobs/rocoto/awips_test.sh rename to jobs/rocoto/awips.sh diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index dc1d3f2621e..2c1ec571433 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -12,14 +12,15 @@ IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" export FHR3 jobid for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - ############################################################### - # Execute the JJOB - ############################################################### - "${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + ############################################################### + # Execute the JJOB + ############################################################### + #"${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" + echo "add J-script here." + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" done exit 0 diff --git a/jobs/rocoto/gempak_test.sh b/jobs/rocoto/gempak_test.sh deleted file mode 100755 index 3908a100484..00000000000 --- a/jobs/rocoto/gempak_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env bash -############################################################### -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -if (( status != 0 )); then exit "${status}"; fi - -export job="gempak" - -# shellcheck disable=SC2153 -IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" - -export FHR3 jobid -for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - ############################################################### - # Execute the JJOB - ############################################################### - echo "add J-script here." -# status=$? -# [[ ${status} -ne 0 ]] && exit "${status}" -done - -exit 0 diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 8e7ba1547e1..6654eb298f4 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -16,7 +16,8 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +#${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +echo "add J-script here." status=$? diff --git a/jobs/rocoto/postsnd_test.sh b/jobs/rocoto/postsnd_test.sh deleted file mode 100755 index 44c6c1edf73..00000000000 --- a/jobs/rocoto/postsnd_test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /usr/bin/env bash -############################################################### -echo "postsnd job passed." -############################################################### -exit 0 - diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index db67146aa53..decc1aac352 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -288,7 +288,7 @@ def postsnd(self): 'dependency': dependencies, 'envars': postsnd_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -344,7 +344,7 @@ def gempak(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -448,7 +448,7 @@ def awips(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/awips.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} From 3fd856c5e7e396ec901be2865adbae01b1cd0b9f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 31 Mar 2025 16:21:08 -0400 Subject: [PATCH 43/70] addressed shellsheck warning --- jobs/rocoto/gempak.sh | 4 ++-- jobs/rocoto/postsnd.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index 2c1ec571433..ca1cde202f4 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -19,8 +19,8 @@ for fhr in "${fhr_list[@]}"; do ############################################################### #"${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" echo "add J-script here." - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" + #status=$? + #[[ ${status} -ne 0 ]] && exit "${status}" done exit 0 diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 6654eb298f4..2c566e5d864 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -17,9 +17,9 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB #${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND -echo "add J-script here." -status=$? - -exit "${status}" +echo "add J-script here." +#status=$? +#exit "${status}" +exit 0 From c835bdca6aa5789c648f716a9a98542684b9d92b Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 31 Mar 2025 16:28:18 -0400 Subject: [PATCH 44/70] added fhrs to awips --- jobs/rocoto/awips.sh | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/awips.sh b/jobs/rocoto/awips.sh index 310ce906c75..1af5bed0a75 100644 --- a/jobs/rocoto/awips.sh +++ b/jobs/rocoto/awips.sh @@ -1,5 +1,26 @@ #! /usr/bin/env bash -############################################################### -echo "awips job passed." -############################################################### + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +if (( status != 0 )); then exit "${status}"; fi + +export job="awips" + +# shellcheck disable=SC2153 +IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" + +export FHR3 jobid +for fhr in "${fhr_list[@]}"; do + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + ############################################################### + # Execute the JJOB + ############################################################### + #"${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" + echo "add J-script here." + #status=$? + #[[ ${status} -ne 0 ]] && exit "${status}" +done + exit 0 From 5cf5bc88555c25d5a2e461252b826c03e8584315 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Mon, 31 Mar 2025 16:28:45 -0400 Subject: [PATCH 45/70] typo --- jobs/rocoto/awips.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/rocoto/awips.sh b/jobs/rocoto/awips.sh index 1af5bed0a75..0288a52e3a2 100644 --- a/jobs/rocoto/awips.sh +++ b/jobs/rocoto/awips.sh @@ -17,7 +17,6 @@ for fhr in "${fhr_list[@]}"; do ############################################################### # Execute the JJOB ############################################################### - #"${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" echo "add J-script here." #status=$? #[[ ${status} -ne 0 ]] && exit "${status}" From c829240dea635a127bfde93098a5e624c683e2cf Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 1 Apr 2025 10:21:43 -0400 Subject: [PATCH 46/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index decc1aac352..46d7a26ffc3 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -274,19 +274,33 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) - postsnd_envars = self.envars.copy() + fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['postsnd']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + max_tasks = self._configs['postsnd']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#'} + 'MEMDIR': 'mem#member#', + 'FHR_LIST': '#fhr_list#'} for key, value in postenvar_dict.items(): - postsnd_envars.append(rocoto.create_envar(name=key, value=str(value))) + postenvars.append(rocoto.create_envar(name=key, value=str(value))) - resources = self.get_resource('postsnd') - task_name = f'{self.run}_postsnd_mem#member#' + task_name = f'{self.run}_postsnd_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': postsnd_envars, + 'envars': postenvars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -294,9 +308,13 @@ def postsnd(self): 'maxtries': '&MAXTRIES;' } + fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_#member#', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} member_metatask_dict = {'task_name': f'{self.run}_postsnd', - 'task_dict': task_dict, + 'task_dict': fhr_metatask_dict, 'var_dict': member_var_dict } From 9766be130533e9f046ecaebd9e2d76f110b2698f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 1 Apr 2025 10:58:11 -0400 Subject: [PATCH 47/70] removed awips --- workflow/rocoto/gefs_tasks.py | 47 ----------------------------------- 1 file changed, 47 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 46d7a26ffc3..6ecc8e4afb7 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -432,53 +432,6 @@ def atmos_ensstat(self): return task - def awips(self): - - resources = self.get_resource('awips') - - deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_atmos_ensstat_#fhr_label#'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - - fhrs = self._get_forecast_hours(self.run, self._configs['awips']) - - # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs - is_replay = self._configs['awips']['REPLAY_ICS'] - if is_replay and 0 in fhrs: - fhrs.remove(0) - - max_tasks = self._configs['awips']['MAX_TASKS'] - fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) - - # Adjust walltime based on the largest group - largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) - resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) - - postenvars = self.envars.copy() - postenvar_dict = {'FHR_LIST': '#fhr_list#'} - for key, value in postenvar_dict.items(): - postenvars.append(rocoto.create_envar(name=key, value=str(value))) - - task_name = f'{self.run}_awips_#fhr_label#' - task_dict = {'task_name': task_name, - 'resources': resources, - 'dependency': dependencies, - 'envars': postenvars, - 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/awips.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', - 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;'} - - fhr_metatask_dict = {'task_name': f'{self.run}_awips', - 'task_dict': task_dict, - 'var_dict': fhr_var_dict} - - task = rocoto.create_task(fhr_metatask_dict) - - return task - def wavepostsbs(self): wave_grid = self._configs['base']['waveGRD'] From cedca464a94528e24a85470d145d2c05e87eadbe Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 1 Apr 2025 11:04:19 -0400 Subject: [PATCH 48/70] added if loop for sh scripts --- jobs/rocoto/gempak.sh | 11 +++++++---- jobs/rocoto/postsnd.sh | 37 +++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index ca1cde202f4..9b22997a9a7 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -17,10 +17,13 @@ for fhr in "${fhr_list[@]}"; do ############################################################### # Execute the JJOB ############################################################### - #"${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" - echo "add J-script here." - #status=$? - #[[ ${status} -ne 0 ]] && exit "${status}" + if [[ "${RUN}" == "gefs" ]]; then + echo "add J-script here for GEFS." + else + "${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" + fi done exit 0 diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 2c566e5d864..aeb29ae610a 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -1,25 +1,26 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -# Source FV3GFS workflow modules - -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? -if [[ ${status} -ne 0 ]]; then - exit "${status}" -fi +if (( status != 0 )); then exit "${status}"; fi export job="postsnd" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -#${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND - -echo "add J-script here." -#status=$? -#exit "${status}" -exit 0 +# shellcheck disable=SC2153 +if [[ "${RUN}" == "gefs" ]]; then + IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" + export FHR3 jobid + for fhr in "${fhr_list[@]}"; do + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + echo "add J-script here for GEFS." + done +else + export jobid="${job}.$$" + ################################################################ + # Execute the JJOB + ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND + status=$? + exit "${status}" +fi From b9e19b1b331b1d7f2819f03711a88e84d7d82dfa Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 1 Apr 2025 11:23:55 -0400 Subject: [PATCH 49/70] removed awips --- ci/cases/yamls/gefs_defaults_ci.yaml | 1 - jobs/rocoto/awips.sh | 25 ------------------------- 2 files changed, 26 deletions(-) delete mode 100644 jobs/rocoto/awips.sh diff --git a/ci/cases/yamls/gefs_defaults_ci.yaml b/ci/cases/yamls/gefs_defaults_ci.yaml index 8f3bf5f88ef..52a5d66ce76 100644 --- a/ci/cases/yamls/gefs_defaults_ci.yaml +++ b/ci/cases/yamls/gefs_defaults_ci.yaml @@ -7,6 +7,5 @@ base: USE_ATM_ENS_PERTURB_FILES: "NO" DO_BUFRSND: "YES" DO_GEMPAK: "YES" - DO_AWIPS: "YES" ocn: MOM6_INTERP_ICS: "NO" diff --git a/jobs/rocoto/awips.sh b/jobs/rocoto/awips.sh deleted file mode 100644 index 0288a52e3a2..00000000000 --- a/jobs/rocoto/awips.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -if (( status != 0 )); then exit "${status}"; fi - -export job="awips" - -# shellcheck disable=SC2153 -IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" - -export FHR3 jobid -for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - ############################################################### - # Execute the JJOB - ############################################################### - echo "add J-script here." - #status=$? - #[[ ${status} -ne 0 ]] && exit "${status}" -done - -exit 0 From c3e6b0972c983fb2302f96a30c3bfcd82d20726f Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Tue, 1 Apr 2025 11:30:36 -0400 Subject: [PATCH 50/70] remove awips2 --- parm/config/gefs/config.awips | 17 ----------------- parm/config/gefs/config.resources | 8 -------- workflow/applications/gefs.py | 6 ------ 3 files changed, 31 deletions(-) delete mode 100644 parm/config/gefs/config.awips diff --git a/parm/config/gefs/config.awips b/parm/config/gefs/config.awips deleted file mode 100644 index eda08330233..00000000000 --- a/parm/config/gefs/config.awips +++ /dev/null @@ -1,17 +0,0 @@ -#! /usr/bin/env bash - -########## config.awips ########## -# GFS awips step specific - -echo "BEGIN: config.awips" - -# Maximum number of rocoto tasks per member -export MAX_TASKS=25 - -# Get task specific resources -. "${EXPDIR}/config.resources" awips - -# No. of concurrent awips jobs -export NAWIPSGRP=42 - -echo "END: config.awips" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index f11ef47a670..ea80b9c398b 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -276,14 +276,6 @@ case ${step} in export is_exclusive=True ;; - "awips") - export walltime="03:30:00" - export ntasks=1 - export tasks_per_node=1 - export threads_per_task=1 - export memory="3GB" - ;; - "oceanice_products") # Walltime is per forecast hour; will be multipled by group size export walltime="00:15:00" diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c82d6939950..591025e23fc 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -39,9 +39,6 @@ def _get_app_configs(self, run): if options['nens'] > 0: configs += ['efcs', 'atmos_ensstat'] - if options['do_awips']: - configs += ['awips'] - if options['do_wave']: configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] if options['do_wave_bnd']: @@ -100,9 +97,6 @@ def get_task_names(self): if options['nens'] > 0: tasks += ['atmos_ensstat'] - if options['do_awips']: - tasks += ['awips'] - if options['do_ocean']: tasks += ['ocean_prod'] From 2514eefb1a93dd609dc50eb9702d6ada97f8449a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 10:45:49 -0400 Subject: [PATCH 51/70] Huber's correctons --- jobs/rocoto/postsnd.sh | 2 +- parm/config/gefs/config.resources | 16 ++-------------- workflow/applications/gefs.py | 4 +++- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index aeb29ae610a..f37861d8f42 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -20,7 +20,7 @@ else export jobid="${job}.$$" ################################################################ # Execute the JJOB - ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND + ${HOMEgfs}/jobs/J${RUN^^}_ATMOS_POSTSND status=$? exit "${status}" fi diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index ea80b9c398b..c77cfb6b087 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -245,20 +245,8 @@ case ${step} in walltime="02:00:00" export ntasks=141 export ntasks_postsndcfp=9 - case "${CASE}" in - "C768") - tasks_per_node=21 - threads_per_task=6 - ;; - "C1152") - tasks_per_node=9 - threads_per_task=14 - ;; - *) - tasks_per_node=21 - threads_per_task=6 - ;; - esac + export tasks_per_node=21 + export threads_per_task=6 export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) if (( postsnd_req_cores > max_tasks_per_node )); then diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 591025e23fc..a3b9d0e7347 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -112,11 +112,13 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] + tasks += ['cleanup'] + if options['do_archcom']: tasks += ['arch_tars'] if options['do_globusarch']: tasks += ['globus'] - tasks += ['arch_vrfy', 'cleanup'] + tasks += ['arch_vrfy'] return {f"{self.run}": tasks} From cd2174c3e77efcc0b7db0bda6c258eecda60e37a Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 10:54:04 -0400 Subject: [PATCH 52/70] remove extra cleanup task --- workflow/applications/gefs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index a3b9d0e7347..591025e23fc 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -112,13 +112,11 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] - tasks += ['cleanup'] - if options['do_archcom']: tasks += ['arch_tars'] if options['do_globusarch']: tasks += ['globus'] - tasks += ['arch_vrfy'] + tasks += ['arch_vrfy', 'cleanup'] return {f"{self.run}": tasks} From 6eeb0590ec5a065483e597e9f243129eaa808d4e Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:44:58 -0400 Subject: [PATCH 53/70] Update jobs/rocoto/postsnd.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- jobs/rocoto/postsnd.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index f37861d8f42..5857a2c5bff 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -3,7 +3,9 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? -if (( status != 0 )); then exit "${status}"; fi +if [[ ${status} -ne 0 ]]; then + exit "${status}" +fi export job="postsnd" From 9b918b173d4e87bb5e1093301086aa0fe0173db6 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:45:13 -0400 Subject: [PATCH 54/70] Update parm/config/gefs/config.resources Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- parm/config/gefs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index c77cfb6b087..55e014ce7bf 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -249,7 +249,7 @@ case ${step} in export threads_per_task=6 export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) - if (( postsnd_req_cores > max_tasks_per_node )); then +if [[ ${postsnd_req_cores} -gt ${max_tasks_per_node} ]]; then tasks_per_node=$(( max_tasks_per_node / threads_per_task )) fi export is_exclusive=True From 63f1b5ec902e36d997fa9fcbfc0916c1bf59acc5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 11:46:46 -0400 Subject: [PATCH 55/70] modified parm\config\gefs\config.resources --- parm/config/gefs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 55e014ce7bf..19c31528ef2 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -249,7 +249,7 @@ case ${step} in export threads_per_task=6 export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) -if [[ ${postsnd_req_cores} -gt ${max_tasks_per_node} ]]; then + if [[ ${postsnd_req_cores} -gt ${max_tasks_per_node} ]]; then tasks_per_node=$(( max_tasks_per_node / threads_per_task )) fi export is_exclusive=True From d08ce4206ffe09fc3352846579a8bd957ad767bd Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:15:23 -0400 Subject: [PATCH 56/70] Update jobs/rocoto/gempak.sh Co-authored-by: Kate Friedman --- jobs/rocoto/gempak.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index 9b22997a9a7..e7c1ad2b6ea 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -21,8 +21,10 @@ for fhr in "${fhr_list[@]}"; do echo "add J-script here for GEFS." else "${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" + err=$? + if [[ ${err} -ne 0 ]]: then + exit "${err}" + fi fi done From 838c5fbfc867abe12752a6895762dd0d2c78a870 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:15:43 -0400 Subject: [PATCH 57/70] Update jobs/rocoto/postsnd.sh Co-authored-by: Kate Friedman --- jobs/rocoto/postsnd.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 5857a2c5bff..da8be99f22b 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -23,6 +23,6 @@ else ################################################################ # Execute the JJOB ${HOMEgfs}/jobs/J${RUN^^}_ATMOS_POSTSND - status=$? - exit "${status}" + err=$? + exit "${err}" fi From 38d1794b1917f9e8c228de559a8908c4a7b1e16b Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:16:03 -0400 Subject: [PATCH 58/70] Update jobs/rocoto/postsnd.sh Co-authored-by: Kate Friedman --- jobs/rocoto/postsnd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index da8be99f22b..0e71dd49c62 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -22,7 +22,7 @@ else export jobid="${job}.$$" ################################################################ # Execute the JJOB - ${HOMEgfs}/jobs/J${RUN^^}_ATMOS_POSTSND + ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND err=$? exit "${err}" fi From 5d2c131a917ba645ad3aeecc2b2cb57c78814003 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 13:19:23 -0400 Subject: [PATCH 59/70] modified jobs\rocoto\gempak.sh --- jobs/rocoto/gempak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index e7c1ad2b6ea..01a2d417705 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -20,7 +20,7 @@ for fhr in "${fhr_list[@]}"; do if [[ "${RUN}" == "gefs" ]]; then echo "add J-script here for GEFS." else - "${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" + "${HOMEgfs}/jobs/JGFS_ATMOS_GEMPAK" err=$? if [[ ${err} -ne 0 ]]: then exit "${err}" From aceb1ec4e21791fdd16e87e80b58add517450a5c Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 13:22:17 -0400 Subject: [PATCH 60/70] typo --- jobs/rocoto/gempak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index 01a2d417705..9dbc574f350 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -22,7 +22,7 @@ for fhr in "${fhr_list[@]}"; do else "${HOMEgfs}/jobs/JGFS_ATMOS_GEMPAK" err=$? - if [[ ${err} -ne 0 ]]: then + if [[ ${err} -ne 0 ]]; then exit "${err}" fi fi From f46fb7abae8448cdc9829ecc7f56d6610a5b0efc Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 16:39:38 -0400 Subject: [PATCH 61/70] comments aerorahul --- jobs/rocoto/gempak.sh | 26 +++++++++++++------------- parm/config/gefs/config.resources | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index 9dbc574f350..a0cbb7f09d5 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -12,20 +12,20 @@ IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" export FHR3 jobid for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - ############################################################### - # Execute the JJOB - ############################################################### - if [[ "${RUN}" == "gefs" ]]; then - echo "add J-script here for GEFS." - else - "${HOMEgfs}/jobs/JGFS_ATMOS_GEMPAK" - err=$? - if [[ ${err} -ne 0 ]]; then - exit "${err}" - fi + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + ############################################################### + # Execute the JJOB + ############################################################### + if [[ "${RUN}" == "gefs" ]]; then + echo "add J-script here for GEFS." + else + "${HOMEgfs}/jobs/JGFS_ATMOS_GEMPAK" + err=$? + if [[ ${err} -ne 0 ]]; then + exit "${err}" fi + fi done exit 0 diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 19c31528ef2..88268ba9364 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -242,7 +242,7 @@ case ${step} in ;; "postsnd") - walltime="02:00:00" + export walltime="02:00:00" export ntasks=141 export ntasks_postsndcfp=9 export tasks_per_node=21 From b34152e30db0f5289360d78f91f3fd6254c1e060 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Wed, 2 Apr 2025 16:46:55 -0400 Subject: [PATCH 62/70] changed indentation postsnd.sh --- jobs/rocoto/postsnd.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 0e71dd49c62..30027662224 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -4,25 +4,25 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then - exit "${status}" + exit "${status}" fi export job="postsnd" # shellcheck disable=SC2153 if [[ "${RUN}" == "gefs" ]]; then - IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" - export FHR3 jobid - for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - echo "add J-script here for GEFS." - done + IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" + export FHR3 jobid + for fhr in "${fhr_list[@]}"; do + FHR3=$(printf '%03d' "${fhr}") + jobid="${job}_f${FHR3}.$$" + echo "add J-script here for GEFS." + done else - export jobid="${job}.$$" - ################################################################ - # Execute the JJOB - ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND - err=$? - exit "${err}" + export jobid="${job}.$$" + ################################################################ + # Execute the JJOB + ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND + err=$? + exit "${err}" fi From b13abd3d20895b81eed4998544ea6af6540f87e5 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 10:29:47 -0400 Subject: [PATCH 63/70] modified postsnd.sh --- jobs/rocoto/postsnd.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 30027662224..476367c403e 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -8,18 +8,12 @@ if [[ ${status} -ne 0 ]]; then fi export job="postsnd" +export jobid="${job}.$$" # shellcheck disable=SC2153 if [[ "${RUN}" == "gefs" ]]; then - IFS=', ' read -r -a fhr_list <<< "${FHR_LIST}" - export FHR3 jobid - for fhr in "${fhr_list[@]}"; do - FHR3=$(printf '%03d' "${fhr}") - jobid="${job}_f${FHR3}.$$" - echo "add J-script here for GEFS." - done + echo "add J-script here for GEFS." else - export jobid="${job}.$$" ################################################################ # Execute the JJOB ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND From b78891cf21d526e165f73a5ab101c301fd368c29 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 10:37:56 -0400 Subject: [PATCH 64/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6ecc8e4afb7..9181f91ea41 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -274,33 +274,20 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) - fhrs = self._get_forecast_hours(self.run, self._configs['postsnd']) - - # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs - is_replay = self._configs['postsnd']['REPLAY_ICS'] - if is_replay and 0 in fhrs: - fhrs.remove(0) - - max_tasks = self._configs['postsnd']['MAX_TASKS'] - fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) - - # Adjust walltime based on the largest group - largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) - resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) - - postenvars = self.envars.copy() + postsnd_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', - 'FHR_LIST': '#fhr_list#'} + } for key, value in postenvar_dict.items(): - postenvars.append(rocoto.create_envar(name=key, value=str(value))) + postsnd_envars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'{self.run}_postsnd_mem#member#_#fhr_label#' + resources = self.get_resource('postsnd') + task_name = f'{self.run}_postsnd_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': postenvars, + 'envars': postsnd_envars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -308,13 +295,9 @@ def postsnd(self): 'maxtries': '&MAXTRIES;' } - fhr_metatask_dict = {'task_name': f'{self.run}_postsnd_#member#', - 'task_dict': task_dict, - 'var_dict': fhr_var_dict} - member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} member_metatask_dict = {'task_name': f'{self.run}_postsnd', - 'task_dict': fhr_metatask_dict, + 'task_dict': task_dict, 'var_dict': member_var_dict } From 67b357d7090af95a4a0d3fb0031e620784802c01 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 10:41:28 -0400 Subject: [PATCH 65/70] typo workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 9181f91ea41..47aefa3ee3c 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -276,7 +276,7 @@ def postsnd(self): postsnd_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#', + 'MEMDIR': 'mem#member#' } for key, value in postenvar_dict.items(): From 9b6e45019f88cb63a1beeb62f575eff6f5d1f8e7 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 10:44:35 -0400 Subject: [PATCH 66/70] typo workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 47aefa3ee3c..3f522ac8a17 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -276,8 +276,7 @@ def postsnd(self): postsnd_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#' - } + 'MEMDIR': 'mem#member#'} for key, value in postenvar_dict.items(): postsnd_envars.append(rocoto.create_envar(name=key, value=str(value))) From 3b271309469151eb2af29a8fd805a3bcbed2f206 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 15:33:15 -0400 Subject: [PATCH 67/70] aerorahul changes --- ci/cases/yamls/gefs_defaults_ci.yaml | 4 ++-- jobs/rocoto/gempak.sh | 12 +++--------- jobs/rocoto/postsnd.sh | 15 +++++---------- workflow/rocoto/gefs_tasks.py | 6 +++--- 4 files changed, 13 insertions(+), 24 deletions(-) diff --git a/ci/cases/yamls/gefs_defaults_ci.yaml b/ci/cases/yamls/gefs_defaults_ci.yaml index 52a5d66ce76..99733ca821d 100644 --- a/ci/cases/yamls/gefs_defaults_ci.yaml +++ b/ci/cases/yamls/gefs_defaults_ci.yaml @@ -5,7 +5,7 @@ base: FHOUT_GFS: 6 USE_OCN_ENS_PERTURB_FILES: "NO" USE_ATM_ENS_PERTURB_FILES: "NO" - DO_BUFRSND: "YES" - DO_GEMPAK: "YES" + DO_BUFRSND: "NO" + DO_GEMPAK: "NO" ocn: MOM6_INTERP_ICS: "NO" diff --git a/jobs/rocoto/gempak.sh b/jobs/rocoto/gempak.sh index a0cbb7f09d5..41121089d88 100755 --- a/jobs/rocoto/gempak.sh +++ b/jobs/rocoto/gempak.sh @@ -17,15 +17,9 @@ for fhr in "${fhr_list[@]}"; do ############################################################### # Execute the JJOB ############################################################### - if [[ "${RUN}" == "gefs" ]]; then - echo "add J-script here for GEFS." - else - "${HOMEgfs}/jobs/JGFS_ATMOS_GEMPAK" - err=$? - if [[ ${err} -ne 0 ]]; then - exit "${err}" - fi - fi + "${HOMEgfs}/jobs/J${RUN^^}_ATMOS_GEMPAK" + err=$? + [[ ${err} -ne 0 ]] && exit "${err}" done exit 0 diff --git a/jobs/rocoto/postsnd.sh b/jobs/rocoto/postsnd.sh index 476367c403e..e0a869b8976 100755 --- a/jobs/rocoto/postsnd.sh +++ b/jobs/rocoto/postsnd.sh @@ -10,13 +10,8 @@ fi export job="postsnd" export jobid="${job}.$$" -# shellcheck disable=SC2153 -if [[ "${RUN}" == "gefs" ]]; then - echo "add J-script here for GEFS." -else - ################################################################ - # Execute the JJOB - ${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND - err=$? - exit "${err}" -fi +################################################################ +# Execute the JJOB +${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +err=$? +exit "${err}" diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 3f522ac8a17..f0ac681a42d 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -269,7 +269,7 @@ def postsnd(self): resources = self.get_resource('postsnd') deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} + dep_dict = {'type': 'task', 'name': f'{self.run}_fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -289,7 +289,7 @@ def postsnd(self): 'envars': postsnd_envars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', + 'job_name': f'{self.pslot}_{task_name}', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } @@ -345,7 +345,7 @@ def gempak(self): 'envars': postenvars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', + 'job_name': f'{self.pslot}_{task_name}', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } From df8d809599c790496fa32da3c9966fff57c8a9d2 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 15:51:23 -0400 Subject: [PATCH 68/70] modified workflow\rocoto\gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 9205460325f..9b20b95d32d 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -289,7 +289,7 @@ def postsnd(self): 'envars': postsnd_envars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', - 'job_name': f'{self.pslot}_{task_name}', + 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } @@ -309,10 +309,9 @@ def gempak(self): resources = self.get_resource('gempak') deps = [] - for member in range(0, self.nmem + 1): - task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' - dep_dict = {'type': 'task', 'name': task} - deps.append(rocoto.add_dependency(dep_dict)) + task = f'{self.run}_atmos_prod_mem#member#_#fhr_label#' + dep_dict = {'type': 'task', 'name': task} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -345,7 +344,7 @@ def gempak(self): 'envars': postenvars, 'cycledef': self.run, 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', - 'job_name': f'{self.pslot}_{task_name}', + 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } From c24b33a1d9e77b664056353a40b7f4389fbc2ff1 Mon Sep 17 00:00:00 2001 From: AntonMFernando Date: Thu, 3 Apr 2025 16:10:08 -0400 Subject: [PATCH 69/70] removed config.postsnd --- parm/config/gefs/config.postsnd | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 parm/config/gefs/config.postsnd diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd deleted file mode 100644 index 7fd7ff43cb5..00000000000 --- a/parm/config/gefs/config.postsnd +++ /dev/null @@ -1,17 +0,0 @@ -#! /usr/bin/env bash - -########## config.postsnd ########## -# GFS bufr sounding step specific - -echo "BEGIN: config.postsnd" - -# Maximum number of rocoto tasks -export MAX_TASKS=25 - -# Get task specific resources -. "${EXPDIR}/config.resources" postsnd - -export ENDHOUR=180 -if [[ "${FHMAX_GFS}" -lt "${ENDHOUR}" ]] ; then export ENDHOUR=${FHMAX_GFS} ; fi - -echo "END: config.postsnd" From b2909d073aa06db78dbc3b81576cbeb391a7e0cb Mon Sep 17 00:00:00 2001 From: Anton Fernando Date: Thu, 3 Apr 2025 20:20:23 +0000 Subject: [PATCH 70/70] added config.postsnd --- parm/config/gefs/config.postsnd | 1 + 1 file changed, 1 insertion(+) create mode 120000 parm/config/gefs/config.postsnd diff --git a/parm/config/gefs/config.postsnd b/parm/config/gefs/config.postsnd new file mode 120000 index 00000000000..3ca118e114d --- /dev/null +++ b/parm/config/gefs/config.postsnd @@ -0,0 +1 @@ +../gfs/config.postsnd \ No newline at end of file