From fbe784df6f7c7ed062521ae44e681d47adac6750 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:18:53 +0000 Subject: [PATCH 01/17] Stubs for anlstat job --- jobs/JGLOBAL_ANALYSIS_STATS | 0 jobs/rocoto/anlstat.sh | 0 parm/config/gfs/config.base | 11 ++++++-- scripts/exglobal_analysis_stats.py | 0 workflow/applications/applications.py | 1 + workflow/applications/gfs_cycled.py | 9 ++++++ workflow/rocoto/gfs_tasks.py | 40 +++++++++++++++++++++++++++ 7 files changed, 58 insertions(+), 3 deletions(-) create mode 100755 jobs/JGLOBAL_ANALYSIS_STATS create mode 100755 jobs/rocoto/anlstat.sh create mode 100755 scripts/exglobal_analysis_stats.py diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS new file mode 100755 index 00000000000..e69de29bb2d diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh new file mode 100755 index 00000000000..e69de29bb2d diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index e6a626cfe37..05a3fb76330 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -73,9 +73,6 @@ export DO_NPOESS="@DO_NPOESS@" # NPOESS products export DO_TRACKER="@DO_TRACKER@" # Hurricane track verification export DO_GENESIS="@DO_GENESIS@" # Cyclone genesis verification export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU) -export DO_VERFOZN="YES" # Ozone data assimilation monitoring -export DO_VERFRAD="YES" # Radiance data assimilation monitoring -export DO_VMINMON="YES" # GSI minimization monitoring export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel @@ -474,6 +471,14 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring + export DO_ANLSTAT="YES" # JEDI-based analysis statistics +else + export DO_VERFOZN="YES" # Ozone data assimilation monitoring + export DO_VERFRAD="YES" # Radiance data assimilation monitoring + export DO_VMINMON="YES" # GSI minimization monitoring + if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then + export DO_ANLSTAT="YES" # JEDI-based analysis statistics + fi fi # If starting ICs that are not at cycle hour diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py new file mode 100755 index 00000000000..e69de29bb2d diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 8c1f69735ef..9f8283eae3b 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -63,6 +63,7 @@ def __init__(self, conf: Configuration) -> None: self.do_verfozn = _base.get('DO_VERFOZN', True) self.do_verfrad = _base.get('DO_VERFRAD', True) self.do_vminmon = _base.get('DO_VMINMON', True) + self.do_anlstat = _base.get('DO_ANLSTAT', False) self.do_tracker = _base.get('DO_TRACKER', True) self.do_genesis = _base.get('DO_GENESIS', True) self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e049a7d422f..825925cb06c 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -74,6 +74,9 @@ def _get_app_configs(self): if self.do_vminmon: configs += ['vminmon'] + if self.do_anlstat: + configs += ['anlstat'] + if self.do_tracker: configs += ['tracker'] @@ -206,6 +209,9 @@ def get_task_names(self): if self.do_vminmon: gdas_tasks += ['vminmon'] + if self.do_anlstat: + gdas_tasks += ['anlstat'] + if self.do_gempak: gdas_tasks += ['gempak', 'gempakmetancdc'] @@ -239,6 +245,9 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] + + if self.do_anlstat: + gfs_tasks += ['anlstat'] if self.do_tracker: gfs_tasks += ['tracker'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9d9b28fb175..2fabc870897 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1731,6 +1731,40 @@ def vminmon(self): return task + def anlstat(self): + deps = [] + if self.app_config.do_jediatmvar: + dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jediocnvar: + dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jedisnowda: + dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_aero: + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('anlstat') + task_name = f'{self.run}anlstat' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/anlstat.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def tracker(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}atmos_prod'} @@ -2225,6 +2259,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2240,6 +2277,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}tracker'} deps.append(rocoto.add_dependency(dep_dict)) From cce943c474708b301e5deaa87dfa3dd172a6c22e Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:22:27 +0000 Subject: [PATCH 02/17] update for pynorms --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 825925cb06c..881694bcfbc 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -245,7 +245,7 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] - + if self.do_anlstat: gfs_tasks += ['anlstat'] From 3f7a4ae14752ccdb331799c051c39dc8eda51504 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:57:40 +0000 Subject: [PATCH 03/17] add in stub scripts --- jobs/JGLOBAL_ANALYSIS_STATS | 35 ++++++++++++++++++++++++++++++ jobs/rocoto/anlstat.sh | 18 +++++++++++++++ scripts/exglobal_analysis_stats.py | 20 +++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index e69de29bb2d..6543735fb19 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + + +############################################################### +# Run relevant script + +EXSCRIPT=${ANLSTATSPY:-${SCRgfs}/exglobal_analysis_stats.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 \ No newline at end of file diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh index e69de29bb2d..52292a582c2 100755 --- a/jobs/rocoto/anlstat.sh +++ b/jobs/rocoto/anlstat.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="anlstat" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS" +status=$? +exit "${status}" \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index e69de29bb2d..045fc914b3e 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# exglobal_analysis_stats.py +# This script will run the OOPS/JEDI code to + +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atm_analysis import AtmAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # placeholder + print('Do nothing right now!') \ No newline at end of file From 0ac03dcf91bf38a1df6e02f813a184af62015e55 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:59:46 +0000 Subject: [PATCH 04/17] EOLs --- jobs/JGLOBAL_ANALYSIS_STATS | 2 +- jobs/rocoto/anlstat.sh | 2 +- scripts/exglobal_analysis_stats.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 6543735fb19..da0193ea82d 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -32,4 +32,4 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi -exit 0 \ No newline at end of file +exit 0 diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh index 52292a582c2..ac7d8af16a2 100755 --- a/jobs/rocoto/anlstat.sh +++ b/jobs/rocoto/anlstat.sh @@ -15,4 +15,4 @@ export jobid="${job}.$$" # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS" status=$? -exit "${status}" \ No newline at end of file +exit "${status}" diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 045fc914b3e..e96a50e7b87 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -17,4 +17,4 @@ config = cast_strdict_as_dtypedict(os.environ) # placeholder - print('Do nothing right now!') \ No newline at end of file + print('Do nothing right now!') From 80649390283e9764bb41a1acb831d94312598533 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 20 Aug 2024 13:42:14 +0000 Subject: [PATCH 05/17] update parm/config/gfs --- parm/config/gfs/config.anlstat | 11 +++++++++++ parm/config/gfs/config.resources | 10 +++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 parm/config/gfs/config.anlstat diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat new file mode 100644 index 00000000000..d9ce48dc0c8 --- /dev/null +++ b/parm/config/gfs/config.anlstat @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.anlstat ########## +# Analysis Stat + +echo "BEGIN: config.anlstat" + +# Get task specific resources +source "${EXPDIR}/config.resources" anlstat + +echo "END: config.anlstat" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cec2aef2389..e84f23a1355 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -17,7 +17,7 @@ if (( $# != 1 )); then echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" - echo "anal sfcanl analcalc analdiag fcst echgres" + echo "anal sfcanl analcalc analdiag anlstat fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" echo "verfozn verfrad vminmon fit2obs metp arch cleanup" @@ -626,6 +626,14 @@ case ${step} in memory="48GB" ;; + "anlstat") + walltime="00:30:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="24GB" + ;; + "sfcanl") walltime="00:20:00" ntasks=${ntiles:-6} From f6933900a62a9168b65936d335d893d5ca3ebc04 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 20 Aug 2024 18:04:56 +0000 Subject: [PATCH 06/17] typos --- workflow/rocoto/gfs_tasks.py | 6 +++--- workflow/rocoto/tasks.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 2fabc870897..4d2f3704353 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1736,13 +1736,13 @@ def anlstat(self): if self.app_config.do_jediatmvar: dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jediocnvar: + if self.app_config.do_jediocnvar: dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jedisnowda: + if self.app_config.do_jedisnowda: dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_aero: + if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 64952498d45..3c8e1c8933c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,6 +22,7 @@ class Tasks: 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', + 'anlstat', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', From e4f0733fc532148715706e5b5227454358b0c143 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 5 Sep 2024 15:58:18 +0000 Subject: [PATCH 07/17] first pass at stat analyis python script. Copying aerostat file --- ush/python/pygfs/task/stat_analysis.py | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 ush/python/pygfs/task/stat_analysis.py diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py new file mode 100644 index 00000000000..8a452bfa573 --- /dev/null +++ b/ush/python/pygfs/task/stat_analysis.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import Dict, List +from pprint import pformat +import numpy as np +from netCDF4 import Dataset + +from wxflow import (AttrDict, + FileHandler, + to_fv3time, to_YMD, to_YMDH, to_timedelta, add_to_datetime, + rm_p, + parse_j2yaml, save_as_yaml, + Jinja, + logit, + Executable, + WorkflowException) +from pygfs.task.analysis import Analysis + +logger = getLogger(__name__.split('.')[-1]) + +class StatAnalysis(Analysis): + """ + Class for global stat analysis tasks + """ + + @logit(logger, name="StatAnalysis") + def __init__(self, config): + super().__init__(config) + + _res = int(self.task_config['CASE'][1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _letkfoi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.letkfoi.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'jedi_yaml': _letkfoi_yaml + } + ) + + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self: Analysis) -> None: + """ + Initialize global stat analysis. + """ + super().initialize() + + logger.info(f"Copying files to {self.task_config.DATA}/stats") + + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + dest = os.path.join(self.task_config.DATA, "stats") + statlist = [aerostat, dest] + FileHandler({'copy': statlist}).sync() + + \ No newline at end of file From d412762c519d9741713fb8e6b48e485cd9518e3b Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 5 Sep 2024 16:06:24 +0000 Subject: [PATCH 08/17] add exglobal stat analysis script --- scripts/exglobal_stat_analysis.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 scripts/exglobal_stat_analysis.py diff --git a/scripts/exglobal_stat_analysis.py b/scripts/exglobal_stat_analysis.py new file mode 100644 index 00000000000..7ae192b89b1 --- /dev/null +++ b/scripts/exglobal_stat_analysis.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# exglobal_stat_analysis.py +# This script creates an StatAnalysis class +# and runs the initialize, execute and finalize methods +# for a global stat analysis +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.stat_analysis import StatAnalysis + +# Initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the snow analysis task + anl = StatAnalysis(config) + anl.initialize() \ No newline at end of file From 17dbc1593e738934de09bc6044a70ca3c69f3ed4 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 5 Sep 2024 16:13:11 +0000 Subject: [PATCH 09/17] update scripts --- scripts/exglobal_analysis_stats.py | 7 ++++--- scripts/exglobal_stat_analysis.py | 22 ---------------------- 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 scripts/exglobal_stat_analysis.py diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index e96a50e7b87..23c2a3a7eb2 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -5,7 +5,7 @@ import os from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.atm_analysis import AtmAnalysis +from pygfs.task.stat_analysis import StatAnalysis # Initialize root logger logger = Logger(level='DEBUG', colored_log=True) @@ -16,5 +16,6 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # placeholder - print('Do nothing right now!') + # Call StatAnalysis + anl = StatAnalysis(config) + anl.initialize() diff --git a/scripts/exglobal_stat_analysis.py b/scripts/exglobal_stat_analysis.py deleted file mode 100644 index 7ae192b89b1..00000000000 --- a/scripts/exglobal_stat_analysis.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# exglobal_stat_analysis.py -# This script creates an StatAnalysis class -# and runs the initialize, execute and finalize methods -# for a global stat analysis -import os - -from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.stat_analysis import StatAnalysis - -# Initialize root logger -logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) - - -if __name__ == '__main__': - - # Take configuration from environment and cast it as python dictionary - config = cast_strdict_as_dtypedict(os.environ) - - # Instantiate the snow analysis task - anl = StatAnalysis(config) - anl.initialize() \ No newline at end of file From b621034da0eddd2d8cd27b82dc74f080de4677e9 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 5 Sep 2024 17:02:30 +0000 Subject: [PATCH 10/17] pycodestyle --- ush/python/pygfs/task/stat_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 8a452bfa573..4b8a072d6ba 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -20,6 +20,7 @@ logger = getLogger(__name__.split('.')[-1]) + class StatAnalysis(Analysis): """ Class for global stat analysis tasks @@ -57,5 +58,3 @@ def initialize(self: Analysis) -> None: dest = os.path.join(self.task_config.DATA, "stats") statlist = [aerostat, dest] FileHandler({'copy': statlist}).sync() - - \ No newline at end of file From 243e1e0d009eb1a4a268d6efff7c3d87a75a46a0 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 9 Sep 2024 16:36:59 +0000 Subject: [PATCH 11/17] add anlstat config file --- parm/config/gfs/config.analstat | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 parm/config/gfs/config.analstat diff --git a/parm/config/gfs/config.analstat b/parm/config/gfs/config.analstat new file mode 100644 index 00000000000..ba3a4596a13 --- /dev/null +++ b/parm/config/gfs/config.analstat @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.anlstat ########## +# Analysis Stat + +echo "BEGIN: config.anlstat" + +# Get task specific resources +source "${EXPDIR}/config.resources" anlstat + +echo "END: config.anlstat" \ No newline at end of file From 587a7c53c89f6ba2d2123445f193402c34034d14 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 9 Sep 2024 16:47:36 +0000 Subject: [PATCH 12/17] add global stats job --- jobs/JGLOBAL_ANALYSIS_STATS | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index da0193ea82d..cfdb809f880 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -1,6 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}statanl_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## From 4d0b219eeeeebe731b520037e489026ab8307be3 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 9 Sep 2024 19:15:46 +0000 Subject: [PATCH 13/17] add symlink from gdasapp to parm file --- sorc/link_workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index ae30e7a6451..978c2331a1c 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -252,7 +252,7 @@ fi #------------------------------ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/parm/gdas" || exit 1 - declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms") + declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "stats" "jcb-gdas" "jcb-algorithms") for comp in "${gdasapp_comps[@]}"; do [[ -d "${comp}" ]] && rm -rf "${comp}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" . From c7e5f05fb826ffbf93eca8f35696cb44479e2d26 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 10 Sep 2024 17:23:54 +0000 Subject: [PATCH 14/17] revert sorc symlink addition due to changes in GDASApp --- sorc/link_workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 978c2331a1c..ae30e7a6451 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -252,7 +252,7 @@ fi #------------------------------ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/parm/gdas" || exit 1 - declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "stats" "jcb-gdas" "jcb-algorithms") + declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms") for comp in "${gdasapp_comps[@]}"; do [[ -d "${comp}" ]] && rm -rf "${comp}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" . From 668c39d99e8dd1bc4ce1f44499dc03e647203aae Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 10 Sep 2024 17:26:56 +0000 Subject: [PATCH 15/17] remove extra config file --- parm/config/gfs/config.analstat | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 parm/config/gfs/config.analstat diff --git a/parm/config/gfs/config.analstat b/parm/config/gfs/config.analstat deleted file mode 100644 index ba3a4596a13..00000000000 --- a/parm/config/gfs/config.analstat +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.anlstat ########## -# Analysis Stat - -echo "BEGIN: config.anlstat" - -# Get task specific resources -source "${EXPDIR}/config.resources" anlstat - -echo "END: config.anlstat" \ No newline at end of file From 7893aa18e89f1695fbb0194ba0775246f5873f73 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 10 Sep 2024 17:28:57 +0000 Subject: [PATCH 16/17] remove DATA line --- jobs/JGLOBAL_ANALYSIS_STATS | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index cfdb809f880..da0193ea82d 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -1,7 +1,6 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -export DATA=${DATA:-${DATAROOT}/${RUN}statanl_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## From 9cd2fcdd864747c1fd42b3c851de32dca76981bb Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 17:44:44 +0000 Subject: [PATCH 17/17] Revert "remove DATA line" This reverts commit 7893aa18e89f1695fbb0194ba0775246f5873f73. --- jobs/JGLOBAL_ANALYSIS_STATS | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index da0193ea82d..cfdb809f880 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -1,6 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}statanl_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ##############################################