From fbe784df6f7c7ed062521ae44e681d47adac6750 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:18:53 +0000 Subject: [PATCH 001/142] Stubs for anlstat job --- jobs/JGLOBAL_ANALYSIS_STATS | 0 jobs/rocoto/anlstat.sh | 0 parm/config/gfs/config.base | 11 ++++++-- scripts/exglobal_analysis_stats.py | 0 workflow/applications/applications.py | 1 + workflow/applications/gfs_cycled.py | 9 ++++++ workflow/rocoto/gfs_tasks.py | 40 +++++++++++++++++++++++++++ 7 files changed, 58 insertions(+), 3 deletions(-) create mode 100755 jobs/JGLOBAL_ANALYSIS_STATS create mode 100755 jobs/rocoto/anlstat.sh create mode 100755 scripts/exglobal_analysis_stats.py diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS new file mode 100755 index 00000000000..e69de29bb2d diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh new file mode 100755 index 00000000000..e69de29bb2d diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index e6a626cfe37..05a3fb76330 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -73,9 +73,6 @@ export DO_NPOESS="@DO_NPOESS@" # NPOESS products export DO_TRACKER="@DO_TRACKER@" # Hurricane track verification export DO_GENESIS="@DO_GENESIS@" # Cyclone genesis verification export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU) -export DO_VERFOZN="YES" # Ozone data assimilation monitoring -export DO_VERFRAD="YES" # Radiance data assimilation monitoring -export DO_VMINMON="YES" # GSI minimization monitoring export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel @@ -474,6 +471,14 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring + export DO_ANLSTAT="YES" # JEDI-based analysis statistics +else + export DO_VERFOZN="YES" # Ozone data assimilation monitoring + export DO_VERFRAD="YES" # Radiance data assimilation monitoring + export DO_VMINMON="YES" # GSI minimization monitoring + if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then + export DO_ANLSTAT="YES" # JEDI-based analysis statistics + fi fi # If starting ICs that are not at cycle hour diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py new file mode 100755 index 00000000000..e69de29bb2d diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 8c1f69735ef..9f8283eae3b 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -63,6 +63,7 @@ def __init__(self, conf: Configuration) -> None: self.do_verfozn = _base.get('DO_VERFOZN', True) self.do_verfrad = _base.get('DO_VERFRAD', True) self.do_vminmon = _base.get('DO_VMINMON', True) + self.do_anlstat = _base.get('DO_ANLSTAT', False) self.do_tracker = _base.get('DO_TRACKER', True) self.do_genesis = _base.get('DO_GENESIS', True) self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e049a7d422f..825925cb06c 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -74,6 +74,9 @@ def _get_app_configs(self): if self.do_vminmon: configs += ['vminmon'] + if self.do_anlstat: + configs += ['anlstat'] + if self.do_tracker: configs += ['tracker'] @@ -206,6 +209,9 @@ def get_task_names(self): if self.do_vminmon: gdas_tasks += ['vminmon'] + if self.do_anlstat: + gdas_tasks += ['anlstat'] + if self.do_gempak: gdas_tasks += ['gempak', 'gempakmetancdc'] @@ -239,6 +245,9 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] + + if self.do_anlstat: + gfs_tasks += ['anlstat'] if self.do_tracker: gfs_tasks += ['tracker'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9d9b28fb175..2fabc870897 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1731,6 +1731,40 @@ def vminmon(self): return task + def anlstat(self): + deps = [] + if self.app_config.do_jediatmvar: + dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jediocnvar: + dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jedisnowda: + dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_aero: + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('anlstat') + task_name = f'{self.run}anlstat' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/anlstat.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def tracker(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}atmos_prod'} @@ -2225,6 +2259,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2240,6 +2277,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}tracker'} deps.append(rocoto.add_dependency(dep_dict)) From cce943c474708b301e5deaa87dfa3dd172a6c22e Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:22:27 +0000 Subject: [PATCH 002/142] update for pynorms --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 825925cb06c..881694bcfbc 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -245,7 +245,7 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] - + if self.do_anlstat: gfs_tasks += ['anlstat'] From 3f7a4ae14752ccdb331799c051c39dc8eda51504 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:57:40 +0000 Subject: [PATCH 003/142] add in stub scripts --- jobs/JGLOBAL_ANALYSIS_STATS | 35 ++++++++++++++++++++++++++++++ jobs/rocoto/anlstat.sh | 18 +++++++++++++++ scripts/exglobal_analysis_stats.py | 20 +++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index e69de29bb2d..6543735fb19 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + + +############################################################### +# Run relevant script + +EXSCRIPT=${ANLSTATSPY:-${SCRgfs}/exglobal_analysis_stats.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 \ No newline at end of file diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh index e69de29bb2d..52292a582c2 100755 --- a/jobs/rocoto/anlstat.sh +++ b/jobs/rocoto/anlstat.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="anlstat" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS" +status=$? +exit "${status}" \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index e69de29bb2d..045fc914b3e 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# exglobal_analysis_stats.py +# This script will run the OOPS/JEDI code to + +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atm_analysis import AtmAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # placeholder + print('Do nothing right now!') \ No newline at end of file From 0ac03dcf91bf38a1df6e02f813a184af62015e55 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:59:46 +0000 Subject: [PATCH 004/142] EOLs --- jobs/JGLOBAL_ANALYSIS_STATS | 2 +- jobs/rocoto/anlstat.sh | 2 +- scripts/exglobal_analysis_stats.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 6543735fb19..da0193ea82d 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -32,4 +32,4 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi -exit 0 \ No newline at end of file +exit 0 diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh index 52292a582c2..ac7d8af16a2 100755 --- a/jobs/rocoto/anlstat.sh +++ b/jobs/rocoto/anlstat.sh @@ -15,4 +15,4 @@ export jobid="${job}.$$" # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS" status=$? -exit "${status}" \ No newline at end of file +exit "${status}" diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 045fc914b3e..e96a50e7b87 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -17,4 +17,4 @@ config = cast_strdict_as_dtypedict(os.environ) # placeholder - print('Do nothing right now!') \ No newline at end of file + print('Do nothing right now!') From f6933900a62a9168b65936d335d893d5ca3ebc04 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 20 Aug 2024 18:04:56 +0000 Subject: [PATCH 005/142] typos --- workflow/rocoto/gfs_tasks.py | 6 +++--- workflow/rocoto/tasks.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 2fabc870897..4d2f3704353 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1736,13 +1736,13 @@ def anlstat(self): if self.app_config.do_jediatmvar: dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jediocnvar: + if self.app_config.do_jediocnvar: dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jedisnowda: + if self.app_config.do_jedisnowda: dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - if self.do_aero: + if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 64952498d45..3c8e1c8933c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,6 +22,7 @@ class Tasks: 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', + 'anlstat', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', From 20fb88c057b22991ed3b889c7388a271cc5db99d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 28 Aug 2024 17:01:06 +0000 Subject: [PATCH 006/142] replace separate JEDI radiance bias correction files with tarball (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 ++ parm/stage/analysis.yaml.j2 | 9 +---- ush/python/pygfs/task/analysis.py | 36 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 48 ++++++++++----------------- 4 files changed, 38 insertions(+), 57 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 9d862920657..7a011671bed 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -32,6 +32,8 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% else %} + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index d30389644a3..4068f1e9280 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,17 +10,10 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} - {% if DO_JEDIATMVAR %} - {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} - {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} - - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] - {% endfor %} - {% endfor %} - {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950e..b8e78092635 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -42,8 +42,7 @@ def initialize(self) -> None: FileHandler(obs_dict).sync() # some analyses need to stage bias corrections - bias_dict = self.get_bias_dict() - FileHandler(bias_dict).sync() + self.get_bias() # link jedi executable to run directory self.link_jediexe() @@ -128,25 +127,21 @@ def get_obs_dict(self) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + def get_bias(self) -> None: + """Stage radiance bias correciton files - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- + Task: GDAS task Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ - logger.info(f"Extracting a list of bias correction files from Jedi config file") + logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -156,17 +151,22 @@ def get_bias_dict(self) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.radbcor" + copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + FileHandler(bias_dict).sync() + + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c5..e32dcdf8153 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -207,37 +207,23 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # copy bias correction files to ROTDIR - logger.info("Copy bias correction files from DATA/ to COM/") - biasdir = os.path.join(self.task_config.DATA, 'bc') - biasls = os.listdir(biasdir) - biaslist = [] - for bfile in biasls: - src = os.path.join(biasdir, bfile) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - biaslist.append([src, dest]) - - gprefix = f"{self.task_config.GPREFIX}" - gsuffix = f"{to_YMDH(self.task_config.previous_cycle)}" + ".txt" - aprefix = f"{self.task_config.APREFIX}" - asuffix = f"{to_YMDH(self.task_config.current_cycle)}" + ".txt" - - logger.info(f"Copying {gprefix}*{gsuffix} from DATA/ to COM/ as {aprefix}*{asuffix}") - obsdir = os.path.join(self.task_config.DATA, 'obs') - obsls = os.listdir(obsdir) - for ofile in obsls: - if ofile.endswith(".txt"): - src = os.path.join(obsdir, ofile) - tfile = ofile.replace(gprefix, aprefix) - tfile = tfile.replace(gsuffix, asuffix) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile) - biaslist.append([src, dest]) - - bias_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': biaslist, - } - FileHandler(bias_copy).sync() + # path of output radiance bias correction tarfile + bfile = f"{self.task_config.APREFIX}radbcor" + radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) + + # get lists of radiance bias correction files to put in tarball + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + + # tar radiance bias correction files to ROTDIR + logger.info(f"Creating radiance bias correction tar file {radtar}") + with tarfile.open(radtar, 'w') as radbcor: + for satfile in satlist: + radbcor.add(satfile, arcname=os.path.basename(satfile)) + for tlapfile in tlaplist: + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) + logger.info(f"Add {radbcor.getnames()}") + radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From 73aa233f743ccd9232481e34fd5a3c72ace17a2f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 29 Aug 2024 10:26:14 +0000 Subject: [PATCH 007/142] rename JEDI radiance bias correction tarball file to be more self-describing (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/stage/analysis.yaml.j2 | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 7a011671bed..fc5ce9478df 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -33,7 +33,7 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index 4068f1e9280..2d4b7d28545 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,7 +10,7 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "rad_varbc_params.tar"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index b8e78092635..ee674f8ef0c 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -152,7 +152,7 @@ def get_bias(self) -> None: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.radbcor" + bfile = f"{prefix}.rad_varbc_params.tar" copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) break diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e32dcdf8153..99c6e600cb5 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -208,7 +208,7 @@ def finalize(self: Analysis) -> None: FileHandler(yaml_copy).sync() # path of output radiance bias correction tarfile - bfile = f"{self.task_config.APREFIX}radbcor" + bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) # get lists of radiance bias correction files to put in tarball From 1203866f068eea2a9ccd28ad9d65b4c6c7d62311 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 7 Sep 2024 11:36:35 +0000 Subject: [PATCH 008/142] update gdas.cd hash to include recent commits to GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18f..554c55a6ad1 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 From 5985e9f718ded5316007f6d38ac16b5abb2c9649 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 9 Sep 2024 14:22:13 +0000 Subject: [PATCH 009/142] update pygfs jedi class and atmospheric tasks to extract JEDI radiance bias correction files from tarball (#2862) --- ush/python/pygfs/jedi/jedi.py | 9 +++-- ush/python/pygfs/task/analysis.py | 45 ------------------------ ush/python/pygfs/task/atm_analysis.py | 16 +++++++++ ush/python/pygfs/task/atmens_analysis.py | 16 +++++++++ 4 files changed, 36 insertions(+), 50 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 62dcb517cad..dace86c4feb 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -216,11 +216,10 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.rad_varbc_params.tar" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index d9cd3582190..1d8b38483b0 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -41,9 +41,6 @@ def initialize(self) -> None: obs_dict = self.get_obs_dict() FileHandler(obs_dict).sync() - # some analyses need to stage bias corrections - self.get_bias() - # link jedi executable to run directory self.link_jediexe() @@ -126,48 +123,6 @@ def get_obs_dict(self) -> Dict[str, Any]: } return obs_dict - @logit(logger) - def get_bias(self) -> None: - """Stage radiance bias correciton files - - This method stages radiance bias correction files in the obs sub-diretory of the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - - logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") - observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - break - - bias_dict = { - 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], - 'copy': copylist - } - FileHandler(bias_dict).sync() - - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() - @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: """Add cubed-sphere increments to cubed-sphere backgrounds diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ad082063bb8..36f768fa130 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -143,6 +144,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 55e72702b1f..7004bf059c5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -142,6 +143,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) From 73e56a7fa14c553f0d64003b88c8e9118f088bea Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 10:50:33 +0000 Subject: [PATCH 010/142] move radiance bias correction staging to jedi class (#2868) --- ush/python/pygfs/jedi/jedi.py | 29 ++++++++++++++++-------- ush/python/pygfs/task/atm_analysis.py | 20 +--------------- ush/python/pygfs/task/atmens_analysis.py | 20 +--------------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dace86c4feb..4e527460ab1 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 import os +import tarfile from logging import getLogger +from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, + FileHandler, chdir, rm_p, parse_j2yaml, logit, @@ -188,14 +191,10 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- @@ -204,8 +203,7 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -218,14 +216,25 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + radtar = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + + # stage bias corrections + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract radiance bias correction files from tarball + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 36f768fa130..c1a6558adef 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -140,24 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7004bf059c5..1f3a0651332 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -139,24 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 666d7d6d89c8c17ef9627367a3efdb76454d07f7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 15:03:43 +0000 Subject: [PATCH 011/142] initial attempt to generalize processing of variational bias correction files using jedi class (#2862) --- ush/python/pygfs/jedi/jedi.py | 6 +++--- ush/python/pygfs/task/atm_analysis.py | 6 ++++-- ush/python/pygfs/task/atmens_analysis.py | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 4e527460ab1..2b899df4e38 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -191,7 +191,7 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy This method stages radiance bias correction files in the obs sub-diretory of the run directory @@ -215,9 +215,9 @@ def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" + bfile = f"{prefix}.{bias_file}" radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) + copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) break bias_dict = { diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c1a6558adef..454d557e879 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -139,7 +140,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1f3a0651332..01fc4faac04 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -138,7 +139,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 1e4a28434dc475cf533250219383bf20c347977e Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 17:09:42 +0000 Subject: [PATCH 012/142] move VarBcDir inside initialize_analysis method (#2862) --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 454d557e879..2cc738008af 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -140,6 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 01fc4faac04..b57c80e22cc 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -139,6 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) From b48e126341795a11ad3941e0beb56c95644fac99 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 11 Sep 2024 18:29:24 +0000 Subject: [PATCH 013/142] update sorc/gdas.cd hash to bring in radiance tarball ctest changes (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 554c55a6ad1..032b708f6a4 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 +Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 From ba77d4ec1954968de8e88ebc002073ff85ab6953 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:05:43 +0000 Subject: [PATCH 014/142] add extract_tar method to jedi class, use extract_tar in atm and atmens analysis scripts (#2862) --- ush/python/pygfs/jedi/jedi.py | 51 +++++++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 8 +++- ush/python/pygfs/task/atmens_analysis.py | 8 +++- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 2b899df4e38..b5fdb42c7c4 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -3,7 +3,6 @@ import os import tarfile from logging import getLogger -from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, @@ -191,19 +190,24 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: + def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method stages radiance bias correction files in the obs sub-diretory of the run directory + This method extracts 'observers' from the JEDI yaml and determines from that list + if bias correction tar files are to be copied to the run directory + from the component directory. Parameters ---------- task_config: AttrDict Attribute-dictionary of all configuration variables associated with a GDAS task. + bias_file + name of bias correction tar file Returns ---------- - None + bias_dict: Dict + a dictionary containing the list of observation bias files to copy for FileHandler """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -216,8 +220,8 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.{bias_file}" - radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) + tar_file = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.VarBcDir, bfile), tar_file]) break bias_dict = { @@ -225,16 +229,33 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: 'copy': copylist } - # stage bias corrections - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + return bias_dict + + @logit(logger) + def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: + """Extract files from list of tarfiles + + This method extract bias correction files from tarball(s) + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + tar_dict + a dictionary containing the list of tar files + + Returns + ---------- + None + """ - # extract radiance bias correction files from tarball - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + # extract bias correction files from tar file + for tar_file in tar_dict['copy']: + if ".tar" in tar_file[1]: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + tarball.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2cc738008af..900e78ba1ce 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -141,7 +141,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b57c80e22cc..a02932d8f53 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From bc3a6ddab05faaea4466719e96bc36133901a199 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:15:21 +0000 Subject: [PATCH 015/142] correct typo in atmens_analysis.py (2862) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a02932d8f53..8bab6314950 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,7 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - bias_dict = self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 92626a742c99f43e29a451b0f8ba16b02833a91c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 18:52:57 +0000 Subject: [PATCH 016/142] add anlstat config file and update config.resources --- parm/config/gfs/config.anlstat | 11 +++++++++++ parm/config/gfs/config.resources | 10 +++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 parm/config/gfs/config.anlstat diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat new file mode 100644 index 00000000000..d9ce48dc0c8 --- /dev/null +++ b/parm/config/gfs/config.anlstat @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.anlstat ########## +# Analysis Stat + +echo "BEGIN: config.anlstat" + +# Get task specific resources +source "${EXPDIR}/config.resources" anlstat + +echo "END: config.anlstat" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index b50e1c5fbb0..1ae71beb38e 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -17,7 +17,7 @@ if (( $# != 1 )); then echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" - echo "anal sfcanl analcalc analdiag fcst echgres" + echo "anal sfcanl analcalc analdiag anlstat fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" echo "verfozn verfrad vminmon fit2obs metp arch cleanup" @@ -707,6 +707,14 @@ case ${step} in memory="48GB" ;; + "anlstat") + walltime="00:30:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="24GB" + ;; + "sfcanl") walltime="00:20:00" ntasks=${ntiles:-6} From 5a881a78ba285ab298b9b14b342c9c1afad1eb75 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 19:01:59 +0000 Subject: [PATCH 017/142] Add python files that use new JEDI format --- scripts/exglobal_stat_analysis.py | 27 ++++++ ush/python/pygfs/task/stat_analysis.py | 122 +++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 scripts/exglobal_stat_analysis.py create mode 100644 ush/python/pygfs/task/stat_analysis.py diff --git a/scripts/exglobal_stat_analysis.py b/scripts/exglobal_stat_analysis.py new file mode 100644 index 00000000000..94f70cab761 --- /dev/null +++ b/scripts/exglobal_stat_analysis.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exglobal_stat_analysis.py +# This script creates an StatAnalysis class +# and runs the initialize method +# which create and stage the runtime directory +# and create the YAML configuration +# for a global stat analysis +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.stat_analysis import StatAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atm analysis task + StatAnl = StatAnalysis(config, 'statanlvar') + + # Initialize JEDI variational analysis + StatAnl.initialize_jedi() + StatAnl.initialize_analysis() \ No newline at end of file diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py new file mode 100644 index 00000000000..0599e245a4a --- /dev/null +++ b/ush/python/pygfs/task/stat_analysis.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import os +import glob +import gzip +import tarfile +from logging import getLogger +from pprint import pformat +from typing import Optional, Dict, Any + +from wxflow import (AttrDict, + FileHandler, + add_to_datetime, to_fv3time, to_timedelta, to_YMDH, + Task, + parse_j2yaml, save_as_yaml, + logit) +from pygfs.jedi import Jedi + +logger = getLogger(__name__.split('.')[-1]) + + +class StatAnalysis(Task): + """ + Class for JEDI-based global stat analysis tasks + """ + @logit(logger, name="StatAnalysis") + def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + """ + Constructor global stat analysis task + + This method will construct a global stat analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute object + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + yaml_name: str, optional + name of YAML file for JEDI configuration + + Returns + ---------- + None + """ + super().__init__(config) + + _res = int(self.task_config.CASE[1:]) + _res_anl = int(self.task_config.CASE_ANL[1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'jedi_yaml': _letkfoi_yaml + } + ) + + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) + + # Create JEDI object + self.jedi = Jedi(self.task_config, yaml_name) + + @logit(logger) + def initialize_jedi(self): + """Initialize JEDI application + + This method will initialize a JEDI application used in the global stat analysis. + This includes: + - generating and saving JEDI YAML config + - linking the JEDI executable + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + # get JEDI-to-FV3 increment converter config and save to YAML file + logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") + + # save JEDI config to YAML file + logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) + + # link JEDI executable + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") + self.jedi.link_exe(self.task_config) + + @logit(logger) + def initialize_analysis(self) -> None: + """ + Initialize a global stat analysis + + This method will initialize a global stat analysis. + This includes: + - copying stat files + + Parameters + ---------- + None + + Returns + ---------- + None + """ + super().initialize() + + logger.info(f"Copying files to {self.task_config.DATA}/stats") + + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + dest = os.path.join(self.task_config.DATA, "stats") + statlist = [aerostat, dest] + FileHandler({'copy': statlist}).sync() From d57507254d2ceaced584e2c1624765fe09c8b892 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 19:07:24 +0000 Subject: [PATCH 018/142] pycodestyle --- scripts/exglobal_stat_analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_stat_analysis.py b/scripts/exglobal_stat_analysis.py index 94f70cab761..78c035c7657 100644 --- a/scripts/exglobal_stat_analysis.py +++ b/scripts/exglobal_stat_analysis.py @@ -24,4 +24,5 @@ # Initialize JEDI variational analysis StatAnl.initialize_jedi() - StatAnl.initialize_analysis() \ No newline at end of file + StatAnl.initialize_analysis() + From 1635908083bd73f11288e838b203c9a5242dc94f Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 19:21:29 +0000 Subject: [PATCH 019/142] update appropriate py file --- scripts/exglobal_analysis_stats.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index e96a50e7b87..94f70cab761 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 -# exglobal_analysis_stats.py -# This script will run the OOPS/JEDI code to - +# exglobal_stat_analysis.py +# This script creates an StatAnalysis class +# and runs the initialize method +# which create and stage the runtime directory +# and create the YAML configuration +# for a global stat analysis import os from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.atm_analysis import AtmAnalysis +from pygfs.task.stat_analysis import StatAnalysis # Initialize root logger logger = Logger(level='DEBUG', colored_log=True) @@ -16,5 +19,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # placeholder - print('Do nothing right now!') + # Instantiate the atm analysis task + StatAnl = StatAnalysis(config, 'statanlvar') + + # Initialize JEDI variational analysis + StatAnl.initialize_jedi() + StatAnl.initialize_analysis() \ No newline at end of file From dee996ee98c5631d0efaa2841af7d67c32b394b9 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Sep 2024 19:23:30 +0000 Subject: [PATCH 020/142] pycodestyle --- scripts/exglobal_analysis_stats.py | 2 +- scripts/exglobal_stat_analysis.py | 28 ---------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 scripts/exglobal_stat_analysis.py diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 94f70cab761..17fb7724a3d 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -24,4 +24,4 @@ # Initialize JEDI variational analysis StatAnl.initialize_jedi() - StatAnl.initialize_analysis() \ No newline at end of file + StatAnl.initialize_analysis() diff --git a/scripts/exglobal_stat_analysis.py b/scripts/exglobal_stat_analysis.py deleted file mode 100644 index 78c035c7657..00000000000 --- a/scripts/exglobal_stat_analysis.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# exglobal_stat_analysis.py -# This script creates an StatAnalysis class -# and runs the initialize method -# which create and stage the runtime directory -# and create the YAML configuration -# for a global stat analysis -import os - -from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.stat_analysis import StatAnalysis - -# Initialize root logger -logger = Logger(level='DEBUG', colored_log=True) - - -if __name__ == '__main__': - - # Take configuration from environment and cast it as python dictionary - config = cast_strdict_as_dtypedict(os.environ) - - # Instantiate the atm analysis task - StatAnl = StatAnalysis(config, 'statanlvar') - - # Initialize JEDI variational analysis - StatAnl.initialize_jedi() - StatAnl.initialize_analysis() - From 2d7d6e4d1ed05bf1dbdaf202505c29d4f4909817 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Sep 2024 17:37:21 +0000 Subject: [PATCH 021/142] add line to include anlstat --- workflow/applications/applications.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a694129e386..267628bb643 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -55,6 +55,7 @@ def __init__(self, conf: Configuration) -> None: self.do_verfozn = base.get('DO_VERFOZN', True) self.do_verfrad = base.get('DO_VERFRAD', True) self.do_vminmon = base.get('DO_VMINMON', True) + self.do_anlstat = base.get('DO_ANLSTAT', True) self.do_tracker = base.get('DO_TRACKER', True) self.do_genesis = base.get('DO_GENESIS', True) self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) From 0ab7bdd6083051f5cb9b9bc687d2a4bbbe5e2e34 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:20:47 +0000 Subject: [PATCH 022/142] improve error handling for jedi class tarfile.extractall (#2862) --- ush/python/pygfs/jedi/jedi.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b5fdb42c7c4..08cfbc02af7 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -252,10 +252,19 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: if ".tar" in tar_file[1]: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - tarball.close() + try: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) From 8b627a159fdef4bc67e5877b8d75f9a696699675 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:23:52 +0000 Subject: [PATCH 023/142] remove unnecessary close following tarfile add (#2862) --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 900e78ba1ce..27243159fa7 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -287,7 +287,6 @@ def finalize(self) -> None: for tlapfile in tlaplist: radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) logger.info(f"Add {radbcor.getnames()}") - radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From eb4fc374b83b693ac767587452a8d63aca77675f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:45:14 +0000 Subject: [PATCH 024/142] use endswith to clean up scripting (#2862) --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 08cfbc02af7..61ad8203026 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -251,7 +251,7 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: - if ".tar" in tar_file[1]: + if tar_file[1].endswith('.tar'): try: with tarfile.open(tar_file[1], "r") as tarball: tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) From ed52b724ef8762f9c1eea2c347a5276f49c9469b Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 17:13:24 +0000 Subject: [PATCH 025/142] refactor jedi class method extract_tar (#2862) --- ush/python/pygfs/jedi/jedi.py | 43 +++++++++++------------- ush/python/pygfs/task/atm_analysis.py | 5 +-- ush/python/pygfs/task/atmens_analysis.py | 5 +-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 61ad8203026..9e69065bb22 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -232,39 +232,36 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict @logit(logger) - def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: - """Extract files from list of tarfiles + def extract_tar(self, tar_file: str) -> None: + """Extract bias correction files from a tarball - This method extract bias correction files from tarball(s) + This method extract files from a tarball Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. - tar_dict - a dictionary containing the list of tar files + tar_file + path/name of tarball Returns ---------- None """ - # extract bias correction files from tar file - for tar_file in tar_dict['copy']: - if tar_file[1].endswith('.tar'): - try: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") - else: - logger.info() - except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 27243159fa7..badb01a74aa 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -146,8 +146,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 8bab6314950..4b2f8ebbf40 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -145,8 +145,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 80305b67cfbf941e40604d71b7f1413106b2d807 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:04:24 +0000 Subject: [PATCH 026/142] update sorc/gdas.cd to current head of GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 032b708f6a4..55e895f1dcf 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 +Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 From 821ee760945bbd38b5a4c6c7db28d9fa08de7fb2 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:21:10 +0000 Subject: [PATCH 027/142] make jedi method extract_tar static, clean up comments, correct typo (#2862) --- ush/python/pygfs/jedi/jedi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 9e69065bb22..415a0a3c081 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -231,9 +231,10 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict + @staticmethod @logit(logger) - def extract_tar(self, tar_file: str) -> None: - """Extract bias correction files from a tarball + def extract_tar(tar_file: str) -> None: + """Extract files from a tarball This method extract files from a tarball @@ -254,14 +255,14 @@ def extract_tar(self, tar_file: str) -> None: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + if tarfile.is_tarfile(tar_file): + logger.error(f"FATAL ERROR: {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") else: logger.info() except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") @logit(logger) From 042ec5094cd62dff9d5abc1e4770dd2451284d0c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 19 Sep 2024 18:17:17 +0000 Subject: [PATCH 028/142] correctly prefix tlapse radiance bias correction files for cycling (#2862) --- ush/python/pygfs/task/atm_analysis.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index badb01a74aa..5f67ea9d728 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -276,9 +276,21 @@ def finalize(self) -> None: bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - # get lists of radiance bias correction files to put in tarball + # rename and copy tlapse radiance bias correction files from obs to bc + tlapobs = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + copylist = [] + for tlapfile in tlapobs: + obsfile = os.path.basename(tlapfile).split('.', 2) + newfile = f"{self.task_config.APREFIX}{obsfile[2]}" + copylist.append([tlapfile, os.path.join(self.task_config.DATA, 'bc', newfile)]) + tlapse_dict = { + 'copy': copylist + } + FileHandler(tlapse_dict).sync() + + # get lists of radiance bias correction files to add to tarball satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*tlapse.txt')) # tar radiance bias correction files to ROTDIR logger.info(f"Creating radiance bias correction tar file {radtar}") From 5306b069727777d814af4433c9b57d46c47f2269 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 20 Sep 2024 10:01:29 +0000 Subject: [PATCH 029/142] extend duration of C96C48_ufs_hybatmDA CI by one cycle to 2024022406 (#2862) --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 0b5aa7b6acc..b1566d77a0e 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -11,7 +11,7 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2024022318 - edate: 2024022400 + edate: 2024022406 nens: 2 gfs_cyc: 1 start: warm From 3a812884ab7b65fee4c966700088cab82778db0c Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 27 Sep 2024 12:54:34 +0000 Subject: [PATCH 030/142] Initial commit --- ush/python/pygfs/task/atm_analysis.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8d340a5b731..62de16ffbe3 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -129,8 +129,24 @@ def initialize_analysis(self) -> None: ---------- None """ - super().initialize() + # stage observations + logger.info(f"Staging list of observation files generated from JEDI config") + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) + jcb_config['algorithm'] = 'atm_obs_staging' + obs_dict = render(jcb_config) + FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") + + # Test + jcb_config = {} + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) + jcb_config['algorithm'] = 'atm_bias_staging' + bias_dict = render(jcb_config) + logger.debug(f"foo:\n{pformat(bias_dict)}") + # stage observations logger.info(f"Staging list of observation files generated from JEDI config") obs_dict = self.jedi.get_obs_dict(self.task_config) From 43c8075b39e9134c0567dd7a4c06c9879a0010a1 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 27 Sep 2024 13:10:12 +0000 Subject: [PATCH 031/142] correct ORION.env typo, adjust JEDI ORION job configurations (#2862) --- env/ORION.env | 2 +- parm/config/gfs/config.resources.ORION | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index 1bc7eb60d4b..3b8053d060f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -142,7 +142,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_MARINEANLCHKPT="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalecen" ]]; then diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION index d761df7b73d..461b6f14f7c 100644 --- a/parm/config/gfs/config.resources.ORION +++ b/parm/config/gfs/config.resources.ORION @@ -23,6 +23,16 @@ case ${step} in # Remove this block once the GSI issue is resolved. export walltime="00:45:00" ;; + "atmanlvar") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; + "atmensanlobs") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; *) ;; esac From 4f0446a08488a48bd1f82ae04b0c7e42331ae20b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 27 Sep 2024 18:01:00 +0000 Subject: [PATCH 032/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dcf..d39bf615703 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit d39bf61570394730e17cd6508307ff7a624cd3cd From eb476e3a8c1d9e6282343b5afd9eba92073efe4c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Fri, 4 Oct 2024 14:27:04 +0000 Subject: [PATCH 033/142] add most updated scripts for debugging --- parm/config/gfs/config.anlstat | 2 ++ sorc/link_workflow.sh | 1 + ush/python/pygfs/task/stat_analysis.py | 20 ++++++++++++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index d9ce48dc0c8..9ad797513d3 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -8,4 +8,6 @@ echo "BEGIN: config.anlstat" # Get task specific resources source "${EXPDIR}/config.resources" anlstat +export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x + echo "END: config.anlstat" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 270a8bb1c92..d4265b4f207 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -374,6 +374,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_socahybridweights.x" \ "gdasapp_land_ensrecenter.x" \ "bufr2ioda.x" \ + "ioda-stats.x" \ "calcfIMS.exe" \ "apply_incr.exe" ) for gdasexe in "${JEDI_EXE[@]}"; do diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 0599e245a4a..d57cadfadd6 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -47,14 +47,25 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): super().__init__(config) _res = int(self.task_config.CASE[1:]) - _res_anl = int(self.task_config.CASE_ANL[1:]) + # _res_anl = int(self.task_config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + print(_window_begin) # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + #'npx_anl': _res_anl + 1, + #'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config.LEVS - 1, + 'ATM_WINDOW_BEGIN': _window_begin, + 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'jedi_yaml': _letkfoi_yaml + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." } ) @@ -116,7 +127,8 @@ def initialize_analysis(self) -> None: logger.info(f"Copying files to {self.task_config.DATA}/stats") - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS_TMPL, f"{self.task_config['APREFIX']}aerostat") dest = os.path.join(self.task_config.DATA, "stats") - statlist = [aerostat, dest] + statlist = [[aerostat, dest]] + #print(${MEMDIR}) FileHandler({'copy': statlist}).sync() From 7c30e482e1fe1d939ec53c1bec5e203739298466 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 4 Oct 2024 16:31:45 +0000 Subject: [PATCH 034/142] Update --- .../exglobal_atm_analysis_fv3_increment.py | 3 +- scripts/exglobal_atm_analysis_initialize.py | 4 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_initialize.py | 4 +- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 203 +++++++----------- ush/python/pygfs/task/atm_analysis.py | 96 ++------- ush/python/pygfs/task/atmens_analysis.py | 76 ++----- 9 files changed, 118 insertions(+), 274 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 72413ddbd4d..f1422cca89a 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,5 +21,4 @@ AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') # Initialize and execute FV3 increment converter - AtmAnl.initialize_jedi() - AtmAnl.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 9deae07bb37..d9af2712353 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -23,5 +23,5 @@ AtmAnl = AtmAnalysis(config, 'atmanlvar') # Initialize JEDI variational analysis - AtmAnl.initialize_jedi() - AtmAnl.initialize_analysis() + AtmAnl.jedi.initialize(AtmAnl.task_config) + AtmAnl.initialize() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 83595320695..cba7a33a5d0 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config, 'atmanlvar') # Execute JEDI variational analysis - AtmAnl.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) + AtmAnl.jedi.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 326fe80628c..26bb9a6dab0 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -26,5 +26,5 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis - AtmEnsAnl.initialize_jedi() - AtmEnsAnl.initialize_analysis() + AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) + AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index c701f8cb4ea..ac3271272e7 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index be78e694b1f..d93c42ddef4 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,5 +21,5 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.initialize_jedi() + AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 415a0a3c081..366f0f74704 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -4,14 +4,12 @@ import tarfile from logging import getLogger from typing import List, Dict, Any, Optional +from pprint import pformat from jcb import render -from wxflow import (AttrDict, - FileHandler, +from wxflow import (AttrDict, FileHandler, Task, Executable, chdir, rm_p, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit, - Task, - Executable, WorkflowException) logger = getLogger(__name__.split('.')[-1]) @@ -22,7 +20,7 @@ class Jedi: Class for initializing and executing JEDI applications """ @logit(logger, name="Jedi") - def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: + def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: """Constructor for JEDI objects This method will construct a Jedi object. @@ -44,69 +42,51 @@ def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> No None """ - # For provenance, save incoming task_config as a private attribute of JEDI object - self._task_config = task_config + _exe_name = os.path.basename(JEDIEXE) - _exe_name = os.path.basename(task_config.JEDIEXE) - - self.exe = os.path.join(task_config.DATA, _exe_name) + self.exe_src = JEDIEXE + self.rundir = DATA + self.exe = os.path.join(DATA, _exe_name) if yaml_name: - self.yaml = os.path.join(task_config.DATA, yaml_name + '.yaml') + self.yaml = os.path.join(DATA, yaml_name + '.yaml') else: - self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + self.yaml = os.path.join(DATA, os.path.splitext(_exe_name)[0] + '.yaml') + + # Initialize empty JEDI input config attribute-dictionary self.config = AttrDict() - self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + +# self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') @logit(logger) - def set_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: - """Compile a JEDI configuration dictionary from a template file and save to a YAML file + def initialize(self, task_config: AttrDict) -> None: + """Initialize JEDI application - Parameters - ---------- - task_config : AttrDict - Dictionary of all configuration variables associated with a GDAS task. - algorithm (optional) : str - Name of the algorithm used to generate the JEDI configuration dictionary. - It will override the algorithm set in the task_config.JCB_<>_YAML file. - - Returns - ---------- - None + This method will initialize a JEDI application. + This includes: + - generating JEDI YAML config + - saving JEDI YAML config to run directory + - linking the JEDI executable to run directory """ - if 'JCB_BASE_YAML' in task_config.keys(): - # Step 1: Fill templates of the JCB base YAML file - jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) - - # Step 2: If algorithm is present then override the algorithm in the JEDI - # config. Otherwise, if the algorithm J2-YAML is present, fill - # its templates and merge. - if algorithm: - jcb_config['algorithm'] = algorithm - elif 'JCB_ALGO' in task_config.keys(): - jcb_config['algorithm'] = task_config.JCB_ALGO - elif 'JCB_ALGO_YAML' in task_config.keys(): - jcb_algo_config = parse_j2yaml(task_config.JCB_ALGO_YAML, task_config) - jcb_config.update(jcb_algo_config) - - # Step 3: Generate the JEDI YAML using JCB - self.config = render(jcb_config) - elif 'JEDIYAML' in task_config.keys(): - # Generate JEDI YAML without using JCB - self.config = parse_j2yaml(task_config.JEDIYAML, task_config, - searchpath=self.j2tmpl_dir) - else: - logger.exception(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - raise KeyError(f"FATAL ERROR: Task config must contain JCB_BASE_YAML or JEDIYAML") + # Render JEDI config dictionary + logger.info(f"Generating JEDI YAML config: {self.yaml}") + self.config = self.get_config(task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") + + # Save JEDI config dictionary to YAML in run directory + logger.debug(f"Writing JEDI YAML config to: {self.yaml}") + save_as_yaml(self.config, self.yaml) + # Link JEDI executable to run directory + logger.info(f"Linking JEDI executable {self.exe_src} to {self.exe}") + self.link_exe() + @logit(logger) - def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. aprun_cmd: str String comprising the run command for the JEDI executable. jedi_args (optional): List @@ -118,7 +98,7 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis Attribute-dictionary of JEDI configuration rendered from a template. """ - chdir(task_config.DATA) + chdir(self.rundir) exec_cmd = Executable(aprun_cmd) exec_cmd.add_default_arg(self.exe) @@ -127,6 +107,7 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.yaml) + logger.info(f"Executing {exec_cmd}") try: exec_cmd() except OSError: @@ -134,103 +115,82 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis except Exception: raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") - @staticmethod @logit(logger) - def link_exe(task_config: AttrDict) -> None: - """Link JEDI executable to run directory + def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: + """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + task_config : AttrDict + Dictionary of all configuration variables associated with a GDAS task. + algorithm (optional) : str + Name of the algorithm used to generate the JEDI configuration dictionary. + It will override the algorithm set in the task_config.JCB_ALGO_YAML file. Returns ---------- None """ - # TODO: linking is not permitted per EE2. - # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(task_config.JEDIEXE, exe_dest) + # Fill JCB base YAML template and build JCB config dictionary + jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + + # Add JCB algorithm YAML, if it exists, to JCB config dictionary + if 'JCB_ALGO_YAML' in task_config.keys(): + jcb_config.update(parse_j2yaml(task_config.JCB_ALGO_YAML, task_config)) - @logit(logger) - def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML + if algorithm: + jcb_config['algorithm'] = algorithm - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation files that are to be copied to the run directory - from the observation input directory + # Generate JEDI YAML config by rendering JCB config dictionary + jedi_config = render(jcb_config) + + return jedi_config + + @logit(logger) + def link_exe(self) -> None: + """Link JEDI executable to run directory Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + None Returns ---------- - obs_dict: Dict - a dictionary containing the list of observation files to copy for FileHandler + None """ - observations = find_value_in_nested_dict(self.config, 'observations') - - copylist = [] - for ob in observations['observers']: - obfile = ob['obs space']['obsdatain']['engine']['obsfile'] - basename = os.path.basename(obfile) - copylist.append([os.path.join(task_config.COM_OBS, basename), obfile]) - obs_dict = { - 'mkdir': [os.path.join(task_config.DATA, 'obs')], - 'copy': copylist - } - return obs_dict + # TODO: linking is not permitted per EE2. + # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] + logger.warn("Linking is not permitted per EE2.") + if os.path.exists(self.exe): + rm_p(self.exe) + os.symlink(self.exe_src, self.exe) + @staticmethod @logit(logger) - def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy - - This method extracts 'observers' from the JEDI yaml and determines from that list - if bias correction tar files are to be copied to the run directory - from the component directory. + def remove_redundant(input_list: List) -> List: + """Remove reduncancies from list with possible redundant, non-mutable elements Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. - bias_file - name of bias correction tar file + input_list : List + List with possible redundant, non-mutable elements Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + output_list : List + Input list but with redundancies removed """ - observations = find_value_in_nested_dict(self.config, 'observations') - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.{bias_file}" - tar_file = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.VarBcDir, bfile), tar_file]) - break - - bias_dict = { - 'mkdir': [os.path.join(task_config.DATA, 'bc')], - 'copy': copylist - } - - return bias_dict + output_list = [] + for item in input_list: + if item not in output_list: + output_list.append(item); + return output_list + @staticmethod @logit(logger) def extract_tar(tar_file: str) -> None: @@ -264,7 +224,6 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") - @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 6582db100d6..8a21b1552fe 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -6,8 +6,7 @@ import tarfile from logging import getLogger from pprint import pformat -from typing import Optional, Dict, Any - +from typing import Any, Dict, List, Optional from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, @@ -74,45 +73,15 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = Jedi(self.task_config, yaml_name) - - @logit(logger) - def initialize_jedi(self): - """Initialize JEDI application - - This method will initialize a JEDI application used in the global atm analysis. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) + self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) @logit(logger) - def initialize_analysis(self) -> None: + def initialize(self) -> None: """Initialize a global atm analysis This method will initialize a global atm analysis. This includes: + - initializing JEDI variational application - staging observation files - staging bias correction files - staging CRTM fix files @@ -130,36 +99,30 @@ def initialize_analysis(self) -> None: None """ - # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) - jcb_config['algorithm'] = 'atm_obs_staging' - obs_dict = render(jcb_config) - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") - - # Test - jcb_config = {} - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_config.update(parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)) - jcb_config['algorithm'] = 'atm_bias_staging' - bias_dict = render(jcb_config) - logger.debug(f"foo:\n{pformat(bias_dict)}") + # initialize JEDI variational application + logger.info(f"Initializing JEDI ensemble DA application") + self.jedi.initialize(self.task_config) # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi.get_obs_dict(self.task_config) + logger.info(f"Staging list of observation files") + obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") + # stage bias corrections + logger.info(f"Staging list of bias correction files") + bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + logger.debug(f"Bias correction files bar2:\n{pformat(bias_dict)}") # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") @@ -208,31 +171,6 @@ def initialize_analysis(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: - """Run JEDI executable - - This method will run JEDI executables for the global atm analysis - - Parameters - ---------- - aprun_cmd : str - Run command for JEDI application on HPC system - jedi_args : List - List of additional optional arguments for JEDI application - - Returns - ---------- - None - """ - - if jedi_args: - logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") - else: - logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - - self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - @logit(logger) def finalize(self) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 4b2f8ebbf40..73cf5ea8c85 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -74,45 +74,15 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = Jedi(self.task_config, yaml_name) + self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) @logit(logger) - def initialize_jedi(self): - """Initialize JEDI application - - This method will initialize a JEDI application used in the global atmens analysis. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - # get JEDI config and save to YAML file - logger.info(f"Generating JEDI config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI-to-FV3 increment converter executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) - - @logit(logger) - def initialize_analysis(self) -> None: + def initialize(self) -> None: """Initialize a global atmens analysis This method will initialize a global atmens analysis. This includes: + - initialize JEDI ensemble DA application - staging observation files - staging bias correction files - staging CRTM fix files @@ -128,19 +98,21 @@ def initialize_analysis(self) -> None: ---------- None """ - super().initialize() + + # initialize JEDI ensemble DA application + logger.info(f"Initializing JEDI ensemble DA application") + self.jedi.initialize(self.task_config) # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi.get_obs_dict(self.task_config) + logger.info(f"Staging list of observation files") + obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections - logger.info(f"Staging list of bias correction files generated from JEDI config") - self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" - bias_file = f"rad_varbc_params.tar" - bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) + logger.info(f"Staging list of bias correction files") + bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -175,30 +147,6 @@ def initialize_analysis(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: - """Run JEDI executable - - This method will run JEDI executables for the global atmens analysis - - Parameters - ---------- - aprun_cmd : str - Run command for JEDI application on HPC system - jedi_args : List - List of additional optional arguments for JEDI application - Returns - ---------- - None - """ - - if jedi_args: - logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") - else: - logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - - self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis From f18d4ce5e585053a64c6b3a933997a472f09feae Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 17:48:53 +0000 Subject: [PATCH 035/142] Initialize all JEDI applications on the initialize jobs --- parm/config/gfs/config.atmanl | 6 +- parm/config/gfs/config.atmanlfv3inc | 3 - parm/config/gfs/config.atmensanl | 13 +-- parm/config/gfs/config.atmensanlfv3inc | 3 - parm/config/gfs/config.atmensanlobs | 2 - parm/config/gfs/config.atmensanlsol | 2 - parm/config/gfs/yaml/defaults.yaml | 9 +- .../exglobal_atm_analysis_fv3_increment.py | 4 +- scripts/exglobal_atm_analysis_initialize.py | 3 +- scripts/exglobal_atm_analysis_variational.py | 4 +- .../exglobal_atmens_analysis_fv3_increment.py | 5 +- .../exglobal_atmens_analysis_initialize.py | 10 +-- scripts/exglobal_atmens_analysis_letkf.py | 9 +- scripts/exglobal_atmens_analysis_obs.py | 4 +- scripts/exglobal_atmens_analysis_sol.py | 5 +- ush/python/pygfs/jedi/jedi.py | 75 ++++++++-------- ush/python/pygfs/task/atm_analysis.py | 51 +++++++++-- ush/python/pygfs/task/atmens_analysis.py | 89 +++++++++++++++++-- 18 files changed, 195 insertions(+), 102 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 9a06088eccf..a2baadde7bc 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -6,7 +6,8 @@ echo "BEGIN: config.atmanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ +export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ +export JCB_ALGO_FV3INC="fv3jedi_fv3inc_variational" export STATICB_TYPE=@STATICB_TYPE@ export LOCALIZATION_TYPE="bump" @@ -33,6 +34,7 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/gdas.x +export JEDIEXE_VAR="${EXECgfs}/gdas.x" +export JEDIEXE_FV3INC="${EXECgfs}/fv3jedi_fv3inc.x" echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index ab7efa3a606..4e7714628e6 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -8,7 +8,4 @@ echo "BEGIN: config.atmanlfv3inc" # Get task specific resources . "${EXPDIR}/config.resources" atmanlfv3inc -export JCB_ALGO=fv3jedi_fv3inc_variational -export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x - echo "END: config.atmanlfv3inc" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index f5a12782485..2c575258343 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -6,11 +6,11 @@ echo "BEGIN: config.atmensanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -if [[ ${lobsdiag_forenkf} = ".false." ]] ; then - export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@ -else - export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@ -fi + +export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ +export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ +export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ +export JCB_ALGO_FV3INC="fv3jedi_fv3inc_lgetkf" export INTERP_METHOD='barycentric' @@ -24,6 +24,7 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/gdas.x +export JEDIEXE_LETKF=${EXECgfs}/gdas.x +export JEDIEXE_FV3INC=${EXECgfs}/fv3jedi_fv3inc.x echo "END: config.atmensanl" diff --git a/parm/config/gfs/config.atmensanlfv3inc b/parm/config/gfs/config.atmensanlfv3inc index 2dc73f3f6e1..fe3337e5a28 100644 --- a/parm/config/gfs/config.atmensanlfv3inc +++ b/parm/config/gfs/config.atmensanlfv3inc @@ -8,7 +8,4 @@ echo "BEGIN: config.atmensanlfv3inc" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlfv3inc -export JCB_ALGO=fv3jedi_fv3inc_lgetkf -export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x - echo "END: config.atmensanlfv3inc" diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs index dff3fa3095a..c7e050b009e 100644 --- a/parm/config/gfs/config.atmensanlobs +++ b/parm/config/gfs/config.atmensanlobs @@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlobs" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlobs -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ - echo "END: config.atmensanlobs" diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol index dac161373b7..8ef905d1bde 100644 --- a/parm/config/gfs/config.atmensanlsol +++ b/parm/config/gfs/config.atmensanlsol @@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlsol" # Get task specific resources . "${EXPDIR}/config.resources" atmensanlsol -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ - echo "END: config.atmensanlsol" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index dfc67d12375..caadf234646 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -23,7 +23,7 @@ base: FHMAX_ENKF_GFS: 12 atmanl: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" + JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" STATICB_TYPE: "gsibec" LAYOUT_X_ATMANL: 8 LAYOUT_Y_ATMANL: 8 @@ -33,16 +33,11 @@ atmanl: atmensanl: JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" + JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 - -atmensanlobs: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" - -atmensanlsol: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" aeroanl: IO_LAYOUT_X: 1 diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index f1422cca89a..d49dbe103c0 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis object - AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') + AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi_fv3inc.execute() diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index d9af2712353..444a532f6c8 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -20,8 +20,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config, 'atmanlvar') + AtmAnl = AtmAnalysis(config) # Initialize JEDI variational analysis - AtmAnl.jedi.initialize(AtmAnl.task_config) AtmAnl.initialize() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index cba7a33a5d0..1200fe147cc 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config, 'atmanlvar') + AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) + AtmAnl.jedi_var.execute() diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 48eb6a6a1e0..dab8803d9c7 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -18,8 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis object - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlfv3inc') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.initialize_jedi() - AtmEnsAnl.execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.jedi_fv3inc.execute() diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 26bb9a6dab0..010a6f10758 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -20,11 +20,11 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - if not config.lobsdiag_forenkf: - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') - else: - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + AtmEnsAnl = AtmEnsAnalysis(config) +# if not config.lobsdiag_forenkf: +# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') +# else: +# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis - AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 45b06524fee..33d2fd3b8fe 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -18,7 +18,12 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + AtmEnsAnl = AtmEnsAnalysis(config) + # Initalize JEDI ensemble DA application + # Note: This is normally done in AtmEnsAnl.initialize(), but the that now + # initializes the split observer-solver. This case is just for testing. + AtmEnsAnl.jedi_letkf.initialize() + # Execute the JEDI ensemble DA analysis - AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf.execute() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index ac3271272e7..660276d3e42 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf_obs.execute() diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index d93c42ddef4..a1e71dc1cf2 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -18,8 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi.initialize(AtmEnsAnl.task_config) - AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) + AtmEnsAnl.jedi_letkf_sol.execute() diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 366f0f74704..a57871d6192 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -20,12 +20,11 @@ class Jedi: Class for initializing and executing JEDI applications """ @logit(logger, name="Jedi") - def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: + def __init__(self, config) -> None: """Constructor for JEDI objects This method will construct a Jedi object. This includes: - - save a copy of task_config for provenance - set the default JEDI YAML and executable names - set an empty AttrDict for the JEDI config - set the default directory for J2-YAML templates @@ -34,28 +33,23 @@ def __init__(self, DATA: str, JEDIEXE: str, yaml_name: Optional[str]) -> None: ---------- task_config: AttrDict Attribute-dictionary of all configuration variables associated with a GDAS task. - yaml_name: str, optional - Name of YAML file for JEDI configuration Returns ---------- None """ - - _exe_name = os.path.basename(JEDIEXE) - - self.exe_src = JEDIEXE - self.rundir = DATA - self.exe = os.path.join(DATA, _exe_name) - if yaml_name: - self.yaml = os.path.join(DATA, yaml_name + '.yaml') - else: - self.yaml = os.path.join(DATA, os.path.splitext(_exe_name)[0] + '.yaml') - - # Initialize empty JEDI input config attribute-dictionary - self.config = AttrDict() -# self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + # Create the configuration dictionary for JEDI object + self.jedi_config = config.deepcopy() + + local_dict = AttrDict( + { + 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), + 'yaml': os.path.join(DATA, config.yaml_name + '.yaml'), + 'input_config': None + } + ) + self.jedi_config.update(local_dict) @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -69,16 +63,16 @@ def initialize(self, task_config: AttrDict) -> None: """ # Render JEDI config dictionary - logger.info(f"Generating JEDI YAML config: {self.yaml}") - self.config = self.get_config(task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.info(f"Generating JEDI YAML config: {self.jedi_config.yaml}") + self.jedi_config.input_config = self.render_jcb(task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi_config.input_config)}") # Save JEDI config dictionary to YAML in run directory - logger.debug(f"Writing JEDI YAML config to: {self.yaml}") - save_as_yaml(self.config, self.yaml) + logger.debug(f"Writing JEDI YAML config to: {self.jedi_config.yaml}") + save_as_yaml(self.jedi_config.input_config, self.jedi_config.yaml) # Link JEDI executable to run directory - logger.info(f"Linking JEDI executable {self.exe_src} to {self.exe}") + logger.info(f"Linking JEDI executable {self.jedi_config.exe_src} to {self.jedi_config.exe}") self.link_exe() @logit(logger) @@ -100,12 +94,12 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: chdir(self.rundir) - exec_cmd = Executable(aprun_cmd) - exec_cmd.add_default_arg(self.exe) - if jedi_args: - for arg in jedi_args: + exec_cmd = Executable(self.jedi_config.aprun_cmd) + exec_cmd.add_default_arg(self.jedi_config.exe) + if self.jedi_config.jedi_args: + for arg in self.jedi_config.jedi_args: exec_cmd.add_default_arg(arg) - exec_cmd.add_default_arg(self.yaml) + exec_cmd.add_default_arg(self.jedi_config.yaml) logger.info(f"Executing {exec_cmd}") try: @@ -116,7 +110,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") @logit(logger) - def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: + def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters @@ -125,19 +119,24 @@ def get_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Dictionary of all configuration variables associated with a GDAS task. algorithm (optional) : str Name of the algorithm used to generate the JEDI configuration dictionary. - It will override the algorithm set in the task_config.JCB_ALGO_YAML file. + It will override the algorithm set in the jedi_config.jcb_algo_yaml file. Returns ---------- None - """ + """ + if not self.jedi_config.jcb_algo_yaml and not.algorithm: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be + specified as an input to jedi.render_jcb") + # Fill JCB base YAML template and build JCB config dictionary - jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) # Add JCB algorithm YAML, if it exists, to JCB config dictionary - if 'JCB_ALGO_YAML' in task_config.keys(): - jcb_config.update(parse_j2yaml(task_config.JCB_ALGO_YAML, task_config)) + if self.jedi_config.jcb_algo_yaml: + jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML if algorithm: @@ -164,9 +163,8 @@ def link_exe(self) -> None: # TODO: linking is not permitted per EE2. # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] logger.warn("Linking is not permitted per EE2.") - if os.path.exists(self.exe): - rm_p(self.exe) - os.symlink(self.exe_src, self.exe) + if not os.path.exists(self.jedi_config.exe): + os.symlink(self.jedi_config.exe_src, self.jedi_config.exe) @staticmethod @logit(logger) @@ -224,6 +222,7 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") +# TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f9c81d5456d..0e038165f73 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -23,7 +23,7 @@ class AtmAnalysis(Task): Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any]): """Constructor global atm analysis task This method will construct a global atm analysis task. @@ -35,8 +35,6 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): ---------- config: Dict dictionary object containing task configuration - yaml_name: str, optional - name of YAML file for JEDI configuration Returns ---------- @@ -72,8 +70,35 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create JEDI object - self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) + # Create JEDI variational object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_VAR, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMANLVAR, + 'yaml_name': 'atmanlvar', + 'jedi_args': ['fv3jedi', 'variational'] + } + ) + self.jedi_var = Jedi(jedi_config) + + # Create JEDI FV3 increment converter object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_FV3INC, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo_yaml': None, + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, + 'yaml_name': 'atmanlfv3inc', + 'jedi_args': None + } + ) + self.jedi_fv3inc = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: @@ -81,7 +106,7 @@ def initialize(self) -> None: This method will initialize a global atm analysis. This includes: - - initializing JEDI variational application + - initialize JEDI applications - staging observation files - staging bias correction files - staging CRTM fix files @@ -98,16 +123,24 @@ def initialize(self) -> None: ---------- None """ + + # initialize JEDI variational application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_var.initialize() + + # initialize JEDI FV3 increment conversion application + logger.info(f"Initializing JEDI FV3 increment conversion application") + self.jedi_fv3inc.initialize() # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -115,7 +148,7 @@ def initialize(self) -> None: # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi.extract_tar(tar_file) + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 859372e6e53..1ca12484519 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -28,7 +28,7 @@ class AtmEnsAnalysis(Task): Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any]): """Constructor global atmens analysis task This method will construct a global atmens analysis task. @@ -40,8 +40,6 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): ---------- config: Dict dictionary object containing task configuration - yaml_name: str, optional - name of YAML file for JEDI configuration Returns ---------- @@ -73,15 +71,76 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create JEDI object - self.jedi = Jedi(self.task_config.DATA, self.task_config.JEDIEXE, yaml_name) + # Create JEDI LETKF observer object + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS + 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, + 'yaml_name': 'atmensanlobs', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf_obs = Jedi(jedi_config) + # Create JEDI LETKF solver object + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL + 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, + 'yaml_name': 'atmensanlsol', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf_sol = Jedi(jedi_config) + + # Create JEDI FV3 increment converter + jedi_config = AttrDict( + { + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_FV3INC, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo_yaml': None + 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, + 'yaml_name': 'atmensanlfv3inc', + 'jedi_args': None + } + ) + self.jedi_fv3inc = Jedi(jedi_config) + + # Note: Since we now use the split observer-solvers, the following + # is only for testing. + + # Create JEDI LETKF object + jedi_config = AttrDict( + { + 'exe_src': self.task_config.JEDIEXE_LETKF, + 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_algo': None + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF + 'rundir': self.task_config.DATA, + 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, + 'yaml_name': 'atmensanlletkf', + 'jedi_args': ['fv3jedi', 'localensembleda'] + } + ) + self.jedi_letkf = Jedi(jedi_config) + @logit(logger) def initialize(self) -> None: """Initialize a global atmens analysis This method will initialize a global atmens analysis. This includes: + - initialize JEDI applications - staging observation files - staging bias correction files - staging CRTM fix files @@ -98,23 +157,35 @@ def initialize(self) -> None: None """ + # initialize JEDI LETKF observer application + logger.info(f"Initializing JEDI LETKF observer application") + self.jedi_letkf_obs.initialize() + + # initialize JEDI LETKF solver application + logger.info(f"Initializing JEDI LETKF solver application") + self.jedi_letkf_sol.initialize() + + # initialize JEDI FV3 increment conversion application + logger.info(f"Initializing JEDI FV3 increment conversion application") + self.jedi_fv3inc.initialize() + # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi.get_config(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi.get_config(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict['copy'] = jedi_letkf_obs.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi.extract_tar(tar_file) + self.jedi_letkf_obs..extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From d5e476b7a8a8ba4406e3cbb62dd13463c0708508 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 17:57:38 +0000 Subject: [PATCH 036/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d39bf615703..12633b1fddd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d39bf61570394730e17cd6508307ff7a624cd3cd +Subproject commit 12633b1fddd230bb16a7d88b438542c88b4623f2 From cec645fca86e57d540eb789d1727dec586832533 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:08:46 +0000 Subject: [PATCH 037/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 27 ++++++++++++------------ ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 8 +++---- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index a57871d6192..ff925692293 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -38,10 +38,10 @@ def __init__(self, config) -> None: ---------- None """ - + # Create the configuration dictionary for JEDI object self.jedi_config = config.deepcopy() - + local_dict = AttrDict( { 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), @@ -74,7 +74,7 @@ def initialize(self, task_config: AttrDict) -> None: # Link JEDI executable to run directory logger.info(f"Linking JEDI executable {self.jedi_config.exe_src} to {self.jedi_config.exe}") self.link_exe() - + @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application @@ -101,7 +101,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.jedi_config.yaml) - logger.info(f"Executing {exec_cmd}") + logger.info(f"Executing {exec_cmd}") try: exec_cmd() except OSError: @@ -124,16 +124,16 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Returns ---------- None - """ + """ - if not self.jedi_config.jcb_algo_yaml and not.algorithm: + if not self.jedi_config.jcb_algo_yaml and not algorithm: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb") - + specified as an input to jedi.render_jcb()") + # Fill JCB base YAML template and build JCB config dictionary jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) - + # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) @@ -146,7 +146,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> jedi_config = render(jcb_config) return jedi_config - + @logit(logger) def link_exe(self) -> None: """Link JEDI executable to run directory @@ -185,10 +185,10 @@ def remove_redundant(input_list: List) -> List: output_list = [] for item in input_list: if item not in output_list: - output_list.append(item); + output_list.append(item) return output_list - + @staticmethod @logit(logger) def extract_tar(tar_file: str) -> None: @@ -222,7 +222,8 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") -# TODO: remove since no longer used + +# TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 0e038165f73..b5fedbf994e 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -130,8 +130,8 @@ def initialize(self) -> None: # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() - + self.jedi_fv3inc.initialize() + # stage observations logger.info(f"Staging list of observation files") obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1ca12484519..ebd1aede62b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -118,7 +118,7 @@ def __init__(self, config: Dict[str, Any]): # Note: Since we now use the split observer-solvers, the following # is only for testing. - + # Create JEDI LETKF object jedi_config = AttrDict( { @@ -133,7 +133,7 @@ def __init__(self, config: Dict[str, Any]): } ) self.jedi_letkf = Jedi(jedi_config) - + @logit(logger) def initialize(self) -> None: """Initialize a global atmens analysis @@ -157,11 +157,11 @@ def initialize(self) -> None: None """ - # initialize JEDI LETKF observer application + # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") self.jedi_letkf_obs.initialize() - # initialize JEDI LETKF solver application + # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") self.jedi_letkf_sol.initialize() From bc7fbbd9414ba8ffc461346e94d07d644eb45435 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:11:33 +0000 Subject: [PATCH 038/142] pynorms #2 --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index ff925692293..47d4ba09930 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -129,7 +129,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> if not self.jedi_config.jcb_algo_yaml and not algorithm: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb()") + specified as an input to jedi.render_jcb()") # Fill JCB base YAML template and build JCB config dictionary jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) From b3d323994e92290f64e03cc5cbf8db7a74111e7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Oct 2024 22:14:06 +0000 Subject: [PATCH 039/142] pynorms #3 --- scripts/exglobal_atmens_analysis_letkf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 33d2fd3b8fe..b6b47a22641 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -21,9 +21,9 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initalize JEDI ensemble DA application - # Note: This is normally done in AtmEnsAnl.initialize(), but the that now + # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. AtmEnsAnl.jedi_letkf.initialize() - + # Execute the JEDI ensemble DA analysis AtmEnsAnl.jedi_letkf.execute() From afada5d224d9b0873b60688481e180018f56f7d2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 00:21:57 +0000 Subject: [PATCH 040/142] Fix some python bugs --- ush/python/pygfs/jedi/jedi.py | 19 +++++++++++++------ ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 18 +++++++++--------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 47d4ba09930..ec0cd8050f0 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -126,13 +126,12 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> None """ - if not self.jedi_config.jcb_algo_yaml and not algorithm: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI config must contain jcb_algo_yaml or algorithm be - specified as an input to jedi.render_jcb()") - # Fill JCB base YAML template and build JCB config dictionary - jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) + if self.jedi_config.jcb_base_yaml: + jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) + else: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: JEDI config must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: @@ -141,6 +140,14 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML if algorithm: jcb_config['algorithm'] = algorithm + elif self.jedi_config.jcb_algo: + jcb_config['algorithm'] = self.jedi_config.jcb_algo + elif 'algorithm' in jcb_config: + pass + else: + logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + logger.error(f"FATAL ERROR: algorithm must be specified in JEDI config, " + + "JCB algorithm YAML, or as input to jedi.render_jcb()") # Generate JEDI YAML config by rendering JCB config dictionary jedi_config = render(jcb_config) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index b5fedbf994e..02aeb47e47a 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -90,7 +90,7 @@ def __init__(self, config: Dict[str, Any]): { 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, 'rundir': self.task_config.DATA, 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index ebd1aede62b..410244138b8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -77,8 +77,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS, 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] @@ -92,8 +92,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL, 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] @@ -107,8 +107,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC - 'jcb_algo_yaml': None + 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, + 'jcb_algo_yaml': None, 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None @@ -124,8 +124,8 @@ def __init__(self, config: Dict[str, Any]): { 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.jcb_base_yaml, - 'jcb_algo': None - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF + 'jcb_algo': None, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, 'yaml_name': 'atmensanlletkf', @@ -185,7 +185,7 @@ def initialize(self) -> None: # extract bias corrections tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_letkf_obs..extract_tar(tar_file) + self.jedi_letkf_obs.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 9e57e52ae0e4e78f169e75109c4fa2fb65629e9d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 01:12:30 +0000 Subject: [PATCH 041/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 12633b1fddd..3bf7b50f324 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 12633b1fddd230bb16a7d88b438542c88b4623f2 +Subproject commit 3bf7b50f324d4073cb357d0088bd11d324038593 From 25fad0e4587f3de7faa6af1c7de62d00bcbab5b7 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 03:05:50 +0000 Subject: [PATCH 042/142] Fixing bugs --- .../exglobal_atm_analysis_fv3_increment.py | 2 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_fv3_increment.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 4 +-- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 27 ++++++++-------- ush/python/pygfs/task/atm_analysis.py | 21 +++++++------ ush/python/pygfs/task/atmens_analysis.py | 31 +++++++++---------- 9 files changed, 46 insertions(+), 47 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index d49dbe103c0..594ae6e7f7a 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi_fv3inc.execute() + AtmAnl.jedi_fv3inc.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 1200fe147cc..c7929f6b19e 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi_var.execute() + AtmAnl.jedi_var.execute(config.APRUN_ATMANLVAR) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index dab8803d9c7..42d0afceed7 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi_fv3inc.execute() + AtmEnsAnl.jedi_fv3inc.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index b6b47a22641..050449334ec 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -23,7 +23,7 @@ # Initalize JEDI ensemble DA application # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi_letkf.initialize() + AtmEnsAnl.jedi_letkf.initialize(AtmEnsAnl.task_config) # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi_letkf.execute() + AtmEnsAnl.jedi_letkf.execute(config.APRUN_ATMENSANLLETKF) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 660276d3e42..6cd961c99fa 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi_letkf_obs.execute() + AtmEnsAnl.jedi_letkf_obs.execute(config.APRUN_ATMENSANLOBS) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index a1e71dc1cf2..dab5206daf4 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi_letkf_sol.execute() + AtmEnsAnl.jedi_letkf_sol.execute(config.APRUN_ATMENSANLSOL) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index ec0cd8050f0..dfbd1674079 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -40,16 +40,17 @@ def __init__(self, config) -> None: """ # Create the configuration dictionary for JEDI object - self.jedi_config = config.deepcopy() - local_dict = AttrDict( { - 'exe': os.path.join(self.config.run_dir, os.path.basename(self.config.exe_src)), - 'yaml': os.path.join(DATA, config.yaml_name + '.yaml'), + 'exe': os.path.join(config.rundir, os.path.basename(config.exe_src)), + 'yaml': os.path.join(config.rundir, config.yaml_name + '.yaml'), 'input_config': None } ) - self.jedi_config.update(local_dict) + self.jedi_config = AttrDict(**config, **local_dict) + + # Save a copy of jedi_config + self._jedi_config = self.jedi_config.deepcopy() @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -76,15 +77,13 @@ def initialize(self, task_config: AttrDict) -> None: self.link_exe() @logit(logger) - def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + def execute(self, aprun_cmd: str) -> None: """Execute JEDI application Parameters ---------- aprun_cmd: str String comprising the run command for the JEDI executable. - jedi_args (optional): List - List of strings comprising optional input arguments for the JEDI executable. Returns ---------- @@ -92,9 +91,9 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: Attribute-dictionary of JEDI configuration rendered from a template. """ - chdir(self.rundir) + chdir(self.jedi_config.rundir) - exec_cmd = Executable(self.jedi_config.aprun_cmd) + exec_cmd = Executable(aprun_cmd) exec_cmd.add_default_arg(self.jedi_config.exe) if self.jedi_config.jedi_args: for arg in self.jedi_config.jedi_args: @@ -131,13 +130,13 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI config must contain jcb_base_yaml.") + logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) - # Set algorithm in JCB config dictionary or override the one set by JCB_ALGO_YAML + # Set algorithm in JCB config dictionary if algorithm: jcb_config['algorithm'] = algorithm elif self.jedi_config.jcb_algo: @@ -146,8 +145,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> pass else: logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: algorithm must be specified in JEDI config, " + - "JCB algorithm YAML, or as input to jedi.render_jcb()") + logger.error(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary jedi_config = render(jcb_config) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 02aeb47e47a..04f161c81d1 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -74,11 +74,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_VAR, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMANLVAR, 'yaml_name': 'atmanlvar', 'jedi_args': ['fv3jedi', 'variational'] } @@ -89,11 +88,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMANLFV3INC, 'yaml_name': 'atmanlfv3inc', 'jedi_args': None } @@ -126,11 +124,11 @@ def initialize(self) -> None: # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") - self.jedi_var.initialize() + self.jedi_var.initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() + self.jedi_fv3inc.initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") @@ -141,14 +139,17 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = self.jedi_var.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") - logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + for item in bias_dict['copy']: + bias_file = os.path.basename(item[0]) + if os.path.splitext(bias_file)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{bias_file}" + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 410244138b8..332172fa8d4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -76,10 +76,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_OBS, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLOBS, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -91,10 +90,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF_SOL, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLSOL, + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -106,10 +104,9 @@ def __init__(self, config: Dict[str, Any]): { 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLFV3INC, 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None } @@ -123,11 +120,10 @@ def __init__(self, config: Dict[str, Any]): jedi_config = AttrDict( { 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.jcb_base_yaml, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, - 'aprun_cmd': self.task_config.APRUN_ATMENSANLLETKF, 'yaml_name': 'atmensanlletkf', 'jedi_args': ['fv3jedi', 'localensembleda'] } @@ -159,15 +155,15 @@ def initialize(self) -> None: # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") - self.jedi_letkf_obs.initialize() + self.jedi_letkf_obs.initialize(self.task_config) # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") - self.jedi_letkf_sol.initialize() + self.jedi_letkf_sol.initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize() + self.jedi_fv3inc.initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") @@ -183,9 +179,12 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") - logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_letkf_obs.extract_tar(tar_file) + for item in bias_dict['copy']: + bias_file = os.path.basename(item[0]) + if os.path.splitext(bias_file)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{bias_file}" + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi_var.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 78bf67d1fe991d71c22adaad9d9da0741d56a569 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 03:12:37 +0000 Subject: [PATCH 043/142] Bug fix --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 04f161c81d1..60c904396ff 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -139,7 +139,7 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = self.jedi_var.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -149,7 +149,7 @@ def initialize(self) -> None: if os.path.splitext(bias_file)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{bias_file}" logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + Jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 332172fa8d4..178a8ee097c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -174,7 +174,7 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = jedi_letkf_obs.remove_redundant(bias_dict['copy']) + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") @@ -184,7 +184,7 @@ def initialize(self) -> None: if os.path.splitext(bias_file)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{bias_file}" logger.info(f"Extract bias correction files from {tar_file}") - self.jedi_var.extract_tar(tar_file) + Jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 1cf5f7bae0ced145ea4a14b81424b00199b57959 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 8 Oct 2024 13:08:39 +0000 Subject: [PATCH 044/142] scripts updated to run copy successfully --- jobs/JGLOBAL_ANALYSIS_STATS | 8 +++++++- ush/python/pygfs/task/stat_analysis.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index da0193ea82d..5de337783c4 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -6,12 +6,18 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## # Set variables used in the script ############################################## - +# shellcheck disable=SC2153 +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} +GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS ############################################################### # Run relevant script diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index d57cadfadd6..c6c3f32deb8 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -127,8 +127,7 @@ def initialize_analysis(self) -> None: logger.info(f"Copying files to {self.task_config.DATA}/stats") - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS_TMPL, f"{self.task_config['APREFIX']}aerostat") + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") dest = os.path.join(self.task_config.DATA, "stats") statlist = [[aerostat, dest]] - #print(${MEMDIR}) FileHandler({'copy': statlist}).sync() From dadda23f2a5f6cff16ec5556b03e80ebda471cd9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:37:15 +0000 Subject: [PATCH 045/142] Remove redundant code for tarball extraction, update comments, and update GDAS hash --- sorc/gdas.cd | 2 +- ush/python/pygfs/jedi/jedi.py | 107 ++++++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 9 +- ush/python/pygfs/task/atmens_analysis.py | 9 +- 4 files changed, 79 insertions(+), 48 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 3bf7b50f324..8a501f79054 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 3bf7b50f324d4073cb357d0088bd11d324038593 +Subproject commit 8a501f7905497f5b65649fd05c825f5869b2a6c8 diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dfbd1674079..bda70e86d40 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -25,14 +25,13 @@ def __init__(self, config) -> None: This method will construct a Jedi object. This includes: - - set the default JEDI YAML and executable names - - set an empty AttrDict for the JEDI config - - set the default directory for J2-YAML templates + - create the jedi_config AttrDict and extend it with additional required entries + - save a coy of jedi_config Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + config: AttrDict + Attribute-dictionary of all configuration variables required for the Jedi class Returns ---------- @@ -58,9 +57,18 @@ def initialize(self, task_config: AttrDict) -> None: This method will initialize a JEDI application. This includes: - - generating JEDI YAML config - - saving JEDI YAML config to run directory + - generating JEDI input YAML config + - saving JEDI input YAML config to run directory - linking the JEDI executable to run directory + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + + Returns + ---------- + None """ # Render JEDI config dictionary @@ -87,8 +95,7 @@ def execute(self, aprun_cmd: str) -> None: Returns ---------- - jedi_config: AttrDict - Attribute-dictionary of JEDI configuration rendered from a template. + None """ chdir(self.jedi_config.rundir) @@ -122,7 +129,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> Returns ---------- - None + jedi_input_config: AttrDict + Attribute-dictionary of JEDI configuration rendered from a template. """ # Fill JCB base YAML template and build JCB config dictionary @@ -149,9 +157,9 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary - jedi_config = render(jcb_config) + jedi_input_config = render(jcb_config) - return jedi_config + return jedi_input_config @logit(logger) def link_exe(self) -> None: @@ -197,37 +205,70 @@ def remove_redundant(input_list: List) -> List: @staticmethod @logit(logger) - def extract_tar(tar_file: str) -> None: - """Extract files from a tarball - - This method extract files from a tarball + def extract_tar_from_fh_dict(fh_dict) -> None: + """Extract tarballs from FileHandler input dictionary + This method extracts files from tarballs specified in a FileHander + input dictionary for the 'copy' action. + Parameters ---------- - tar_file - path/name of tarball + fh_dict + Input dictionary for FileHandler Returns ---------- None """ - - # extract files from tar file - tar_path = os.path.dirname(tar_file) - try: - with tarfile.open(tar_file, "r") as tarball: - tarball.extractall(path=tar_path) - logger.info(f"Extract {tarball.getnames()}") - except tarfile.ReadError as err: - if tarfile.is_tarfile(tar_file): - logger.error(f"FATAL ERROR: {tar_file} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") + + for item in fh_dict['copy']: + # Use the filename from the destination entry if it's a file path + # Otherwise, it's a directory, so use the source entry filename + if os.path.isfile(item[1]): + filename = os.path.basename(item[1]) else: - logger.info() - except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") + filename = os.path.basename(item[0]) + + # Extract if file is a tarball + if os.path.splitext(filename)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{filename}" + if os.path.isfile(tar_file): + logger.info(f"Extract files from {tar_file}") + extract_tar(tar_file) + else: + logger.error(f"FATAL ERROR: {tar_file} could not be read") + logger.error(f"FATAL ERROR: {tar_file} does not exist!") +@logit(logger) +def extract_tar(tar_file: str) -> None: + """Extract files from a tarball + + This method extract files from a tarball + + Parameters + ---------- + tar_file + path/name of tarball + Returns + ---------- + None + """ + + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tar_file): + logger.error(f"FATAL ERROR: {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") # TODO: remove since no longer used @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 60c904396ff..20d5264420b 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -29,7 +29,7 @@ def __init__(self, config: Dict[str, Any]): This method will construct a global atm analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - - instantiate the Jedi attribute object + - instantiate the Jedi attribute objects Parameters ---------- @@ -144,12 +144,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - for item in bias_dict['copy']: - bias_file = os.path.basename(item[0]) - if os.path.splitext(bias_file)[1] == '.tar': - tar_file = f"{os.path.dirname(item[1])}/{bias_file}" - logger.info(f"Extract bias correction files from {tar_file}") - Jedi.extract_tar(tar_file) + Jedi.extract_tar_from_fh_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 178a8ee097c..b00dc4d9e83 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -34,7 +34,7 @@ def __init__(self, config: Dict[str, Any]): This method will construct a global atmens analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - - instantiate the Jedi attribute object + - instantiate the Jedi attribute objects Parameters ---------- @@ -179,12 +179,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - for item in bias_dict['copy']: - bias_file = os.path.basename(item[0]) - if os.path.splitext(bias_file)[1] == '.tar': - tar_file = f"{os.path.dirname(item[1])}/{bias_file}" - logger.info(f"Extract bias correction files from {tar_file}") - Jedi.extract_tar(tar_file) + Jedi.extract_tar_from_fh_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From ee22db3861f99e063141b9b25982038dc68c2a55 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:39:54 +0000 Subject: [PATCH 046/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index bda70e86d40..f64dacf9bb8 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -208,9 +208,9 @@ def remove_redundant(input_list: List) -> List: def extract_tar_from_fh_dict(fh_dict) -> None: """Extract tarballs from FileHandler input dictionary - This method extracts files from tarballs specified in a FileHander + This method extracts files from tarballs specified in a FileHander input dictionary for the 'copy' action. - + Parameters ---------- fh_dict @@ -220,7 +220,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: ---------- None """ - + for item in fh_dict['copy']: # Use the filename from the destination entry if it's a file path # Otherwise, it's a directory, so use the source entry filename @@ -238,6 +238,8 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: logger.error(f"FATAL ERROR: {tar_file} could not be read") logger.error(f"FATAL ERROR: {tar_file} does not exist!") + + @logit(logger) def extract_tar(tar_file: str) -> None: """Extract files from a tarball @@ -254,7 +256,7 @@ def extract_tar(tar_file: str) -> None: None """ - # extract files from tar file + # extract files from tar file tar_path = os.path.dirname(tar_file) try: with tarfile.open(tar_file, "r") as tarball: From 057a5e95a61109bf4c5fd583e3c0903cedace5b9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 8 Oct 2024 16:41:56 +0000 Subject: [PATCH 047/142] pynorms #2 --- ush/python/pygfs/jedi/jedi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f64dacf9bb8..b062abb43f5 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -229,7 +229,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: filename = os.path.basename(item[0]) - # Extract if file is a tarball + # Extract if file is a tarball if os.path.splitext(filename)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{filename}" if os.path.isfile(tar_file): @@ -272,6 +272,7 @@ def extract_tar(tar_file: str) -> None: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") + # TODO: remove since no longer used @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: From 408372812a0c9bf2d756d6190589b535e11719de Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 8 Oct 2024 18:42:20 +0000 Subject: [PATCH 048/142] add gunzip --- scripts/exglobal_analysis_stats.py | 2 +- ush/python/pygfs/task/stat_analysis.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 17fb7724a3d..c28c196d279 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -23,5 +23,5 @@ StatAnl = StatAnalysis(config, 'statanlvar') # Initialize JEDI variational analysis - StatAnl.initialize_jedi() StatAnl.initialize_analysis() + StatAnl.initialize_jedi() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index c6c3f32deb8..afae6c34126 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -58,8 +58,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - #'npx_anl': _res_anl + 1, - #'npy_anl': _res_anl + 1, + # 'npx_anl': _res_anl + 1, + # 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", @@ -127,7 +127,23 @@ def initialize_analysis(self) -> None: logger.info(f"Copying files to {self.task_config.DATA}/stats") + # Copy stat files to DATA path aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") - dest = os.path.join(self.task_config.DATA, "stats") + dest = os.path.join(self.task_config.DATA, "aerostats") statlist = [[aerostat, dest]] FileHandler({'copy': statlist}).sync() + + # Open tar file + logger.info(f"Open tarred stat file in {dest}") + with tarfile.open(dest, "r") as tar: + # Extract all files to the current directory + tar.extractall() + + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + + for diagfile in gz_files: + with gzip.open(diagfile, 'rb') as f_in: + with open(diagfile[:-3], 'wb') as f_out: + f_out.write(f_in.read()) From e12b28b45822e1205094999a2c093ec66b5d0174 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 8 Oct 2024 19:04:58 +0000 Subject: [PATCH 049/142] update to fix shellnorms --- jobs/JGLOBAL_ANALYSIS_STATS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 5de337783c4..43f47961301 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -8,9 +8,9 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## # shellcheck disable=SC2153 GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" +# gPDY=${GDATE:0:8} +# gcyc=${GDATE:8:2} +# GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work From 52c7bfc7aed8c553c6f69a1ff42f9ad744ba98e5 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 8 Oct 2024 19:06:51 +0000 Subject: [PATCH 050/142] update to fix shellnorms pt. 2 --- jobs/JGLOBAL_ANALYSIS_STATS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 43f47961301..3512f1a7d55 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Set variables used in the script ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +# GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") # gPDY=${GDATE:0:8} # gcyc=${GDATE:8:2} # GDUMP="gdas" From ccd11fc3b8eccee88668a967babde0020f0f5c53 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 9 Oct 2024 15:27:10 +0000 Subject: [PATCH 051/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 8a501f79054..29696c82ab9 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 8a501f7905497f5b65649fd05c825f5869b2a6c8 +Subproject commit 29696c82ab994e02dcfbeefc4b30953de352605b From c8f38bed573ba21138ca5fbfb0be45abcb1bb0e0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 9 Oct 2024 23:56:36 +0000 Subject: [PATCH 052/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 29696c82ab9..140dc0e6a65 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 29696c82ab994e02dcfbeefc4b30953de352605b +Subproject commit 140dc0e6a6550321b6cf1db6630f3d6ca067dc12 From 700204d6c0c5cf77f36a4c66939d449f7d22c77e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Oct 2024 01:40:13 +0000 Subject: [PATCH 053/142] Address Cory's comments --- ush/python/pygfs/jedi/jedi.py | 20 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b062abb43f5..6a1ec3855cd 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -205,7 +205,7 @@ def remove_redundant(input_list: List) -> List: @staticmethod @logit(logger) - def extract_tar_from_fh_dict(fh_dict) -> None: + def extract_tar_from_filehandler_dict(filehandler_dict) -> None: """Extract tarballs from FileHandler input dictionary This method extracts files from tarballs specified in a FileHander @@ -213,7 +213,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: Parameters ---------- - fh_dict + filehandler_dict Input dictionary for FileHandler Returns @@ -221,7 +221,7 @@ def extract_tar_from_fh_dict(fh_dict) -> None: None """ - for item in fh_dict['copy']: + for item in filehandler_dict['copy']: # Use the filename from the destination entry if it's a file path # Otherwise, it's a directory, so use the source entry filename if os.path.isfile(item[1]): @@ -229,16 +229,13 @@ def extract_tar_from_fh_dict(fh_dict) -> None: else: filename = os.path.basename(item[0]) - # Extract if file is a tarball + # Check if file is a tar ball if os.path.splitext(filename)[1] == '.tar': tar_file = f"{os.path.dirname(item[1])}/{filename}" - if os.path.isfile(tar_file): - logger.info(f"Extract files from {tar_file}") - extract_tar(tar_file) - else: - logger.error(f"FATAL ERROR: {tar_file} could not be read") - logger.error(f"FATAL ERROR: {tar_file} does not exist!") + # Extract tarball + logger.info(f"Extract files from {tar_file}") + extract_tar(tar_file) @logit(logger) def extract_tar(tar_file: str) -> None: @@ -262,6 +259,9 @@ def extract_tar(tar_file: str) -> None: with tarfile.open(tar_file, "r") as tarball: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") + except tarfile.FileExistsError as err: + logger.exception(f"FATAL ERROR: {tar_file} does not exist") + raise tarfile.FileExistsError(f"FATAL ERROR: {tar_file} does not exist") except tarfile.ReadError as err: if tarfile.is_tarfile(tar_file): logger.error(f"FATAL ERROR: {tar_file} could not be read") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 20d5264420b..7b91ecc909e 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -144,7 +144,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - Jedi.extract_tar_from_fh_dict(bias_dict) + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b00dc4d9e83..c532dcb9f5c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -179,7 +179,7 @@ def initialize(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - Jedi.extract_tar_from_fh_dict(bias_dict) + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 62783ce5af36226efb59f9ddf10b90beda92e7db Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Oct 2024 01:44:29 +0000 Subject: [PATCH 054/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 6a1ec3855cd..c9871e4ae3d 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -237,6 +237,7 @@ def extract_tar_from_filehandler_dict(filehandler_dict) -> None: logger.info(f"Extract files from {tar_file}") extract_tar(tar_file) + @logit(logger) def extract_tar(tar_file: str) -> None: """Extract files from a tarball From f2a7a5585d27521d93352d53eb38de809b405394 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 12:46:21 +0000 Subject: [PATCH 055/142] Slight change for readability --- ush/python/pygfs/task/atm_analysis.py | 14 ++++++-------- ush/python/pygfs/task/atmens_analysis.py | 20 ++++++++------------ 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 7b91ecc909e..6d7b1145ff6 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -71,32 +71,30 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI variational object - jedi_config = AttrDict( + self.jedi_var = Jedi(AttrDict( { + 'yaml_name': 'atmanlvar', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_VAR, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, - 'rundir': self.task_config.DATA, - 'yaml_name': 'atmanlvar', 'jedi_args': ['fv3jedi', 'variational'] } ) - self.jedi_var = Jedi(jedi_config) # Create JEDI FV3 increment converter object - jedi_config = AttrDict( + self.jedi_fv3inc = Jedi(AttrDict( { + 'yaml_name': 'atmanlfv3inc', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'rundir': self.task_config.DATA, - 'yaml_name': 'atmanlfv3inc', 'jedi_args': None } ) - self.jedi_fv3inc = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index c532dcb9f5c..334b2622c3f 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -72,63 +72,59 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI LETKF observer object - jedi_config = AttrDict( + self.jedi_letkf_obs = Jedi(AttrDict( { + 'yaml_name': 'atmensanlobs', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, - 'yaml_name': 'atmensanlobs', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf_obs = Jedi(jedi_config) # Create JEDI LETKF solver object - jedi_config = AttrDict( + self.jedi_letkf_sol = Jedi(AttrDict( { + 'yaml_name': 'atmensanlsol', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, - 'yaml_name': 'atmensanlsol', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf_sol = Jedi(jedi_config) # Create JEDI FV3 increment converter - jedi_config = AttrDict( + self.jedi_fv3inc = Jedi(AttrDict( { + 'yaml_name': 'atmensanlfv3inc', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, 'jcb_algo_yaml': None, - 'yaml_name': 'atmensanlfv3inc', 'jedi_args': None } ) - self.jedi_fv3inc = Jedi(jedi_config) # Note: Since we now use the split observer-solvers, the following # is only for testing. # Create JEDI LETKF object - jedi_config = AttrDict( + self.jedi_letkf = Jedi(AttrDict( { + 'yaml_name': 'atmensanlletkf', 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'rundir': self.task_config.DATA, - 'yaml_name': 'atmensanlletkf', 'jedi_args': ['fv3jedi', 'localensembleda'] } ) - self.jedi_letkf = Jedi(jedi_config) @logit(logger) def initialize(self) -> None: From 4baa1d53f9ce6fc9d42bc51c3bbc149b2618a730 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:28:52 +0000 Subject: [PATCH 056/142] Add key checking to JEDI class constructor --- ush/python/pygfs/jedi/jedi.py | 7 ++++++- ush/python/pygfs/task/atm_analysis.py | 15 +++++++++------ ush/python/pygfs/task/atmens_analysis.py | 23 +++++++++++++---------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c9871e4ae3d..fea1c176d7e 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -38,6 +38,11 @@ def __init__(self, config) -> None: None """ + _key_list = ['yaml_name', 'rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] + for key in _key_list: + if key not in config.keys: + raise KeyError(f"Key '{key}' not found in the nested dictionary") + # Create the configuration dictionary for JEDI object local_dict = AttrDict( { @@ -46,7 +51,7 @@ def __init__(self, config) -> None: 'input_config': None } ) - self.jedi_config = AttrDict(**config, **local_dict) + self.jedi_config.update(local_dict) # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 6d7b1145ff6..f08c7e55895 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -70,8 +70,11 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of JEDI objects + self.jedi = AttrDict() + # Create JEDI variational object - self.jedi_var = Jedi(AttrDict( + self.jedi['atmanlvar'] = Jedi(AttrDict( { 'yaml_name': 'atmanlvar', 'rundir': self.task_config.DATA, @@ -84,7 +87,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI FV3 increment converter object - self.jedi_fv3inc = Jedi(AttrDict( + self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', 'rundir': self.task_config.DATA, @@ -122,21 +125,21 @@ def initialize(self) -> None: # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") - self.jedi_var.initialize(self.task_config) + self.jedi['atmanlvar'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize(self.task_config) + self.jedi['atmanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi_var.render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_var.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 334b2622c3f..dc4075971ad 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -71,8 +71,11 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of JEDI objects + self.jedi = AttrDict() + # Create JEDI LETKF observer object - self.jedi_letkf_obs = Jedi(AttrDict( + self.jedi['atmensanlobs'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlobs', 'rundir': self.task_config.DATA, @@ -85,7 +88,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI LETKF solver object - self.jedi_letkf_sol = Jedi(AttrDict( + self.jedi['atmensanlsol'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlsol', 'rundir': self.task_config.DATA, @@ -98,7 +101,7 @@ def __init__(self, config: Dict[str, Any]): ) # Create JEDI FV3 increment converter - self.jedi_fv3inc = Jedi(AttrDict( + self.jedi['atmensanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlfv3inc', 'rundir': self.task_config.DATA, @@ -114,14 +117,14 @@ def __init__(self, config: Dict[str, Any]): # is only for testing. # Create JEDI LETKF object - self.jedi_letkf = Jedi(AttrDict( + self.jedi['atmensanlletkf'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlletkf', + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_LETKF, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': None, 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, - 'rundir': self.task_config.DATA, 'jedi_args': ['fv3jedi', 'localensembleda'] } ) @@ -151,25 +154,25 @@ def initialize(self) -> None: # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") - self.jedi_letkf_obs.initialize(self.task_config) + self.jedi['atmensanlobs'].initialize(self.task_config) # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") - self.jedi_letkf_sol.initialize(self.task_config) + self.jedi['atmensanlsol'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi_fv3inc.initialize(self.task_config) + self.jedi['atmensanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_letkf_obs.render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 2f72ecccfd1c09702faeeefdfeac9e73d54e3945 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:33:19 +0000 Subject: [PATCH 057/142] Update --- ush/python/pygfs/task/atm_analysis.py | 8 ++++---- ush/python/pygfs/task/atmens_analysis.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f08c7e55895..698567cb60d 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -73,7 +73,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - # Create JEDI variational object + # atmanlvar self.jedi['atmanlvar'] = Jedi(AttrDict( { 'yaml_name': 'atmanlvar', @@ -84,9 +84,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, 'jedi_args': ['fv3jedi', 'variational'] } - ) + )) - # Create JEDI FV3 increment converter object + # atmanlfv3inc self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', @@ -97,7 +97,7 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index dc4075971ad..e3196b7e984 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -74,7 +74,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - # Create JEDI LETKF observer object + # atmensanlobs self.jedi['atmensanlobs'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlobs', @@ -85,9 +85,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) - # Create JEDI LETKF solver object + # atmensanlsol self.jedi['atmensanlsol'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlsol', @@ -98,9 +98,9 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) - # Create JEDI FV3 increment converter + # atmensanlfv3inc self.jedi['atmensanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlfv3inc', @@ -111,12 +111,12 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # Note: Since we now use the split observer-solvers, the following # is only for testing. - # Create JEDI LETKF object + # atmensanlletkf self.jedi['atmensanlletkf'] = Jedi(AttrDict( { 'yaml_name': 'atmensanlletkf', @@ -127,7 +127,7 @@ def __init__(self, config: Dict[str, Any]): 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, 'jedi_args': ['fv3jedi', 'localensembleda'] } - ) + )) @logit(logger) def initialize(self) -> None: From e3ffaf0b68b181d2819b54a0611a520da4717839 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:49:16 +0000 Subject: [PATCH 058/142] Update --- scripts/exglobal_atm_analysis_fv3_increment.py | 6 +++--- scripts/exglobal_atm_analysis_initialize.py | 4 ++-- scripts/exglobal_atm_analysis_variational.py | 4 ++-- scripts/exglobal_atmens_analysis_fv3_increment.py | 4 ++-- scripts/exglobal_atmens_analysis_initialize.py | 8 ++------ scripts/exglobal_atmens_analysis_letkf.py | 8 ++++---- scripts/exglobal_atmens_analysis_obs.py | 4 ++-- scripts/exglobal_atmens_analysis_sol.py | 4 ++-- 8 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 594ae6e7f7a..39a9b4e25a2 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the initialize_fv3inc and execute methods -# which convert the JEDI increment into an FV3 increment +# and runs the execute method of its Jedi +# object attribute import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi_fv3inc.execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi['atmanlfv3inc'].execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 444a532f6c8..749d320111f 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -2,8 +2,8 @@ # exglobal_atm_analysis_initialize.py # This script creates an AtmAnalysis class # and runs the initialize method -# which create and stage the runtime directory -# and create the YAML configuration +# which creates and stages the runtime directory +# and creates the YAML configuration # for a global atm variational analysis import os diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index c7929f6b19e..21d99da3a24 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_variational.py # This script creates an AtmAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm variational analysis import os @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi_var.execute(config.APRUN_ATMANLVAR) + AtmAnl.jedi['atmanlvar'].execute(config.APRUN_ATMANLVAR) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 42d0afceed7..288c043adce 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_fv3_increment.py # This script creates an AtmEnsAnalysis object -# and runs the initialize_fv3inc and execute methods +# and runs the execute method of its Jedi object attribute # which convert the JEDI increment into an FV3 increment import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi_fv3inc.execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.jedi['atmensanlfv3inc'].execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 010a6f10758..124e755594e 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -2,8 +2,8 @@ # exglobal_atmens_analysis_initialize.py # This script creates an AtmEnsAnalysis class # and runs the initialize method -# which create and stage the runtime directory -# and create the YAML configuration +# which creates and stages the runtime directory +# and creates the YAML configuration # for a global atm local ensemble analysis import os @@ -21,10 +21,6 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) -# if not config.lobsdiag_forenkf: -# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') -# else: -# AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis AtmEnsAnl.initialize() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 050449334ec..c0516003d66 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the execute method which executes -# the global atm local ensemble analysis +# and runs the execute method of its Jedi object attribute +# which executes the global atm local ensemble analysis import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -23,7 +23,7 @@ # Initalize JEDI ensemble DA application # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi_letkf.initialize(AtmEnsAnl.task_config) + AtmEnsAnl.jedi['atmensanlletkf'].initialize(AtmEnsAnl.task_config) # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi_letkf.execute(config.APRUN_ATMENSANLLETKF) + AtmEnsAnl.jedi['atmensanlletkf'].execute(config.APRUN_ATMENSANLLETKF) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 6cd961c99fa..84b1f280960 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_obs.py # This script creates an AtmEnsAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm local ensemble analysis in observer mode import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi_letkf_obs.execute(config.APRUN_ATMENSANLOBS) + AtmEnsAnl.jedi['atmensanlobs'].execute(config.APRUN_ATMENSANLOBS) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index dab5206daf4..e6c112f97fc 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_sol.py # This script creates an AtmEnsAnalysis object -# and runs the execute method +# and runs the execute method of its Jedi object attribute # which executes the global atm local ensemble analysis in solver mode import os @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi_letkf_sol.execute(config.APRUN_ATMENSANLSOL) + AtmEnsAnl.jedi['atmensanlsol'].execute(config.APRUN_ATMENSANLSOL) From 955f19112700b31422634f46d355c7953821d481 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:53:18 +0000 Subject: [PATCH 059/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 140dc0e6a65..c71d3b17214 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 140dc0e6a6550321b6cf1db6630f3d6ca067dc12 +Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d From e8baba30c32bee0d01fcb0b3d1e48bce1b6be23e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 13:57:03 +0000 Subject: [PATCH 060/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index fea1c176d7e..89c81c93dc0 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -42,7 +42,7 @@ def __init__(self, config) -> None: for key in _key_list: if key not in config.keys: raise KeyError(f"Key '{key}' not found in the nested dictionary") - + # Create the configuration dictionary for JEDI object local_dict = AttrDict( { diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 698567cb60d..5ddb678036c 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -90,7 +90,7 @@ def __init__(self, config: Dict[str, Any]): self.jedi['atmanlfv3inc'] = Jedi(AttrDict( { 'yaml_name': 'atmanlfv3inc', - 'rundir': self.task_config.DATA, + 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_FV3INC, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index e3196b7e984..46072dfe8b1 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -73,7 +73,7 @@ def __init__(self, config: Dict[str, Any]): # Create dictionary of JEDI objects self.jedi = AttrDict() - + # atmensanlobs self.jedi['atmensanlobs'] = Jedi(AttrDict( { From 0805a08e9ceb1f75162394b6f23c58d0d5b1bfe3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 18:29:14 +0000 Subject: [PATCH 061/142] Initial commit --- parm/config/gfs/config.marineanl | 21 +- parm/config/gfs/config.marinebmat | 8 - ush/python/pygfs/task/marine_bmat.py | 282 ++++++++++++--------------- 3 files changed, 144 insertions(+), 167 deletions(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index a19fc015e2e..cf056387ca8 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -15,6 +15,25 @@ export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2 export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" -export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" + +export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" +#export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" + +export JCB_ALGO_GRIDGEN="soca_gridgen" +export JCB_ALGO_DIAGB="soca_diagb" +export JCB_ALGO_SETCORSCALES="soca_setcorscales" +export JCB_ALGO_PARAMETERS_DIFFUSION_HZ="soca_parameters_diffusion_hz" +export JCB_ALGO_VTSCALES="soca_vtscales" +export JCB_ALGO_PARAMETERS_DIFFUSION_VT="soca_parameters_diffusion_vt" +export JCB_ALGO_ENSB="soca_ensb" +export JCB_ALGO_ENSWEIGHTS="soca_ensweights" + +export JEDIEXE_GRIDGEN="${EXECgfs}/gdas_soca_gridgen.x" +export JEDIEXE_DIAGB="${EXECgfs}/gdas_soca_diagb.x" +export JEDIEXE_SETCORSCALES="${EXECgfs}/soca_setcorscales.x" +export JEDIEXE_PARAMETERS_DIFFUSION_HZ="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" +export JEDIEXE_PARAMETERS_DIFFUSION_VT="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" +export JEDIEXE_ENSB="${EXECgfs}/gdas_ens_handler.x" +export JEDIEXE_ENSWEIGHTS="${EXECgfs}/gdas_socahybridweights.x" echo "END: config.marineanl" diff --git a/parm/config/gfs/config.marinebmat b/parm/config/gfs/config.marinebmat index 00352737d0c..d88739dcedc 100644 --- a/parm/config/gfs/config.marinebmat +++ b/parm/config/gfs/config.marinebmat @@ -8,12 +8,4 @@ echo "BEGIN: config.marinebmat" # Get task specific resources . "${EXPDIR}/config.resources" marinebmat -export BERROR_DIAGB_YAML="${PARMgfs}/gdas/soca/berror/soca_diagb.yaml.j2" -export BERROR_VTSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_vtscales.yaml.j2" -export BERROR_DIFFV_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_vt.yaml.j2" -export BERROR_HZSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_setcorscales.yaml" -export BERROR_DIFFH_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_hz.yaml.j2" -export BERROR_ENS_RECENTER_YAML="${PARMgfs}/gdas/soca/berror/soca_ensb.yaml.j2" -export BERROR_HYB_WEIGHTS_YAML="${PARMgfs}/gdas/soca/berror/soca_ensweights.yaml.j2" - echo "END: config.marinebmat" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 93329f05ac0..b316b221f93 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -50,6 +50,100 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of Jedi objects + self.jedi = AttrDict() + + # gridgen + self.jedi['gridgen'] = Jedi(AttrDict( + { + 'yaml_name': 'gridgen', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_GRIDGEN, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_GRIDGEN, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + )) + + # soca_diagb + self.jedi['soca_diagb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_diagb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_DIAGB, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_DIAGB, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + )) + + # soca_parameters_diffusion_vt + self.jedi['soca_parameters_diffusion_vt'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_parameters_diffusion_vt', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_VT, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_VT, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_setcorscales + self.jedi['soca_setcorscales'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_setcorscales', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_SETCORSCALES, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_SETCORSCALES, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_parameters_diffusion_hz + self.jedi['soca_parameters_diffusion_hz'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_parameters_diffusion_hz', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_HZ, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_HZ, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_ensb + self.jedi['soca_ensb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_ensb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_ENSB, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_ENSB, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + + # soca_ensweights + self.jedi['soca_ensb'] = Jedi(AttrDict( + { + 'yaml_name': 'soca_ensb', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE_ENSWEIGHTS, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO_ENSWEIGHTS, + 'jcb_algo_yaml': None, + 'jedi_args': None + } + ) + @logit(logger) def initialize(self: Task) -> None: """Initialize a global B-matrix @@ -63,7 +157,6 @@ def initialize(self: Task) -> None: - generating the YAML files for the JEDI and GDASApp executables - creating output directories """ - super().initialize() # stage fix files logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") @@ -78,54 +171,27 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() - # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) - logger.info(f"Staging SOCA utility yaml files") - soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) - FileHandler(soca_utility_list).sync() - - # generate the variance partitioning YAML file - logger.info(f"Generate variance partitioning YAML file from {self.task_config.BERROR_DIAGB_YAML}") - diagb_config = parse_j2yaml(path=self.task_config.BERROR_DIAGB_YAML, data=self.task_config) - diagb_config.save(os.path.join(self.task_config.DATA, 'soca_diagb.yaml')) - - # generate the vertical decorrelation scale YAML file - logger.info(f"Generate the vertical correlation scale YAML file from {self.task_config.BERROR_VTSCALES_YAML}") - vtscales_config = parse_j2yaml(path=self.task_config.BERROR_VTSCALES_YAML, data=self.task_config) - vtscales_config.save(os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) - - # generate vertical diffusion scale YAML file - logger.info(f"Generate vertical diffusion YAML file from {self.task_config.BERROR_DIFFV_YAML}") - diffvz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFV_YAML, data=self.task_config) - diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) - - # generate the horizontal diffusion YAML files - if True: # TODO(G): skip this section once we have optimized the scales - # stage the correlation scale configuration - logger.info(f"Generate correlation scale YAML file from {self.task_config.BERROR_HZSCALES_YAML}") - FileHandler({'copy': [[self.task_config.BERROR_HZSCALES_YAML, - os.path.join(self.task_config.DATA, 'soca_setcorscales.yaml')]]}).sync() - - # generate horizontal diffusion scale YAML file - logger.info(f"Generate horizontal diffusion scale YAML file from {self.task_config.BERROR_DIFFH_YAML}") - diffhz_config = parse_j2yaml(path=self.task_config.BERROR_DIFFH_YAML, data=self.task_config) - diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) - - # hybrid EnVAR case + # initialize vtscales python script + vtscales_config = self.render_jcb(task_config, 'vtscales') + save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml') + FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), + os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() + + # initialize JEDI applications + self.jedi['gridgen'].initialize(self.task_config) + self.jedi['soca_diagb'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) + self.jedi['soca_setcorscales'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: + self.jedi['soca_ensb'.initialize(self.task_config) + self.jedi['soca_ensweights'].initialize(self.task_config) + + # stage ensemble members for the hybrid background error if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - # stage ensemble membersfiles for use in hybrid background error logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) - - # generate ensemble recentering/rebalancing YAML file - logger.debug("Generate ensemble recentering YAML file") - ensrecenter_config = parse_j2yaml(path=self.task_config.BERROR_ENS_RECENTER_YAML, data=self.task_config) - ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensb.yaml')) - - # generate ensemble weights YAML file - logger.debug("Generate hybrid-weigths YAML file") - hybridweights_config = parse_j2yaml(path=self.task_config.BERROR_HYB_WEIGHTS_YAML, data=self.task_config) - hybridweights_config.save(os.path.join(self.task_config.DATA, 'soca_ensweights.yaml')) - + # create the symbolic link to the static B-matrix directory link_target = os.path.join(self.task_config.DATAstaticb) link_name = os.path.join(self.task_config.DATA, 'staticb') @@ -134,130 +200,30 @@ def initialize(self: Task) -> None: os.symlink(link_target, link_name) @logit(logger) - def gridgen(self: Task) -> None: - # link gdas_soca_gridgen.x - mdau.link_executable(self.task_config, 'gdas_soca_gridgen.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_gridgen.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('gridgen.yaml') - - mdau.run(exec_cmd) - - @logit(logger) - def variance_partitioning(self: Task) -> None: - # link the variance partitioning executable, gdas_soca_diagb.x - mdau.link_executable(self.task_config, 'gdas_soca_diagb.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_diagb.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_diagb.yaml') - - mdau.run(exec_cmd) - - @logit(logger) - def horizontal_diffusion(self: Task) -> None: - """Generate the horizontal diffusion coefficients - """ - # link the executable that computes the correlation scales, gdas_soca_setcorscales.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_setcorscales.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_setcorscales.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_setcorscales.yaml') - - # create a files containing the correlation scales - mdau.run(exec_cmd) - - # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_parameters_diffusion_hz.yaml') - - # compute the coefficients of the diffusion operator - mdau.run(exec_cmd) - - @logit(logger) - def vertical_diffusion(self: Task) -> None: + def execute_vtscales(self: Task) -> None: """Generate the vertical diffusion coefficients """ - # compute the vertical correlation scales based on the MLD - FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), - os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() + # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) - - # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, - # and prepare the command to run it - mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_parameters_diffusion_vt.yaml') - - # compute the coefficients of the diffusion operator - mdau.run(exec_cmd) - - @logit(logger) - def ensemble_perturbations(self: Task) -> None: - """Generate the 3D ensemble of perturbation for the 3DEnVAR - - This method will generate ensemble perturbations re-balanced w.r.t the - deterministic background. - This includes: - - computing a storing the unbalanced ensemble perturbations' statistics - - recentering the ensemble members around the deterministic background and - accounting for the nonlinear steric recentering - - saving the recentered ensemble statistics - """ - mdau.link_executable(self.task_config, 'gdas_ens_handler.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_ens_handler.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_ensb.yaml') - - # generate the ensemble perturbations - mdau.run(exec_cmd) - - @logit(logger) - def hybrid_weight(self: Task) -> None: - """Generate the hybrid weights for the 3DEnVAR - - This method will generate the 3D fields hybrid weights for the 3DEnVAR for each - variables. - TODO(G): Currently implemented for the specific case of the static ensemble members only - """ - mdau.link_executable(self.task_config, 'gdas_socahybridweights.x') - exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) - exec_name = os.path.join(self.task_config.DATA, 'gdas_socahybridweights.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_ensweights.yaml') - - # compute the ensemble weights - mdau.run(exec_cmd) - + mdau.run(exec_cmd) + @logit(logger) - def execute(self: Task) -> None: + def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. """ - chdir(self.task_config.DATA) - self.gridgen() # TODO: This should be optional in case the geometry file was staged - self.variance_partitioning() - self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales - self.vertical_diffusion() - # hybrid EnVAR case + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged + self.execute_vtscales() + self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.ensemble_perturbations() # TODO: refactor this from the old scripts - self.hybrid_weight() # TODO: refactor this from the old scripts + self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From 8cd354a496360230126f4fce7cfcfdafda9bdfaa Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 18:35:10 +0000 Subject: [PATCH 062/142] Debug --- ush/python/pygfs/jedi/jedi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 89c81c93dc0..f873be10779 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -40,7 +40,7 @@ def __init__(self, config) -> None: _key_list = ['yaml_name', 'rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] for key in _key_list: - if key not in config.keys: + if key not in config: raise KeyError(f"Key '{key}' not found in the nested dictionary") # Create the configuration dictionary for JEDI object @@ -51,7 +51,7 @@ def __init__(self, config) -> None: 'input_config': None } ) - self.jedi_config.update(local_dict) + self.jedi_config = AttrDict(**config, **local_dict) # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() From 8bf1a4293a329f145e921374a060cf6bc78f0521 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Oct 2024 20:25:25 +0000 Subject: [PATCH 063/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c71d3b17214..6e85be032c1 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d +Subproject commit 6e85be032c18419fa245d7d0743264c70bd0592c From 8d53fe663e3abe8bde4aa7d5f63705da2cb75799 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 01:22:53 +0000 Subject: [PATCH 064/142] Saving progress --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 2 +- ush/python/pygfs/task/marine_bmat.py | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6e85be032c1..b70652625a7 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6e85be032c18419fa245d7d0743264c70bd0592c +Subproject commit b70652625a7e87f277e8dc5c0b08588e61ee95ff diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b9064..964ea4a0d39 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -200,7 +200,7 @@ def _prep_variational_yaml(self: Task) -> None: # Add the things to the envconfig in order to template JCB files envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs - envconfig_jcb['nmem_ens'] = self.task_config.NMEM_ENS + envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion' if self.task_config.NMEM_ENS > 3: envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion' diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index b316b221f93..dee992dc0e3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -14,6 +14,8 @@ Executable, Task) +from pygfs.jedi import Jedi + logger = getLogger(__name__.split('.')[-1]) @@ -90,7 +92,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_setcorscales self.jedi['soca_setcorscales'] = Jedi(AttrDict( @@ -103,7 +105,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_parameters_diffusion_hz self.jedi['soca_parameters_diffusion_hz'] = Jedi(AttrDict( @@ -116,7 +118,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_ensb self.jedi['soca_ensb'] = Jedi(AttrDict( @@ -129,7 +131,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) # soca_ensweights self.jedi['soca_ensb'] = Jedi(AttrDict( @@ -142,7 +144,7 @@ def __init__(self, config): 'jcb_algo_yaml': None, 'jedi_args': None } - ) + )) @logit(logger) def initialize(self: Task) -> None: @@ -172,8 +174,8 @@ def initialize(self: Task) -> None: FileHandler(bkg_list).sync() # initialize vtscales python script - vtscales_config = self.render_jcb(task_config, 'vtscales') - save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml') + vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') + save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() @@ -184,7 +186,7 @@ def initialize(self: Task) -> None: self.jedi['soca_setcorscales'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'.initialize(self.task_config) + self.jedi['soca_ensb'].initialize(self.task_config) self.jedi['soca_ensweights'].initialize(self.task_config) # stage ensemble members for the hybrid background error From 694ad55b656761992f6ff307a5f2ee903d64ffbb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 02:27:29 +0000 Subject: [PATCH 065/142] Saving progress --- scripts/exglobal_marinebmat.py | 2 +- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_bmat.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_marinebmat.py b/scripts/exglobal_marinebmat.py index e285e646ac0..fd8770c18e7 100755 --- a/scripts/exglobal_marinebmat.py +++ b/scripts/exglobal_marinebmat.py @@ -20,5 +20,5 @@ # Create an instance of the MarineBMat task marineBMat = MarineBMat(config) marineBMat.initialize() - marineBMat.execute() + marineBMat.execute(config.APRUN_MARINEBMAT) marineBMat.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b70652625a7..52bc8d5ef7a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b70652625a7e87f277e8dc5c0b08588e61ee95ff +Subproject commit 52bc8d5ef7ad436e07da6baf5f15bbd8dac03104 diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index dee992dc0e3..a6384f38b9d 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -9,7 +9,7 @@ FileHandler, add_to_datetime, to_timedelta, chdir, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit, Executable, Task) @@ -40,12 +40,14 @@ def __init__(self, config): local_dict = AttrDict( { 'PARMsoca': os.path.join(self.task_config.PARMgfs, 'gdas', 'soca'), + 'CALC_SCALE_EXEC': _calc_scale_exec, 'MARINE_WINDOW_BEGIN': _window_begin, - 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, + 'MARINE_WINDOW_END': _window_end, + 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", 'ENSPERT_RELPATH': _enspert_relpath, - 'CALC_SCALE_EXEC': _calc_scale_exec, - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." } ) From cc4a0d80a46714b028d41d82d3e41ffb950a8804 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 13:10:59 +0000 Subject: [PATCH 066/142] Update GDAS hash to develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c71d3b17214..e024564f72e 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c71d3b17214007d59f91cc9c0988d4a8a279b22d +Subproject commit e024564f72e8b8b617e2a6a1cc06053e6dfb5786 From 9d8aa9382569daea83fb61e52f426c457e30de87 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 15 Oct 2024 15:03:14 +0000 Subject: [PATCH 067/142] Clean up exception handling --- ush/python/pygfs/jedi/jedi.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f873be10779..f4692e82d59 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -142,8 +142,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> if self.jedi_config.jcb_base_yaml: jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") + raise KeyError(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary if self.jedi_config.jcb_algo_yaml: @@ -157,9 +156,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> elif 'algorithm' in jcb_config: pass else: - logger.error(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") - logger.error(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + - "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + raise KeyError(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) From ebfcf41fee07bfb7aa7328d4a7cc7f3694d9e18a Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 16 Oct 2024 14:22:37 +0000 Subject: [PATCH 068/142] add ob space stat yaml and update Python script --- parm/stat/obspace_stat.yaml | 5 +++++ ush/python/pygfs/task/stat_analysis.py | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 parm/stat/obspace_stat.yaml diff --git a/parm/stat/obspace_stat.yaml b/parm/stat/obspace_stat.yaml new file mode 100644 index 00000000000..0b25906d84a --- /dev/null +++ b/parm/stat/obspace_stat.yaml @@ -0,0 +1,5 @@ +obs space: + viirs_npp: + path: path/to/gdasapp/parm/stat_yamls + viirs_n20: + path: path/to/gdasapp/parm/stat_yamls diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index afae6c34126..b3d3ed1ed1e 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -4,6 +4,7 @@ import glob import gzip import tarfile +import yaml from logging import getLogger from pprint import pformat from typing import Optional, Dict, Any @@ -65,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'OBSPACE_YAML': "/scratch1/NCEPDEV/da/Kevin.Dougherty/global-workflow/parm/stat/obspace_stat.yaml" } ) @@ -147,3 +149,22 @@ def initialize_analysis(self) -> None: with gzip.open(diagfile, 'rb') as f_in: with open(diagfile[:-3], 'wb') as f_out: f_out.write(f_in.read()) + + # Get list of .nc4 files + obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + + for path in obs_space_paths: + filename = os.path.basename(path) + obspace = '_'.join(filename.split('_')[1:3]) + + # Load g-w obs space intermediate yaml file + with open(self.task_config.OBSPACE_YAML, 'r') as yaml_file: + parsed_yaml_file = yaml.safe_load(yaml_file) + + obs_space_template = parsed_yaml_file['obs space'][obspace]['path'] + + # Load specific GDASApp yaml file + with open(obs_space_template, 'r') as obspace_yaml: + parsed_obspace_yaml_file = yaml.safe_load(obspace_yaml) + + print("open obspace yaml and do stuff") From b83ff6e59779cc68d7e143d2404891b717ff510f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 16 Oct 2024 19:18:54 +0000 Subject: [PATCH 069/142] Save progress --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_bmat.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 52bc8d5ef7a..9b2eabd1087 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 52bc8d5ef7ad436e07da6baf5f15bbd8dac03104 +Subproject commit 9b2eabd10879357c3354623306b2d516becdab03 diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index a6384f38b9d..748197a02cd 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -182,7 +182,8 @@ def initialize(self: Task) -> None: os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() # initialize JEDI applications - self.jedi['gridgen'].initialize(self.task_config) + self.jedi['gridgen'].initialize(self.task_config) + logger.error('foobar') # Test self.jedi['soca_diagb'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) @@ -221,10 +222,11 @@ def execute(self, aprun_cmd: str) -> None: This method will generate the full B-matrix according to the configuration. """ self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged - self.execute_vtscales() - self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi['soca_diagb'].execute(aprun_cmd) self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.execute_vtscales() + self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts From ac83f07335968619197b17bce2ebdf8283f0d981 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 17 Oct 2024 13:08:33 +0000 Subject: [PATCH 070/142] update paths --- parm/stat/obspace_stat.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/stat/obspace_stat.yaml b/parm/stat/obspace_stat.yaml index 0b25906d84a..7ae55b6770d 100644 --- a/parm/stat/obspace_stat.yaml +++ b/parm/stat/obspace_stat.yaml @@ -1,5 +1,5 @@ obs space: viirs_npp: - path: path/to/gdasapp/parm/stat_yamls + path: {FIXgdas}/parm/stat/viirs_npp_template.yaml viirs_n20: - path: path/to/gdasapp/parm/stat_yamls + path: {FIXgdas}/parm/stat/viirs_n20_template.yaml From 1ebc6e5177985a82c4c56d851aa270eb558c4c21 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 17 Oct 2024 13:11:11 +0000 Subject: [PATCH 071/142] pycodestyle --- ush/python/pygfs/task/stat_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index b3d3ed1ed1e..634e4bcab3f 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -152,7 +152,7 @@ def initialize_analysis(self) -> None: # Get list of .nc4 files obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) - + for path in obs_space_paths: filename = os.path.basename(path) obspace = '_'.join(filename.split('_')[1:3]) From 00adfeadda43450713e3604d44ec6b17cff82d46 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Oct 2024 13:03:47 +0000 Subject: [PATCH 072/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e024564f72e..93e7ec60bbc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e024564f72e8b8b617e2a6a1cc06053e6dfb5786 +Subproject commit 93e7ec60bbc354a3db42d174eb59f8ed1a170f48 From 3c98b82b4e6ba05a5cbe9187f15e6d86fd47b961 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Oct 2024 13:28:45 +0000 Subject: [PATCH 073/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 9b2eabd1087..136bcc09a49 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 9b2eabd10879357c3354623306b2d516becdab03 +Subproject commit 136bcc09a4994b7b15dd67aa83ae26543c7f9fae From 4dfbbb717c6d3deab23670990d0da815a10f6405 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 23 Oct 2024 15:41:26 +0000 Subject: [PATCH 074/142] adding to keep track of paths --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18f..7f342751d0c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 7f342751d0c51332f8151a5d7670fc0e3658ace4 From e79353f1729088480849a169d190a262d32bc2f4 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 23 Oct 2024 15:44:22 +0000 Subject: [PATCH 075/142] Revert "adding to keep track of paths" This reverts commit 4dfbbb717c6d3deab23670990d0da815a10f6405. --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7f342751d0c..faa95efb18f 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7f342751d0c51332f8151a5d7670fc0e3658ace4 +Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 From cc74aa4be96e87041206bc6f703d8278728b99ad Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 15:59:22 +0000 Subject: [PATCH 076/142] Bugfixes --- parm/config/gfs/config.marineanl | 2 +- ush/python/pygfs/task/marine_bmat.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index cf056387ca8..273dec91787 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -30,7 +30,7 @@ export JCB_ALGO_ENSWEIGHTS="soca_ensweights" export JEDIEXE_GRIDGEN="${EXECgfs}/gdas_soca_gridgen.x" export JEDIEXE_DIAGB="${EXECgfs}/gdas_soca_diagb.x" -export JEDIEXE_SETCORSCALES="${EXECgfs}/soca_setcorscales.x" +export JEDIEXE_SETCORSCALES="${EXECgfs}/gdas_soca_setcorscales.x" export JEDIEXE_PARAMETERS_DIFFUSION_HZ="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" export JEDIEXE_PARAMETERS_DIFFUSION_VT="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" export JEDIEXE_ENSB="${EXECgfs}/gdas_ens_handler.x" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 748197a02cd..69bdbfbedd3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -175,6 +175,11 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() + # stage the soca utility yamls (fields and ufo mapping yamls) + logger.info(f"Staging SOCA utility yaml files") + soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) + FileHandler(soca_utility_list).sync() + # initialize vtscales python script vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) @@ -183,7 +188,6 @@ def initialize(self: Task) -> None: # initialize JEDI applications self.jedi['gridgen'].initialize(self.task_config) - logger.error('foobar') # Test self.jedi['soca_diagb'].initialize(self.task_config) self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) From c45544580a069aadb1e332aec037cc52459e62a4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 15:59:47 +0000 Subject: [PATCH 077/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 136bcc09a49..a69df434d67 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 136bcc09a4994b7b15dd67aa83ae26543c7f9fae +Subproject commit a69df434d678419ca36b813f4f4c2836db5d28d5 From 2dc75c72aed7227d34793a1417aa16c25e07bb94 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 16:06:32 +0000 Subject: [PATCH 078/142] pynorms --- ush/python/pygfs/task/marine_bmat.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 69bdbfbedd3..f4490227ec9 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -56,11 +56,11 @@ def __init__(self, config): # Create dictionary of Jedi objects self.jedi = AttrDict() - + # gridgen self.jedi['gridgen'] = Jedi(AttrDict( { - 'yaml_name': 'gridgen', + 'yaml_name': 'gridgen', 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE_GRIDGEN, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, @@ -95,7 +95,7 @@ def __init__(self, config): 'jedi_args': None } )) - + # soca_setcorscales self.jedi['soca_setcorscales'] = Jedi(AttrDict( { @@ -134,7 +134,7 @@ def __init__(self, config): 'jedi_args': None } )) - + # soca_ensweights self.jedi['soca_ensb'] = Jedi(AttrDict( { @@ -175,23 +175,23 @@ def initialize(self: Task) -> None: bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() - # stage the soca utility yamls (fields and ufo mapping yamls) + # stage the soca utility yamls (fields and ufo mapping yamls) logger.info(f"Staging SOCA utility yaml files") soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) FileHandler(soca_utility_list).sync() - + # initialize vtscales python script vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() - + # initialize JEDI applications self.jedi['gridgen'].initialize(self.task_config) - self.jedi['soca_diagb'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) + self.jedi['soca_diagb'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi['soca_setcorscales'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) + self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.jedi['soca_ensb'].initialize(self.task_config) self.jedi['soca_ensweights'].initialize(self.task_config) @@ -200,7 +200,7 @@ def initialize(self: Task) -> None: if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) - + # create the symbolic link to the static B-matrix directory link_target = os.path.join(self.task_config.DATAstaticb) link_name = os.path.join(self.task_config.DATA, 'staticb') @@ -212,28 +212,28 @@ def initialize(self: Task) -> None: def execute_vtscales(self: Task) -> None: """Generate the vertical diffusion coefficients """ - # compute the vertical correlation scales based on the MLD + # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) - + mdau.run(exec_cmd) + @logit(logger) def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. """ - self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged self.jedi['soca_diagb'].execute(aprun_cmd) - self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales self.execute_vtscales() self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts - self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From ac3d385cceab8c670c48c5517ff627bf10cc0e59 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 16:18:36 +0000 Subject: [PATCH 079/142] Remove comment --- parm/config/gfs/config.marineanl | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index 273dec91787..fc3719f1069 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -17,7 +17,6 @@ export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_me export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" -#export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" export JCB_ALGO_GRIDGEN="soca_gridgen" export JCB_ALGO_DIAGB="soca_diagb" From 4cd585c218dd9af0a6790f33f700e1522e0d5cea Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:20:52 +0000 Subject: [PATCH 080/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a69df434d67..b8ce7e16662 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a69df434d678419ca36b813f4f4c2836db5d28d5 +Subproject commit b8ce7e16662eef62eec57244322aeac3d7e74d9a From 7e4defade6ed7cdaccb5b674c7a694ddf5717d5f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:24:10 +0000 Subject: [PATCH 081/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b8ce7e16662..a00b9191ce5 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b8ce7e16662eef62eec57244322aeac3d7e74d9a +Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 From 1c0fcf1fd5b43edaf0e79057298ab46408f897ce Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 17:38:48 +0000 Subject: [PATCH 082/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 93e7ec60bbc..f1222ec3792 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 93e7ec60bbc354a3db42d174eb59f8ed1a170f48 +Subproject commit f1222ec37924d567a8d935f0cad1a6a705045e4e From 97bf8e83c1693ddd78d53b8102f593958478eee1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 18:08:49 +0000 Subject: [PATCH 083/142] Add comment blocks to methods --- ush/python/pygfs/task/marine_bmat.py | 62 +++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index f4490227ec9..59515bd15ae 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -25,7 +25,24 @@ class MarineBMat(Task): """ @logit(logger, name="MarineBMat") def __init__(self, config): + """Constructor for marine B-matrix task + + This method will construct the marine B-matrix task object + This includes: + - extending the task_config AttrDict to include parameters required for this task + - instantiate the Jedi attribute objects + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + + Returns + ---------- + None + """ super().__init__(config) + _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, @@ -158,8 +175,17 @@ def initialize(self: Task) -> None: - staging SOCA fix files - staging static ensemble members (optional) - staging ensemble members (optional) - - generating the YAML files for the JEDI and GDASApp executables + - initializing the soca_vtscales Python script + - initializing the JEDI applications - creating output directories + + Parameters + ---------- + None + + Returns + ---------- + None """ # stage fix files @@ -210,7 +236,20 @@ def initialize(self: Task) -> None: @logit(logger) def execute_vtscales(self: Task) -> None: - """Generate the vertical diffusion coefficients + """Execute vertical diffusion coefficients generator + + This method will execute a Python script which generatres the vertical diffusion coefficients + This includes: + - constructing the executable object + - running the executable object + + Parameters + ---------- + None + + Returns + ---------- + None """ # compute the vertical correlation scales based on the MLD exec_cmd = Executable("python") @@ -224,7 +263,19 @@ def execute(self, aprun_cmd: str) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. + This includes: + - running all JEDI application and Python scripts required to generate the B-matrix + + Parameters + ---------- + aprun_cmd: str + String comprising the run command for the JEDI executable. + + Returns + ---------- + None """ + self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged self.jedi['soca_diagb'].execute(aprun_cmd) self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales @@ -246,6 +297,13 @@ def finalize(self: Task) -> None: - keep the re-balanced ensemble perturbation files in DATAenspert - ... + Parameters + ---------- + None + + Returns + ---------- + None """ # Copy the soca grid if it was created grid_file = os.path.join(self.task_config.DATA, 'soca_gridspec.nc') From 66129a297e695bf4a205f6fe662016ea79f54300 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 18:17:29 +0000 Subject: [PATCH 084/142] pynorms --- ush/python/pygfs/task/marine_bmat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 59515bd15ae..ca8730a7065 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -31,7 +31,7 @@ def __init__(self, config): This includes: - extending the task_config AttrDict to include parameters required for this task - instantiate the Jedi attribute objects - + Parameters ---------- config: Dict @@ -42,7 +42,7 @@ def __init__(self, config): None """ super().__init__(config) - + _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, From 74e5efa27034f01971892ae5297dd4093180a2f7 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 20:14:07 +0000 Subject: [PATCH 085/142] Make jcb rendering method save copies of jcb_config and task_ocnfig --- ush/python/pygfs/jedi/jedi.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f4692e82d59..f4e9ac67c41 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -56,6 +56,14 @@ def __init__(self, config) -> None: # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() + # Create a dictionary of dictionaries for saving copies of the jcb_config + # associated with each algorithm + self._jcb_configs = AttrDict() + + # Create a dictionary of dictionaries for saving copies of the task_config + # used to render each JCB template + self._task_configs_for_jcb = AttrDict() + @logit(logger) def initialize(self, task_config: AttrDict) -> None: """Initialize JEDI application @@ -156,9 +164,13 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> elif 'algorithm' in jcb_config: pass else: - raise KeyError(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + raise Exception(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + # Save copies of the task_config and jcb_config used to render this JCB template + self._task_configs_for_jcb[jcb_config['algorithm']] = task_config.deepcopy() + self._jcb_configs[jcb_config['algorithm']] = jcb_config.deepcopy() + # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) From 371c4c097c04436b6af8a9675cdea8fb86f64e11 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 23 Oct 2024 21:06:47 +0000 Subject: [PATCH 086/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index f4e9ac67c41..67b42133058 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -58,11 +58,11 @@ def __init__(self, config) -> None: # Create a dictionary of dictionaries for saving copies of the jcb_config # associated with each algorithm - self._jcb_configs = AttrDict() + self._jcb_config_dict = AttrDict() # Create a dictionary of dictionaries for saving copies of the task_config # used to render each JCB template - self._task_configs_for_jcb = AttrDict() + self._task_config_dict = AttrDict() @logit(logger) def initialize(self, task_config: AttrDict) -> None: @@ -165,15 +165,15 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> pass else: raise Exception(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + - "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") - # Save copies of the task_config and jcb_config used to render this JCB template - self._task_configs_for_jcb[jcb_config['algorithm']] = task_config.deepcopy() - self._jcb_configs[jcb_config['algorithm']] = jcb_config.deepcopy() - # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) + # Save copies of the task_config and jcb_config used to render this JCB template + self._task_config_dict[jcb_config['algorithm']] = task_config.deepcopy() + self._jcb_config_dict[jcb_config['algorithm']] = jcb_config.deepcopy() + return jedi_input_config @logit(logger) From 7db0c4bdc366c76a7bee69e9f4235c63dc306304 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 24 Oct 2024 14:43:29 +0000 Subject: [PATCH 087/142] update all changes in sorc --- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_utils.fd | 2 +- sorc/verif-global.fd | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a00b9191ce5..7f342751d0c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 +Subproject commit 7f342751d0c51332f8151a5d7670fc0e3658ace4 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index a00cc0949e2..bd8f13d8677 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit a00cc0949e2f901e73b58d54834517743916c69a +Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 9f44c8798c2..529bb796bea 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 9f44c8798c2087aca06df8f629699632e57df431 +Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 278ee629e87..e1f9f21af16 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 +Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index a6ea311e5c8..9382fd01c2a 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014 +Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f63..3ef2e6bd725 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index b2ee80cac79..92904d2c431 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea +Subproject commit 92904d2c431969345968f74e676717057ec0042a From f403b5cc8758b1712feb088c464ca6d32c127eab Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 24 Oct 2024 14:44:18 +0000 Subject: [PATCH 088/142] update all changes in sorc --- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_utils.fd | 2 +- sorc/verif-global.fd | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a00b9191ce5..7f342751d0c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 +Subproject commit 7f342751d0c51332f8151a5d7670fc0e3658ace4 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index a00cc0949e2..bd8f13d8677 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit a00cc0949e2f901e73b58d54834517743916c69a +Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 9f44c8798c2..529bb796bea 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 9f44c8798c2087aca06df8f629699632e57df431 +Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 278ee629e87..e1f9f21af16 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 +Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index a6ea311e5c8..9382fd01c2a 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014 +Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f63..3ef2e6bd725 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index b2ee80cac79..92904d2c431 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea +Subproject commit 92904d2c431969345968f74e676717057ec0042a From f710d0ffbec0be6904c7c7985f4d6a4444b9ef17 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:51:48 +0000 Subject: [PATCH 089/142] Fix to make sure bkg_list.yaml isn't loaded directly in marine JCB base --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a00b9191ce5..1660f1ae50d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a00b9191ce5253e984539024ab21ab3c44ece7f4 +Subproject commit 1660f1ae50da47a1f6bc0c31c625bd3d185d93ca diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 964ea4a0d39..f9fdbb61bbc 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -15,7 +15,7 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_timedelta, to_YMD, - parse_j2yaml, + parse_j2yaml, parse_yaml, logit, Executable, Task, @@ -192,7 +192,7 @@ def _prep_variational_yaml(self: Task) -> None: mdau.gen_bkg_list(bkg_path='./bkg', window_begin=self.task_config.MARINE_WINDOW_BEGIN, yaml_name='bkg_list.yaml') - + # Make a copy of the env config before modifying to avoid breaking something else envconfig_jcb = copy.deepcopy(self.task_config) logger.info(f"---------------- Prepare the yaml configuration") @@ -225,7 +225,7 @@ def _prep_variational_yaml(self: Task) -> None: jcb_algo_config = YAMLFile(path=jcb_algo_yaml) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) - + # Override base with the application specific config jcb_config = {**jcb_base_config, **jcb_algo_config} @@ -233,6 +233,9 @@ def _prep_variational_yaml(self: Task) -> None: jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') + # Current hack so that this is not done directly in the JCB base yaml + jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) From 215188aeedc246ecab77e63d5a4ba45cb1971f20 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:53:38 +0000 Subject: [PATCH 090/142] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index f9fdbb61bbc..0cffb3fdcce 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -192,7 +192,7 @@ def _prep_variational_yaml(self: Task) -> None: mdau.gen_bkg_list(bkg_path='./bkg', window_begin=self.task_config.MARINE_WINDOW_BEGIN, yaml_name='bkg_list.yaml') - + # Make a copy of the env config before modifying to avoid breaking something else envconfig_jcb = copy.deepcopy(self.task_config) logger.info(f"---------------- Prepare the yaml configuration") From cbf1c6f61fca90c5e5cdb5761b211862df8ef558 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 15:54:31 +0000 Subject: [PATCH 091/142] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 0cffb3fdcce..71aa6ea67f7 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -225,7 +225,7 @@ def _prep_variational_yaml(self: Task) -> None: jcb_algo_config = YAMLFile(path=jcb_algo_yaml) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) - + # Override base with the application specific config jcb_config = {**jcb_base_config, **jcb_algo_config} From 54223bf620cc4b4a16b33957b74aa0ed207407c2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 18:18:11 +0000 Subject: [PATCH 092/142] pynorms --- ush/python/pygfs/task/marine_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 71aa6ea67f7..b311c6874f5 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -235,7 +235,7 @@ def _prep_variational_yaml(self: Task) -> None: # Current hack so that this is not done directly in the JCB base yaml jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') - + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) From abdbe31115b34857970c233d67241a2989bc21b5 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 24 Oct 2024 19:27:01 +0000 Subject: [PATCH 093/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 1660f1ae50d..12643982fad 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 1660f1ae50da47a1f6bc0c31c625bd3d185d93ca +Subproject commit 12643982fade77a47e573da41a90198457e356be From cbe619ca5f9184243068ca692fd4f5d422df61bb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 25 Oct 2024 12:50:10 +0000 Subject: [PATCH 094/142] Adding back accidentally deleted line --- parm/config/gfs/config.marineanl | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index fc3719f1069..8aacc521354 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -15,6 +15,7 @@ export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2 export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" +export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" From 272bced697216924d18f078e538a8116df18287d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 28 Oct 2024 14:10:54 +0000 Subject: [PATCH 095/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 12643982fad..9be49de2a58 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 12643982fade77a47e573da41a90198457e356be +Subproject commit 9be49de2a5833ee6e70ee3469e7aa019649b22d3 From a3f9877d9335b2a71af7482e5d87cb487e47688c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 29 Oct 2024 18:21:19 +0000 Subject: [PATCH 096/142] update scripts for new JEDI scripts --- scripts/exglobal_analysis_stats.py | 5 +-- ush/python/pygfs/task/stat_analysis.py | 60 +++++++++----------------- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index db3a298d383..c9dda5a1cbe 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -22,8 +22,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - StatAnl = StatAnalysis(config, 'statanlvar') + StatAnl = StatAnalysis(config) # Initialize JEDI variational analysis - StatAnl.initialize_analysis() - StatAnl.initialize_jedi() + StatAnl.initialize() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 634e4bcab3f..980b9d63f1f 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -25,21 +25,19 @@ class StatAnalysis(Task): Class for JEDI-based global stat analysis tasks """ @logit(logger, name="StatAnalysis") - def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any]): """ Constructor global stat analysis task This method will construct a global stat analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - - instantiate the Jedi attribute object + - instantiate the Jedi attribute objects Parameters ---------- config: Dict dictionary object containing task configuration - yaml_name: str, optional - name of YAML file for JEDI configuration Returns ---------- @@ -50,7 +48,6 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): _res = int(self.task_config.CASE[1:]) # _res_anl = int(self.task_config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - print(_window_begin) # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -74,47 +71,30 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create JEDI object - self.jedi = Jedi(self.task_config, yaml_name) + # Create dictionary of JEDI objects + self.jedi = AttrDict() - @logit(logger) - def initialize_jedi(self): - """Initialize JEDI application - - This method will initialize a JEDI application used in the global stat analysis. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) + # statanlvar + self.jedi['statanl'] = Jedi(AttrDict( + { + 'yaml_name': 'statanl', + 'rundir': self.task_config.DATA, + 'exe_src': self.task_config.JEDIEXE, + 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + 'jcb_algo': self.task_config.JCB_ALGO, + 'jcb_algo_yaml': None, #self.task_config.JCB_ALGO_YAML_VAR, + 'jedi_args': None #['fv3jedi', 'variational'] + } + )) @logit(logger) - def initialize_analysis(self) -> None: + def initialize(self) -> None: """ Initialize a global stat analysis This method will initialize a global stat analysis. This includes: + - initialize JEDI applications - copying stat files Parameters @@ -125,7 +105,9 @@ def initialize_analysis(self) -> None: ---------- None """ - super().initialize() + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi['statanl'].initialize(self.task_config) logger.info(f"Copying files to {self.task_config.DATA}/stats") From 39915e4250199f3837ccd511162b6291f15ed500 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 31 Oct 2024 17:39:27 +0000 Subject: [PATCH 097/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 9be49de2a58..3d8892c503f 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 9be49de2a5833ee6e70ee3469e7aa019649b22d3 +Subproject commit 3d8892c503f0b57abdbf1e6aec750cfd45e90722 From ef1c1e1168677c8eda75456d2ebe87ae75836605 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 31 Oct 2024 20:40:23 +0000 Subject: [PATCH 098/142] Move Jedi object initialization dictionaries into YAML files for AtmAnalysis and AtmEnsAnalysis --- parm/config/gfs/config.atmanl | 10 +-- parm/config/gfs/config.atmensanl | 15 ++--- parm/gdas/atmanl_jedi_config.yaml.j2 | 11 +++ parm/gdas/atmensanl_jedi_config.yaml.j2 | 23 +++++++ .../exglobal_atm_analysis_fv3_increment.py | 2 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_fv3_increment.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 4 +- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 30 ++++++++- ush/python/pygfs/task/atm_analysis.py | 38 ++--------- ush/python/pygfs/task/atmens_analysis.py | 67 ++----------------- 13 files changed, 89 insertions(+), 119 deletions(-) create mode 100644 parm/gdas/atmanl_jedi_config.yaml.j2 create mode 100644 parm/gdas/atmensanl_jedi_config.yaml.j2 diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index a2baadde7bc..b3474ec1a80 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -5,10 +5,6 @@ echo "BEGIN: config.atmanl" -export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ -export JCB_ALGO_FV3INC="fv3jedi_fv3inc_variational" - export STATICB_TYPE=@STATICB_TYPE@ export LOCALIZATION_TYPE="bump" export INTERP_METHOD='barycentric' @@ -22,19 +18,19 @@ else export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}" fi +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2" export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2" export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2" export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2" +export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ + export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE_VAR="${EXECgfs}/gdas.x" -export JEDIEXE_FV3INC="${EXECgfs}/fv3jedi_fv3inc.x" - echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 2c575258343..3dc29441f13 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -5,26 +5,21 @@ echo "BEGIN: config.atmensanl" -export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" - -export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ -export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ -export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ -export JCB_ALGO_FV3INC="fv3jedi_fv3inc_lgetkf" - export INTERP_METHOD='barycentric' +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2" export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2" +export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ +export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ +export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ + export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE_LETKF=${EXECgfs}/gdas.x -export JEDIEXE_FV3INC=${EXECgfs}/fv3jedi_fv3inc.x - echo "END: config.atmensanl" diff --git a/parm/gdas/atmanl_jedi_config.yaml.j2 b/parm/gdas/atmanl_jedi_config.yaml.j2 new file mode 100644 index 00000000000..f44db56e576 --- /dev/null +++ b/parm/gdas/atmanl_jedi_config.yaml.j2 @@ -0,0 +1,11 @@ +atmanlvar: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}' + jedi_args: ['fv3jedi', 'variational'] +atmanlfv3inc: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo: fv3jedi_fv3inc_variational diff --git a/parm/gdas/atmensanl_jedi_config.yaml.j2 b/parm/gdas/atmensanl_jedi_config.yaml.j2 new file mode 100644 index 00000000000..b274b90abbe --- /dev/null +++ b/parm/gdas/atmensanl_jedi_config.yaml.j2 @@ -0,0 +1,23 @@ +atmensanlobs: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}' + jedi_args: ['fv3jedi', 'localensembleda'] +atmensanlsol: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}' + jedi_args: ['fv3jedi', 'localensembleda'] +atmensanlfv3inc: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo: fv3jedi_fv3inc_lgetkf +atmensanlletkf: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}' + jedi_args: ['fv3jedi', 'localensembleda'] diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 39a9b4e25a2..ba7dcf3cf46 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi['atmanlfv3inc'].execute(config.APRUN_ATMANLFV3INC) + AtmAnl.jedi_dict['atmanlfv3inc'].execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 21d99da3a24..a9605c03046 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi['atmanlvar'].execute(config.APRUN_ATMANLVAR) + AtmAnl.jedi_dict['atmanlvar'].execute(config.APRUN_ATMANLVAR) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 288c043adce..e48a8d4555f 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi['atmensanlfv3inc'].execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.jedi_dict['atmensanlfv3inc'].execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index c0516003d66..423a8e7fd75 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -23,7 +23,7 @@ # Initalize JEDI ensemble DA application # Note: This is normally done in AtmEnsAnl.initialize(), but that method now # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi['atmensanlletkf'].initialize(AtmEnsAnl.task_config) + AtmEnsAnl.jedi_dict['atmensanlletkf'].initialize(AtmEnsAnl.task_config) # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi['atmensanlletkf'].execute(config.APRUN_ATMENSANLLETKF) + AtmEnsAnl.jedi_dict['atmensanlletkf'].execute(config.APRUN_ATMENSANLLETKF) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 84b1f280960..d1d18e58372 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi['atmensanlobs'].execute(config.APRUN_ATMENSANLOBS) + AtmEnsAnl.jedi_dict['atmensanlobs'].execute(config.APRUN_ATMENSANLOBS) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index e6c112f97fc..617e63726f0 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi['atmensanlsol'].execute(config.APRUN_ATMENSANLSOL) + AtmEnsAnl.jedi_dict['atmensanlsol'].execute(config.APRUN_ATMENSANLSOL) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 67b42133058..bc319e10ae6 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -14,11 +14,13 @@ logger = getLogger(__name__.split('.')[-1]) +jedi_key_list = ['rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] class Jedi: """ Class for initializing and executing JEDI applications """ + @logit(logger, name="Jedi") def __init__(self, config) -> None: """Constructor for JEDI objects @@ -38,8 +40,9 @@ def __init__(self, config) -> None: None """ - _key_list = ['yaml_name', 'rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] - for key in _key_list: + if 'yaml_name' not in config: + raise KeyError(f"Key 'yaml_name' not found in the nested dictionary") + for key in jedi_key_list: if key not in config: raise KeyError(f"Key '{key}' not found in the nested dictionary") @@ -195,6 +198,29 @@ def link_exe(self) -> None: if not os.path.exists(self.jedi_config.exe): os.symlink(self.jedi_config.exe_src, self.jedi_config.exe) + @staticmethod + @logit(logger) + def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict): + # Initialize dictionary of Jedi objects + jedi_dict = AttrDict() + + # Parse J2-YAML file for dictionary of JEDI configuration dictionaries + jedi_config_dict = parse_j2yaml(jedi_config_yaml, task_config) + + # Loop through dictionary of Jedi configuration dictionaries + for yaml_name in jedi_config_dict: + # Make sure all required keys present or set to None + jedi_config_dict[yaml_name]['yaml_name'] = yaml_name + for key in jedi_key_list: + if key not in jedi_config_dict[yaml_name]: + jedi_config_dict[yaml_name][key] = None + + # Construct JEDI object + jedi_dict[yaml_name] = Jedi(jedi_config_dict[yaml_name]) + + # Return dictionary of JEDI objects + return jedi_dict + @staticmethod @logit(logger) def remove_redundant(input_list: List) -> List: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5ddb678036c..78f430acb35 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -70,34 +70,8 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of JEDI objects - self.jedi = AttrDict() - - # atmanlvar - self.jedi['atmanlvar'] = Jedi(AttrDict( - { - 'yaml_name': 'atmanlvar', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_VAR, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_VAR, - 'jedi_args': ['fv3jedi', 'variational'] - } - )) - - # atmanlfv3inc - self.jedi['atmanlfv3inc'] = Jedi(AttrDict( - { - 'yaml_name': 'atmanlfv3inc', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) + # Create dictionary of Jedi objects + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) @logit(logger) def initialize(self) -> None: @@ -125,21 +99,21 @@ def initialize(self) -> None: # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") - self.jedi['atmanlvar'].initialize(self.task_config) + self.jedi_dict['atmanlvar'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi['atmanlfv3inc'].initialize(self.task_config) + self.jedi_dict['atmanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_dict['atmanlvar'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi_dict['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 46072dfe8b1..95b04a1aefe 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -72,62 +72,7 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of JEDI objects - self.jedi = AttrDict() - - # atmensanlobs - self.jedi['atmensanlobs'] = Jedi(AttrDict( - { - 'yaml_name': 'atmensanlobs', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_OBS, - 'jedi_args': ['fv3jedi', 'localensembleda'] - } - )) - - # atmensanlsol - self.jedi['atmensanlsol'] = Jedi(AttrDict( - { - 'yaml_name': 'atmensanlsol', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_SOL, - 'jedi_args': ['fv3jedi', 'localensembleda'] - } - )) - - # atmensanlfv3inc - self.jedi['atmensanlfv3inc'] = Jedi(AttrDict( - { - 'yaml_name': 'atmensanlfv3inc', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_FV3INC, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_FV3INC, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # Note: Since we now use the split observer-solvers, the following - # is only for testing. - - # atmensanlletkf - self.jedi['atmensanlletkf'] = Jedi(AttrDict( - { - 'yaml_name': 'atmensanlletkf', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_LETKF, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': None, - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML_LETKF, - 'jedi_args': ['fv3jedi', 'localensembleda'] - } - )) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) @logit(logger) def initialize(self) -> None: @@ -154,25 +99,25 @@ def initialize(self) -> None: # initialize JEDI LETKF observer application logger.info(f"Initializing JEDI LETKF observer application") - self.jedi['atmensanlobs'].initialize(self.task_config) + self.jedi_dict['atmensanlobs'].initialize(self.task_config) # initialize JEDI LETKF solver application logger.info(f"Initializing JEDI LETKF solver application") - self.jedi['atmensanlsol'].initialize(self.task_config) + self.jedi_dict['atmensanlsol'].initialize(self.task_config) # initialize JEDI FV3 increment conversion application logger.info(f"Initializing JEDI FV3 increment conversion application") - self.jedi['atmensanlfv3inc'].initialize(self.task_config) + self.jedi_dict['atmensanlfv3inc'].initialize(self.task_config) # stage observations logger.info(f"Staging list of observation files") - obs_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_obs_staging') + obs_dict = self.jedi_dict['atmensanlobs'].render_jcb(self.task_config, 'atm_obs_staging') FileHandler(obs_dict).sync() logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi['atmensanlobs'].render_jcb(self.task_config, 'atm_bias_staging') + bias_dict = self.jedi_dict['atmensanlobs'].render_jcb(self.task_config, 'atm_bias_staging') bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 5fa0ed179c48d11efd84bf68a8d4f237ef5f2b80 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 31 Oct 2024 23:04:28 +0000 Subject: [PATCH 099/142] Move marine bmat JEDI config to YAML file --- parm/config/gfs/config.marineanl | 18 +--- parm/gdas/soca_bmat_jedi_config.yaml.j2 | 35 +++++++ ush/python/pygfs/task/marine_bmat.py | 124 ++++-------------------- 3 files changed, 53 insertions(+), 124 deletions(-) create mode 100644 parm/gdas/soca_bmat_jedi_config.yaml.j2 diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index 8aacc521354..c850744fff7 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -17,23 +17,7 @@ export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_me export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" -export JCB_ALGO_GRIDGEN="soca_gridgen" -export JCB_ALGO_DIAGB="soca_diagb" -export JCB_ALGO_SETCORSCALES="soca_setcorscales" -export JCB_ALGO_PARAMETERS_DIFFUSION_HZ="soca_parameters_diffusion_hz" -export JCB_ALGO_VTSCALES="soca_vtscales" -export JCB_ALGO_PARAMETERS_DIFFUSION_VT="soca_parameters_diffusion_vt" -export JCB_ALGO_ENSB="soca_ensb" -export JCB_ALGO_ENSWEIGHTS="soca_ensweights" - -export JEDIEXE_GRIDGEN="${EXECgfs}/gdas_soca_gridgen.x" -export JEDIEXE_DIAGB="${EXECgfs}/gdas_soca_diagb.x" -export JEDIEXE_SETCORSCALES="${EXECgfs}/gdas_soca_setcorscales.x" -export JEDIEXE_PARAMETERS_DIFFUSION_HZ="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" -export JEDIEXE_PARAMETERS_DIFFUSION_VT="${EXECgfs}/gdas_soca_error_covariance_toolbox.x" -export JEDIEXE_ENSB="${EXECgfs}/gdas_ens_handler.x" -export JEDIEXE_ENSWEIGHTS="${EXECgfs}/gdas_socahybridweights.x" - echo "END: config.marineanl" diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 new file mode 100644 index 00000000000..0ca8e7062a1 --- /dev/null +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -0,0 +1,35 @@ +gridgen: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_soca_gridgen.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_gridgen +soca_diagb: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_diagb +soca_parameters_diffusion_vt: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_parameters_diffusion_vt +soca_setcorscales: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_setcorscales +soca_parameters_diffusion_hz: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_parameters_diffusion_hz +soca_ensb: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_ens_handler.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_ensb +soca_ensweights: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_ensweights diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 31884770408..fd63cfb9336 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -63,6 +63,7 @@ def __init__(self, config): 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", 'ENSPERT_RELPATH': _enspert_relpath, + 'MOM6_LEVS': mdau.get_mom6_levels(str(self.task_config.OCNRES)), 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." } @@ -72,98 +73,7 @@ def __init__(self, config): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of Jedi objects - self.jedi = AttrDict() - - # gridgen - self.jedi['gridgen'] = Jedi(AttrDict( - { - 'yaml_name': 'gridgen', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_GRIDGEN, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_GRIDGEN, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_diagb - self.jedi['soca_diagb'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_diagb', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_DIAGB, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_DIAGB, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_parameters_diffusion_vt - self.jedi['soca_parameters_diffusion_vt'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_parameters_diffusion_vt', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_VT, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_VT, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_setcorscales - self.jedi['soca_setcorscales'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_setcorscales', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_SETCORSCALES, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_SETCORSCALES, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_parameters_diffusion_hz - self.jedi['soca_parameters_diffusion_hz'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_parameters_diffusion_hz', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_PARAMETERS_DIFFUSION_HZ, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_PARAMETERS_DIFFUSION_HZ, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_ensb - self.jedi['soca_ensb'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_ensb', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_ENSB, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_ENSB, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) - - # soca_ensweights - self.jedi['soca_ensb'] = Jedi(AttrDict( - { - 'yaml_name': 'soca_ensb', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE_ENSWEIGHTS, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO_ENSWEIGHTS, - 'jcb_algo_yaml': None, - 'jedi_args': None - } - )) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) @logit(logger) def initialize(self: Task) -> None: @@ -207,20 +117,20 @@ def initialize(self: Task) -> None: FileHandler(soca_utility_list).sync() # initialize vtscales python script - vtscales_config = self.jedi['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') + vtscales_config = self.jedi_dict['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() # initialize JEDI applications - self.jedi['gridgen'].initialize(self.task_config) - self.jedi['soca_diagb'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_vt'].initialize(self.task_config) - self.jedi['soca_setcorscales'].initialize(self.task_config) - self.jedi['soca_parameters_diffusion_hz'].initialize(self.task_config) + self.jedi_dict['gridgen'].initialize(self.task_config) + self.jedi_dict['soca_diagb'].initialize(self.task_config) + self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config) + self.jedi_dict['soca_setcorscales'].initialize(self.task_config) + self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'].initialize(self.task_config) - self.jedi['soca_ensweights'].initialize(self.task_config) + self.jedi_dict['soca_ensb'].initialize(self.task_config) + self.jedi_dict['soca_ensweights'].initialize(self.task_config) # stage ensemble members for the hybrid background error if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: @@ -276,15 +186,15 @@ def execute(self, aprun_cmd: str) -> None: None """ - self.jedi['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged - self.jedi['soca_diagb'].execute(aprun_cmd) - self.jedi['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged + self.jedi_dict['soca_diagb'].execute(aprun_cmd) + self.jedi_dict['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales self.execute_vtscales() - self.jedi['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi_dict['soca_parameters_diffusion_vt'].execute(aprun_cmd) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts - self.jedi['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi_dict['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi_dict['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From 5e8bd8e2b199c2fee5fdbd633b1372cd210ae0c3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 31 Oct 2024 23:09:16 +0000 Subject: [PATCH 100/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index bc319e10ae6..d3aa6dd984f 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -16,11 +16,12 @@ jedi_key_list = ['rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] + class Jedi: """ Class for initializing and executing JEDI applications """ - + @logit(logger, name="Jedi") def __init__(self, config) -> None: """Constructor for JEDI objects @@ -215,12 +216,12 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict): if key not in jedi_config_dict[yaml_name]: jedi_config_dict[yaml_name][key] = None - # Construct JEDI object + # Construct JEDI object jedi_dict[yaml_name] = Jedi(jedi_config_dict[yaml_name]) # Return dictionary of JEDI objects - return jedi_dict - + return jedi_dict + @staticmethod @logit(logger) def remove_redundant(input_list: List) -> List: From 6f311af4a7ad60924635a7b110696fd2d7f04013 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 31 Oct 2024 23:34:51 +0000 Subject: [PATCH 101/142] Minor change to retrigger GW CI --- ush/python/pygfs/task/marine_bmat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index fd63cfb9336..223c330f660 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -21,7 +21,7 @@ class MarineBMat(Task): """ - Class for global marine B-matrix tasks + Class for global marine B-matrix tasks. """ @logit(logger, name="MarineBMat") def __init__(self, config): From d7a58ccbff3576307b1085c8c43a086fc86549ea Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 1 Nov 2024 09:04:37 +0000 Subject: [PATCH 102/142] Address Walter's comments --- .../exglobal_atm_analysis_fv3_increment.py | 6 +- scripts/exglobal_atm_analysis_variational.py | 6 +- .../exglobal_atmens_analysis_fv3_increment.py | 6 +- scripts/exglobal_atmens_analysis_letkf.py | 12 ++- scripts/exglobal_atmens_analysis_obs.py | 8 +- scripts/exglobal_atmens_analysis_sol.py | 8 +- scripts/exglobal_marinebmat.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 30 +++++++ ush/python/pygfs/task/atmens_analysis.py | 83 +++++++++++++++++++ ush/python/pygfs/task/marine_bmat.py | 19 ++--- 10 files changed, 145 insertions(+), 35 deletions(-) diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index ba7dcf3cf46..eba7cc1cd1f 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the execute method of its Jedi -# object attribute +# and runs the execute method which runs the JEDI +# FV3 increment converter import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.jedi_dict['atmanlfv3inc'].execute(config.APRUN_ATMANLFV3INC) + AtmAnl.execute_fv3inc() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index a9605c03046..c247a1b0a49 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_variational.py # This script creates an AtmAnalysis object -# and runs the execute method of its Jedi object attribute -# which executes the global atm variational analysis +# and runs the execute method which runs the JEDI +# variational analysis application import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.jedi_dict['atmanlvar'].execute(config.APRUN_ATMANLVAR) + AtmAnl.execute_var() diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index e48a8d4555f..4179ec3217d 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_fv3_increment.py # This script creates an AtmEnsAnalysis object -# and runs the execute method of its Jedi object attribute -# which convert the JEDI increment into an FV3 increment +# and runs the execute method which runs the JEDI +# FV3 increment converter application import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.jedi_dict['atmensanlfv3inc'].execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.execute_fv3inc() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 423a8e7fd75..e5497670c18 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the execute method of its Jedi object attribute -# which executes the global atm local ensemble analysis +# and initializes and runs the full JEDI LETKF +# application import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -20,10 +20,8 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - # Initalize JEDI ensemble DA application - # Note: This is normally done in AtmEnsAnl.initialize(), but that method now - # initializes the split observer-solver. This case is just for testing. - AtmEnsAnl.jedi_dict['atmensanlletkf'].initialize(AtmEnsAnl.task_config) + # Initalize JEDI full ensemble DA application + AtmEnsAnl.initialize_letkf() # Execute the JEDI ensemble DA analysis - AtmEnsAnl.jedi_dict['atmensanlletkf'].execute(config.APRUN_ATMENSANLLETKF) + AtmEnsAnl.execute_letkf() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index d1d18e58372..8fc3ddaa253 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_obs.py # This script creates an AtmEnsAnalysis object -# and runs the execute method of its Jedi object attribute -# which executes the global atm local ensemble analysis in observer mode +# and runs the execute method which runs the JEDI LETKF +# application in observer mode import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -20,5 +20,5 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.jedi_dict['atmensanlobs'].execute(config.APRUN_ATMENSANLOBS) + # Execute JEDI ensembler DA analysis in observer mode + AtmEnsAnl.execute_obs() diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index 617e63726f0..8f47da22553 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_sol.py # This script creates an AtmEnsAnalysis object -# and runs the execute method of its Jedi object attribute -# which executes the global atm local ensemble analysis in solver mode +# and runs the execute method which runs the JEDI LETKF +# application in solver mode import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -20,5 +20,5 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - # Initialize and execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.jedi_dict['atmensanlsol'].execute(config.APRUN_ATMENSANLSOL) + # Execute JEDI ensemble DA analysis in solver mode + AtmEnsAnl.execute_sol() diff --git a/scripts/exglobal_marinebmat.py b/scripts/exglobal_marinebmat.py index fd8770c18e7..e285e646ac0 100755 --- a/scripts/exglobal_marinebmat.py +++ b/scripts/exglobal_marinebmat.py @@ -20,5 +20,5 @@ # Create an instance of the MarineBMat task marineBMat = MarineBMat(config) marineBMat.initialize() - marineBMat.execute(config.APRUN_MARINEBMAT) + marineBMat.execute() marineBMat.finalize() diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 78f430acb35..d168767e220 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -163,6 +163,36 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() + @logit(logger) + def execute_var(self) -> None: + """Execute JEDI variational analysis application + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmanlvar'].execute(self.task_config.APRUN_ATMANLVAR) + + @logit(logger) + def execute_fv3inc(self) -> None: + """Execute JEDI FV3 increment converter application + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmanlfv3inc'].execute(self.task_config.APRUN_ATMANLFV3INC) + @logit(logger) def finalize(self) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 95b04a1aefe..a5857eb377b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -151,6 +151,87 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() + @logit(logger) + def initialize_letkf(self) -> None: + """Initialize a global atmens analysis + + Note: This would normally be done in AtmEnsAnalysis.initialize(), but that method + now initializes the split observer-solver. This case is just for testing. + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmensanlletkf'].initialize(self.task_config) + + @logit(logger) + def execute_obs(self) -> None: + """Execute JEDI LETKF application in observer mode + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmensanlobs'].execute(self.task_config.APRUN_ATMENSANLOBS) + + @logit(logger) + def execute_sol(self) -> None: + """Execute JEDI LETKF application in solver mode + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmensanlsol'].execute(self.task_config.APRUN_ATMENSANLSOL) + + @logit(logger) + def execute_fv3inc(self) -> None: + """Execute FV3 increment converter + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmensanlfv3inc'].execute(self.task_config.APRUN_ATMENSANLFV3INC) + + @logit(logger) + def execute_letkf(self) -> None: + """Execute full JEDI LETKF application + + Note: This is just for testing. Operationally, we plan to split the LETKF + into observer and solver modes. + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + self.jedi_dict['atmensanlletkf'].execute(self.task_config.APRUN_ATMENSANLLETKF) + @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis @@ -237,5 +318,7 @@ def finalize(self) -> None: } FileHandler(inc_copy).sync() + + def clean(self): super().clean() diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 223c330f660..14687c8604c 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -169,7 +169,7 @@ def execute_vtscales(self: Task) -> None: mdau.run(exec_cmd) @logit(logger) - def execute(self, aprun_cmd: str) -> None: + def execute(self) -> None: """Generate the full B-matrix This method will generate the full B-matrix according to the configuration. @@ -178,23 +178,22 @@ def execute(self, aprun_cmd: str) -> None: Parameters ---------- - aprun_cmd: str - String comprising the run command for the JEDI executable. + None Returns ---------- None """ - self.jedi_dict['gridgen'].execute(aprun_cmd) # TODO: This should be optional in case the geometry file was staged - self.jedi_dict['soca_diagb'].execute(aprun_cmd) - self.jedi_dict['soca_setcorscales'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi_dict['soca_parameters_diffusion_hz'].execute(aprun_cmd) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: This should be optional in case the geometry file was staged + self.jedi_dict['soca_diagb'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_setcorscales'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['soca_parameters_diffusion_hz'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: Make this optional once we've converged on an acceptable set of scales self.execute_vtscales() - self.jedi_dict['soca_parameters_diffusion_vt'].execute(aprun_cmd) + self.jedi_dict['soca_parameters_diffusion_vt'].execute(self.task_config.APRUN_MARINEBMAT) if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi_dict['soca_ensb'].execute(self.task_config) # TODO: refactor this from the old scripts - self.jedi_dict['soca_ensweights'].execute(self.task_config) # TODO: refactor this from the old scripts + self.jedi_dict['soca_ensb'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: refactor this from the old scripts + self.jedi_dict['soca_ensweights'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: From 0b9ca3b8ca8a11f3b2affdc12416b3baec209c87 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 1 Nov 2024 09:11:50 +0000 Subject: [PATCH 103/142] pynorms --- sorc/gfs_utils.fd | 2 +- ush/python/pygfs/task/atm_analysis.py | 8 ++++---- ush/python/pygfs/task/atmens_analysis.py | 8 +++----- ush/python/pygfs/task/marine_bmat.py | 14 +++++++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 856a42076a6..a00cc0949e2 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca +Subproject commit a00cc0949e2f901e73b58d54834517743916c69a diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index d168767e220..dc982229eef 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -175,7 +175,7 @@ def execute_var(self) -> None: ---------- None """ - + self.jedi_dict['atmanlvar'].execute(self.task_config.APRUN_ATMANLVAR) @logit(logger) @@ -190,9 +190,9 @@ def execute_fv3inc(self) -> None: ---------- None """ - - self.jedi_dict['atmanlfv3inc'].execute(self.task_config.APRUN_ATMANLFV3INC) - + + self.jedi_dict['atmanlfv3inc'].execute(self.task_config.APRUN_ATMANLFV3INC) + @logit(logger) def finalize(self) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a5857eb377b..c41ff554a46 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -168,7 +168,7 @@ def initialize_letkf(self) -> None: """ self.jedi_dict['atmensanlletkf'].initialize(self.task_config) - + @logit(logger) def execute_obs(self) -> None: """Execute JEDI LETKF application in observer mode @@ -213,7 +213,7 @@ def execute_fv3inc(self) -> None: """ self.jedi_dict['atmensanlfv3inc'].execute(self.task_config.APRUN_ATMENSANLFV3INC) - + @logit(logger) def execute_letkf(self) -> None: """Execute full JEDI LETKF application @@ -231,7 +231,7 @@ def execute_letkf(self) -> None: """ self.jedi_dict['atmensanlletkf'].execute(self.task_config.APRUN_ATMENSANLLETKF) - + @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis @@ -317,8 +317,6 @@ def finalize(self) -> None: 'copy': [[src, dest]] } FileHandler(inc_copy).sync() - - def clean(self): super().clean() diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 14687c8604c..b64de87225d 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -185,15 +185,19 @@ def execute(self) -> None: None """ - self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: This should be optional in case the geometry file was staged + # TODO: This should be optional in case the geometry file was staged + self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) self.jedi_dict['soca_diagb'].execute(self.task_config.APRUN_MARINEBMAT) - self.jedi_dict['soca_setcorscales'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi_dict['soca_parameters_diffusion_hz'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: Make this optional once we've converged on an acceptable set of scales + # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['soca_setcorscales'].execute(self.task_config.APRUN_MARINEBMAT) + # TODO: Make this optional once we've converged on an acceptable set of scales + self.jedi_dict['soca_parameters_diffusion_hz'].execute(self.task_config.APRUN_MARINEBMAT) self.execute_vtscales() self.jedi_dict['soca_parameters_diffusion_vt'].execute(self.task_config.APRUN_MARINEBMAT) + # TODO: refactor this from the old scripts if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi_dict['soca_ensb'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: refactor this from the old scripts - self.jedi_dict['soca_ensweights'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: refactor this from the old scripts + self.jedi_dict['soca_ensb'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_ensweights'].execute(self.task_config.APRUN_MARINEBMAT) @logit(logger) def finalize(self: Task) -> None: From 1e6a40f013791158800275d9fbbc6cc01489907f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 1 Nov 2024 09:15:07 +0000 Subject: [PATCH 104/142] pynorms --- ush/python/pygfs/task/atmens_analysis.py | 2 +- ush/python/pygfs/task/marine_bmat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index c41ff554a46..1b437ae46c4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -317,6 +317,6 @@ def finalize(self) -> None: 'copy': [[src, dest]] } FileHandler(inc_copy).sync() - + def clean(self): super().clean() diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index b64de87225d..564e79d80d3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -186,7 +186,7 @@ def execute(self) -> None: """ # TODO: This should be optional in case the geometry file was staged - self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) self.jedi_dict['soca_diagb'].execute(self.task_config.APRUN_MARINEBMAT) # TODO: Make this optional once we've converged on an acceptable set of scales self.jedi_dict['soca_setcorscales'].execute(self.task_config.APRUN_MARINEBMAT) From 874db59d1175c9830c59aece085ccd55e5927b72 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 1 Nov 2024 14:49:06 +0000 Subject: [PATCH 105/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 3d8892c503f..54e21096bbb 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 3d8892c503f0b57abdbf1e6aec750cfd45e90722 +Subproject commit 54e21096bbbea4d2a051149bf9c4073627c79622 From e179122a25f5bb7e91ac8060a3cf4db6255ad74c Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 3 Nov 2024 14:45:42 +0000 Subject: [PATCH 106/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 54e21096bbb..58e31a644f6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 54e21096bbbea4d2a051149bf9c4073627c79622 +Subproject commit 58e31a644f666ae70353d33d2ca6f41fdd4d6b21 From fbda568118f31086df3d128fa86f9c180c284a71 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 3 Nov 2024 14:48:17 +0000 Subject: [PATCH 107/142] Update gdas hash and revert accidentally changes gfs_utils hash --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index a00cc0949e2..856a42076a6 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit a00cc0949e2f901e73b58d54834517743916c69a +Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca From c18e2f270727930ce057899537f4b32c2fefc192 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 3 Nov 2024 15:58:38 +0000 Subject: [PATCH 108/142] Small, mostly asthetic changes --- parm/config/gfs/config.atmanl | 7 ++++--- parm/config/gfs/config.atmensanl | 11 ++++++----- parm/config/gfs/config.marineanl | 6 +++--- parm/gdas/atmanl_jedi_config.yaml.j2 | 4 ++-- parm/gdas/atmensanl_jedi_config.yaml.j2 | 8 ++++---- parm/gdas/soca_bmat_jedi_config.yaml.j2 | 14 +++++++------- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index b3474ec1a80..08fc3762f58 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -5,6 +5,10 @@ echo "BEGIN: config.atmanl" +export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" +export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2" + export STATICB_TYPE=@STATICB_TYPE@ export LOCALIZATION_TYPE="bump" export INTERP_METHOD='barycentric' @@ -18,15 +22,12 @@ else export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}" fi -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2" export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2" export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2" export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2" -export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ - export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 3dc29441f13..b57bac82a91 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -5,17 +5,18 @@ echo "BEGIN: config.atmensanl" +export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" +export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ +export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ +export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2" + export INTERP_METHOD='barycentric' -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2" export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2" -export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ -export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ -export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ - export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index c850744fff7..9a63580d8bb 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -5,6 +5,9 @@ echo "BEGIN: config.marineanl" +export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" + export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@ export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ @@ -17,7 +20,4 @@ export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_me export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" -export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" - echo "END: config.marineanl" diff --git a/parm/gdas/atmanl_jedi_config.yaml.j2 b/parm/gdas/atmanl_jedi_config.yaml.j2 index f44db56e576..0b845305ad1 100644 --- a/parm/gdas/atmanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmanl_jedi_config.yaml.j2 @@ -1,11 +1,11 @@ atmanlvar: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}' jedi_args: ['fv3jedi', 'variational'] atmanlfv3inc: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: fv3jedi_fv3inc_variational diff --git a/parm/gdas/atmensanl_jedi_config.yaml.j2 b/parm/gdas/atmensanl_jedi_config.yaml.j2 index b274b90abbe..816395a3404 100644 --- a/parm/gdas/atmensanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmensanl_jedi_config.yaml.j2 @@ -1,23 +1,23 @@ atmensanlobs: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}' jedi_args: ['fv3jedi', 'localensembleda'] atmensanlsol: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}' jedi_args: ['fv3jedi', 'localensembleda'] atmensanlfv3inc: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: fv3jedi_fv3inc_lgetkf atmensanlletkf: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}' jedi_args: ['fv3jedi', 'localensembleda'] diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index 0ca8e7062a1..055fd0bc0c7 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -1,35 +1,35 @@ gridgen: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_gridgen.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_gridgen soca_diagb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_diagb soca_parameters_diffusion_vt: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_parameters_diffusion_vt soca_setcorscales: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_setcorscales soca_parameters_diffusion_hz: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_parameters_diffusion_hz soca_ensb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_ens_handler.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_ensb soca_ensweights: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ JCB_BASE_YAML }}' jcb_algo: soca_ensweights From 482f2f3bc0742a07688d66e977ca599d41209772 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 04:30:46 +0000 Subject: [PATCH 109/142] Address requested changes --- parm/config/gfs/config.atmanl | 4 +- parm/config/gfs/config.atmensanl | 4 +- parm/config/gfs/config.marineanl | 1 - parm/gdas/atmanl_jedi_config.yaml.j2 | 8 +-- parm/gdas/atmensanl_jedi_config.yaml.j2 | 18 ++++--- parm/gdas/soca_bmat_jedi_config.yaml.j2 | 21 +++++--- .../exglobal_atm_analysis_fv3_increment.py | 2 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_fv3_increment.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 2 +- scripts/exglobal_atmens_analysis_obs.py | 2 +- scripts/exglobal_atmens_analysis_sol.py | 2 +- ush/python/pygfs/jedi/jedi.py | 51 +++++++++++-------- ush/python/pygfs/task/atm_analysis.py | 27 +++------- ush/python/pygfs/task/atmens_analysis.py | 14 ++--- ush/python/pygfs/task/marine_bmat.py | 18 ++++--- 16 files changed, 96 insertions(+), 82 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 08fc3762f58..1d700a479c8 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -5,9 +5,7 @@ echo "BEGIN: config.atmanl" -export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2" export STATICB_TYPE=@STATICB_TYPE@ export LOCALIZATION_TYPE="bump" @@ -24,6 +22,8 @@ fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" + +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2" export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2" export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2" export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index b57bac82a91..2726f655bdf 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -5,16 +5,16 @@ echo "BEGIN: config.atmensanl" -export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@ export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@ export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@ -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2" export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" + +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2" export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2" export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index 9a63580d8bb..0b55fa447db 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -5,7 +5,6 @@ echo "BEGIN: config.marineanl" -export JCB_BASE_YAML="${PARMgfs}/gdas/soca/marine-jcb-base.yaml" export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" diff --git a/parm/gdas/atmanl_jedi_config.yaml.j2 b/parm/gdas/atmanl_jedi_config.yaml.j2 index 0b845305ad1..52fe9c55a0b 100644 --- a/parm/gdas/atmanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmanl_jedi_config.yaml.j2 @@ -1,11 +1,13 @@ atmanlvar: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' - jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}' jedi_args: ['fv3jedi', 'variational'] + mpi_cmd: '{{ APRUN_ATMANLVAR }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}' atmanlfv3inc: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_ATMANLFV3INC }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo: fv3jedi_fv3inc_variational diff --git a/parm/gdas/atmensanl_jedi_config.yaml.j2 b/parm/gdas/atmensanl_jedi_config.yaml.j2 index 816395a3404..73ba96efb8b 100644 --- a/parm/gdas/atmensanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmensanl_jedi_config.yaml.j2 @@ -1,23 +1,27 @@ atmensanlobs: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' - jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}' jedi_args: ['fv3jedi', 'localensembleda'] + mpi_cmd: '{{ APRUN_ATMENSANLOBS }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}' atmensanlsol: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' - jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}' jedi_args: ['fv3jedi', 'localensembleda'] + mpi_cmd: '{{ APRUN_ATMENSANLSOL }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}' atmensanlfv3inc: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_ATMENSANLFV3INC }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo: fv3jedi_fv3inc_lgetkf atmensanlletkf: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' - jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}' jedi_args: ['fv3jedi', 'localensembleda'] + mpi_cmd: '{{ APRUN_ATMENSANLLETKF }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}' diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index 055fd0bc0c7..4e476d31177 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -1,35 +1,42 @@ gridgen: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_gridgen.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_gridgen soca_diagb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_diagb soca_parameters_diffusion_vt: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_vt soca_setcorscales: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_setcorscales soca_parameters_diffusion_hz: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_hz soca_ensb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_ens_handler.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_ensb soca_ensweights: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x' - jcb_base_yaml: '{{ JCB_BASE_YAML }}' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_ensweights diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index eba7cc1cd1f..c5a3e70943a 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Initialize and execute FV3 increment converter - AtmAnl.execute_fv3inc() + AtmAnl.execute('atmanlfv3inc') diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index c247a1b0a49..9ad121f76cf 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -21,4 +21,4 @@ AtmAnl = AtmAnalysis(config) # Execute JEDI variational analysis - AtmAnl.execute_var() + AtmAnl.execute('atmanlvar') diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 4179ec3217d..4506b280337 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute JEDI FV3 increment converter - AtmEnsAnl.execute_fv3inc() + AtmEnsAnl.execute('atmensanlfv3inc') diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index e5497670c18..dea9ace5b82 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -24,4 +24,4 @@ AtmEnsAnl.initialize_letkf() # Execute the JEDI ensemble DA analysis - AtmEnsAnl.execute_letkf() + AtmEnsAnl.execute('atmensanlletkf') diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 8fc3ddaa253..b09c67703f9 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.execute_obs() + AtmEnsAnl.execute('atmensanlobs') diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index 8f47da22553..85dc228a5ad 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -21,4 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config) # Execute JEDI ensemble DA analysis in solver mode - AtmEnsAnl.execute_sol() + AtmEnsAnl.execute('atmensanlsol') diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index d3aa6dd984f..c38cca10fc6 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -14,7 +14,7 @@ logger = getLogger(__name__.split('.')[-1]) -jedi_key_list = ['rundir', 'exe_src', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml', 'jedi_args'] +jedi_key_list = ['rundir', 'exe_src', 'jedi_args', 'mpi_cmd', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml'] class Jedi: @@ -41,6 +41,7 @@ def __init__(self, config) -> None: None """ + # Make sure input dictionary for Jedi class constructor has the required keys if 'yaml_name' not in config: raise KeyError(f"Key 'yaml_name' not found in the nested dictionary") for key in jedi_key_list: @@ -60,14 +61,6 @@ def __init__(self, config) -> None: # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() - # Create a dictionary of dictionaries for saving copies of the jcb_config - # associated with each algorithm - self._jcb_config_dict = AttrDict() - - # Create a dictionary of dictionaries for saving copies of the task_config - # used to render each JCB template - self._task_config_dict = AttrDict() - @logit(logger) def initialize(self, task_config: AttrDict) -> None: """Initialize JEDI application @@ -86,8 +79,8 @@ def initialize(self, task_config: AttrDict) -> None: Returns ---------- None - """ - + """ + # Render JEDI config dictionary logger.info(f"Generating JEDI YAML config: {self.jedi_config.yaml}") self.jedi_config.input_config = self.render_jcb(task_config) @@ -102,13 +95,12 @@ def initialize(self, task_config: AttrDict) -> None: self.link_exe() @logit(logger) - def execute(self, aprun_cmd: str) -> None: + def execute(self) -> None: """Execute JEDI application Parameters ---------- - aprun_cmd: str - String comprising the run command for the JEDI executable. + None Returns ---------- @@ -117,7 +109,7 @@ def execute(self, aprun_cmd: str) -> None: chdir(self.jedi_config.rundir) - exec_cmd = Executable(aprun_cmd) + exec_cmd = Executable(self.jedi_config.mpi_cmd) exec_cmd.add_default_arg(self.jedi_config.exe) if self.jedi_config.jedi_args: for arg in self.jedi_config.jedi_args: @@ -174,10 +166,6 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> # Generate JEDI YAML config by rendering JCB config dictionary jedi_input_config = render(jcb_config) - # Save copies of the task_config and jcb_config used to render this JCB template - self._task_config_dict[jcb_config['algorithm']] = task_config.deepcopy() - self._jcb_config_dict[jcb_config['algorithm']] = jcb_config.deepcopy() - return jedi_input_config @logit(logger) @@ -201,7 +189,22 @@ def link_exe(self) -> None: @staticmethod @logit(logger) - def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict): + def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: Optional[list] = None): + """Get dictionary of Jedi objects from YAML specifying their configuration dictionaries + + Parameters + ---------- + jedi_config_yaml : str + path to YAML specifying configuration dictionaries for Jedi objects + task_config : str + attribute-dictionary of all configuration variables associated with a GDAS task + + + Returns + ---------- + None + """ + # Initialize dictionary of Jedi objects jedi_dict = AttrDict() @@ -219,6 +222,14 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict): # Construct JEDI object jedi_dict[yaml_name] = Jedi(jedi_config_dict[yaml_name]) + # Make sure jedi_dict has the keys we expect + if expected_keys: + for jedi_dict_key in expected_keys: + if jedi_dict_key not in jedi_dict: + raise Exception(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") + if len(jedi_dict) > len(expected_keys): + raise Exception(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") + # Return dictionary of JEDI objects return jedi_dict diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index dc982229eef..c13f4191422 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -71,7 +71,8 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of Jedi objects - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + expected_keys = ['atmanlvar', 'atmanlfv3inc'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -164,34 +165,20 @@ def initialize(self) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def execute_var(self) -> None: - """Execute JEDI variational analysis application + def execute(self, jedi_dict_key: str) -> None: + """Execute JEDI application of atm analysis Parameters ---------- - None - - Returns - ---------- - None - """ - - self.jedi_dict['atmanlvar'].execute(self.task_config.APRUN_ATMANLVAR) - - @logit(logger) - def execute_fv3inc(self) -> None: - """Execute JEDI FV3 increment converter application - - Parameters - ---------- - None + jedi_dict_key + key specifying particular Jedi object in self.jedi_dict Returns ---------- None """ - self.jedi_dict['atmanlfv3inc'].execute(self.task_config.APRUN_ATMANLFV3INC) + self.jedi_dict[jedi_dict_key].execute() @logit(logger) def finalize(self) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1b437ae46c4..f1c6ecae7e8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -72,7 +72,8 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of JEDI objects - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + expected_keys = ['atmensanlobs', 'atmensanlsol', 'atmensanlfv3inc', 'atmensanlletkf'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -156,7 +157,7 @@ def initialize_letkf(self) -> None: """Initialize a global atmens analysis Note: This would normally be done in AtmEnsAnalysis.initialize(), but that method - now initializes the split observer-solver. This case is just for testing. + now initializes the split observer-solver. This method is just for testing. Parameters ---------- @@ -170,19 +171,20 @@ def initialize_letkf(self) -> None: self.jedi_dict['atmensanlletkf'].initialize(self.task_config) @logit(logger) - def execute_obs(self) -> None: - """Execute JEDI LETKF application in observer mode + def execute(self, jedi_dict_key: str) -> None: + """Execute JEDI application of atmens analysis Parameters ---------- - None + jedi_dict_key + key specifying a particular Jedi object in self.jedi_dict Returns ---------- None """ - self.jedi_dict['atmensanlobs'].execute(self.task_config.APRUN_ATMENSANLOBS) + self.jedi_dict[jedi_dict_key].execute() @logit(logger) def execute_sol(self) -> None: diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 564e79d80d3..c11aa1ae19b 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -73,7 +73,9 @@ def __init__(self, config): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of Jedi objects - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales', + 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) def initialize(self: Task) -> None: @@ -186,18 +188,18 @@ def execute(self) -> None: """ # TODO: This should be optional in case the geometry file was staged - self.jedi_dict['gridgen'].execute(self.task_config.APRUN_MARINEBMAT) - self.jedi_dict['soca_diagb'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['gridgen'].execute() + self.jedi_dict['soca_diagb'].execute() # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi_dict['soca_setcorscales'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_setcorscales'].execute() # TODO: Make this optional once we've converged on an acceptable set of scales - self.jedi_dict['soca_parameters_diffusion_hz'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_parameters_diffusion_hz'].execute() self.execute_vtscales() - self.jedi_dict['soca_parameters_diffusion_vt'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_parameters_diffusion_vt'].execute() # TODO: refactor this from the old scripts if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: - self.jedi_dict['soca_ensb'].execute(self.task_config.APRUN_MARINEBMAT) - self.jedi_dict['soca_ensweights'].execute(self.task_config.APRUN_MARINEBMAT) + self.jedi_dict['soca_ensb'].execute() + self.jedi_dict['soca_ensweights'].execute() @logit(logger) def finalize(self: Task) -> None: From e59e88360052d6ab3f94897c61a6c272e488331d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 04:38:09 +0000 Subject: [PATCH 110/142] pynorms --- ush/python/pygfs/jedi/jedi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c38cca10fc6..e72ae34024d 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -79,8 +79,8 @@ def initialize(self, task_config: AttrDict) -> None: Returns ---------- None - """ - + """ + # Render JEDI config dictionary logger.info(f"Generating JEDI YAML config: {self.jedi_config.yaml}") self.jedi_config.input_config = self.render_jcb(task_config) @@ -198,13 +198,13 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: O path to YAML specifying configuration dictionaries for Jedi objects task_config : str attribute-dictionary of all configuration variables associated with a GDAS task - + Returns ---------- None """ - + # Initialize dictionary of Jedi objects jedi_dict = AttrDict() @@ -229,7 +229,7 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: O raise Exception(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") if len(jedi_dict) > len(expected_keys): raise Exception(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") - + # Return dictionary of JEDI objects return jedi_dict From d4227c01fb608cbb7a91ab1700b218282cea6519 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 11:42:50 +0000 Subject: [PATCH 111/142] Fix bug Russ found --- ush/python/pygfs/task/atm_analysis.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c13f4191422..18b7d3a1532 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -115,9 +115,12 @@ def initialize(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files") bias_dict = self.jedi_dict['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + if bias_dict['copy'] is None: + logger.info(f"No bias correction files to stage") + else: + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections Jedi.extract_tar_from_filehandler_dict(bias_dict) From 7ac6ccb2bbf88b25fb533185c5d481cd328415ee Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 11:45:46 +0000 Subject: [PATCH 112/142] Forgot a line --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 18b7d3a1532..5c3aa0f7647 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -122,8 +122,8 @@ def initialize(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - # extract bias corrections - Jedi.extract_tar_from_filehandler_dict(bias_dict) + # extract bias corrections + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 8781c4489b2b1e30d6ad6b8e1ba322bfbea58908 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:33:17 -0500 Subject: [PATCH 113/142] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index e72ae34024d..6bc67cdf1bf 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -23,7 +23,7 @@ class Jedi: """ @logit(logger, name="Jedi") - def __init__(self, config) -> None: + def __init__(self, config: Dict[str, Any]) -> None: """Constructor for JEDI objects This method will construct a Jedi object. From 227718caf71c188a0075007bab9745b1f3468ba8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:33:25 -0500 Subject: [PATCH 114/142] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 6bc67cdf1bf..a4bfce6ba66 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -29,7 +29,7 @@ def __init__(self, config: Dict[str, Any]) -> None: This method will construct a Jedi object. This includes: - create the jedi_config AttrDict and extend it with additional required entries - - save a coy of jedi_config + - save a copy of jedi_config Parameters ---------- From c7f9d1582aa71490d7304fd2433ab092b1450615 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 15:03:59 +0000 Subject: [PATCH 115/142] Log errors before raising exceptions --- ush/python/pygfs/jedi/jedi.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index a4bfce6ba66..c9419a4cd26 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -43,10 +43,12 @@ def __init__(self, config: Dict[str, Any]) -> None: # Make sure input dictionary for Jedi class constructor has the required keys if 'yaml_name' not in config: - raise KeyError(f"Key 'yaml_name' not found in the nested dictionary") + logger.error(f"FATAL ERROR: Key 'yaml_name' not found in the nested dictionary") + raise KeyError(f"FATAL ERROR: Key 'yaml_name' not found in the nested dictionary") for key in jedi_key_list: if key not in config: - raise KeyError(f"Key '{key}' not found in the nested dictionary") + logger.error(f"FATAL ERROR: Key '{key}' not found in the nested dictionary") + raise KeyError(f"FATAL ERROR: Key '{key}' not found in the nested dictionary") # Create the configuration dictionary for JEDI object local_dict = AttrDict( @@ -120,8 +122,10 @@ def execute(self) -> None: try: exec_cmd() except OSError: + logger.error(f"FATAL ERROR: Failed to execute {exec_cmd}") raise OSError(f"FATAL ERROR: Failed to execute {exec_cmd}") except Exception: + logger.error(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") @logit(logger) @@ -146,6 +150,7 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> if self.jedi_config.jcb_base_yaml: jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: + logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") raise KeyError(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") # Add JCB algorithm YAML, if it exists, to JCB config dictionary @@ -160,6 +165,8 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> elif 'algorithm' in jcb_config: pass else: + logger.error(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") raise Exception(f"FATAL ERROR: JCB algorithm must be specified as input to jedi.render_jcb(), " + "in JEDI configuration dictionary as jcb_algo, or in JCB algorithm YAML") @@ -226,8 +233,10 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: O if expected_keys: for jedi_dict_key in expected_keys: if jedi_dict_key not in jedi_dict: + logger.error(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") raise Exception(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") if len(jedi_dict) > len(expected_keys): + logger.error(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") raise Exception(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") # Return dictionary of JEDI objects From 3d2b7130ef649e551fc02d6afcd7c09650932efb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 17:07:35 +0000 Subject: [PATCH 116/142] Make validation of JEDI class configuration dictionary more robust --- ush/python/pygfs/jedi/jedi.py | 76 +++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c9419a4cd26..c0f8cb197ac 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -14,8 +14,8 @@ logger = getLogger(__name__.split('.')[-1]) -jedi_key_list = ['rundir', 'exe_src', 'jedi_args', 'mpi_cmd', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml'] - +required_jedi_keys = ['rundir', 'exe_src', 'mpi_cmd'] +optional_jedi_keys = ['jedi_args', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml'] class Jedi: """ @@ -43,12 +43,12 @@ def __init__(self, config: Dict[str, Any]) -> None: # Make sure input dictionary for Jedi class constructor has the required keys if 'yaml_name' not in config: - logger.error(f"FATAL ERROR: Key 'yaml_name' not found in the nested dictionary") - raise KeyError(f"FATAL ERROR: Key 'yaml_name' not found in the nested dictionary") - for key in jedi_key_list: + logger.error(f"FATAL ERROR: Key 'yaml_name' not found in config") + raise KeyError(f"FATAL ERROR: Key 'yaml_name' not found in config") + for key in required_jedi_keys: if key not in config: - logger.error(f"FATAL ERROR: Key '{key}' not found in the nested dictionary") - raise KeyError(f"FATAL ERROR: Key '{key}' not found in the nested dictionary") + logger.error(f"FATAL ERROR: Required key '{key}' not found in config") + raise KeyError(f"FATAL ERROR: Required key '{key}' not found in config") # Create the configuration dictionary for JEDI object local_dict = AttrDict( @@ -60,6 +60,11 @@ def __init__(self, config: Dict[str, Any]) -> None: ) self.jedi_config = AttrDict(**config, **local_dict) + # Set optional keys in jedi_config to None if not already present + for key in optional_jedi_keys: + if key not in self.jedi_config: + self.jedi_config[key] = None + # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() @@ -113,7 +118,7 @@ def execute(self) -> None: exec_cmd = Executable(self.jedi_config.mpi_cmd) exec_cmd.add_default_arg(self.jedi_config.exe) - if self.jedi_config.jedi_args: + if not self.jedi_config.jedi_args is None: for arg in self.jedi_config.jedi_args: exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.jedi_config.yaml) @@ -147,20 +152,20 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> """ # Fill JCB base YAML template and build JCB config dictionary - if self.jedi_config.jcb_base_yaml: + if not self.jedi_config.jcb_base_yaml is None: jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: - logger.error(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") - raise KeyError(f"FATAL ERROR: JEDI configuration dictionary must contain jcb_base_yaml.") + logger.error(f"FATAL ERROR: JCB base YAML must be specified in order to render YAML using JCB") + raise KeyError(f"FATAL ERROR: JCB base YAML must be specified in order to render YAML using JCB") # Add JCB algorithm YAML, if it exists, to JCB config dictionary - if self.jedi_config.jcb_algo_yaml: + if not self.jedi_config.jcb_algo_yaml is None: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) # Set algorithm in JCB config dictionary - if algorithm: + if not algorithm is None: jcb_config['algorithm'] = algorithm - elif self.jedi_config.jcb_algo: + elif not self.jedi_config.jcb_algo is None: jcb_config['algorithm'] = self.jedi_config.jcb_algo elif 'algorithm' in jcb_config: pass @@ -196,7 +201,7 @@ def link_exe(self) -> None: @staticmethod @logit(logger) - def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: Optional[list] = None): + def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_block_names: Optional[list] = None): """Get dictionary of Jedi objects from YAML specifying their configuration dictionaries Parameters @@ -205,7 +210,8 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: O path to YAML specifying configuration dictionaries for Jedi objects task_config : str attribute-dictionary of all configuration variables associated with a GDAS task - + expected_block_names (optional) : str + list of names of blocks expected to be in jedi_config_yaml YAML file Returns ---------- @@ -219,23 +225,31 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_keys: O jedi_config_dict = parse_j2yaml(jedi_config_yaml, task_config) # Loop through dictionary of Jedi configuration dictionaries - for yaml_name in jedi_config_dict: - # Make sure all required keys present or set to None - jedi_config_dict[yaml_name]['yaml_name'] = yaml_name - for key in jedi_key_list: - if key not in jedi_config_dict[yaml_name]: - jedi_config_dict[yaml_name][key] = None + for block_name in jedi_config_dict: + # yaml_name key is set to name for this block + jedi_config_dict[block_name]['yaml_name'] = block_name + + # Make sure all required keys present + for key in required_jedi_keys: + if key not in jedi_config_dict[block_name]: + logger.error(f"FATAL ERROR: Required key {key} not found in {jedi_config_yaml} for block {block_name}.") + raise KeyError(f"FATAL ERROR: Required key {key} not found in {jedi_config_yaml} for block {block_name}.") + + # Set optional keys to None + for key in optional_jedi_keys: + if key not in jedi_config_dict[block_name]: + jedi_config_dict[block_name][key] = None # Construct JEDI object - jedi_dict[yaml_name] = Jedi(jedi_config_dict[yaml_name]) - - # Make sure jedi_dict has the keys we expect - if expected_keys: - for jedi_dict_key in expected_keys: - if jedi_dict_key not in jedi_dict: - logger.error(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") - raise Exception(f"FATAL ERROR: {jedi_dict_key} not present {jedi_config_yaml}") - if len(jedi_dict) > len(expected_keys): + jedi_dict[block_name] = Jedi(jedi_config_dict[block_name]) + + # Make sure jedi_dict has the blocks we expect + if expected_block_names: + for block_name in expected_block_names: + if block_name not in jedi_dict: + logger.error(f"FATAL ERROR: Expected block {block_name} not present {jedi_config_yaml}") + raise Exception(f"FATAL ERROR: Expected block {block_name} not present {jedi_config_yaml}") + if len(jedi_dict) > len(expected_block_names): logger.error(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") raise Exception(f"FATAL ERROR: {jedi_config_yaml} specifies more Jedi objects than expected.") From f30d4d93bcc15c18a113403dcc998c066621aa80 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 17:30:52 +0000 Subject: [PATCH 117/142] pynorms+ --- ush/python/pygfs/jedi/jedi.py | 89 ++++------------------------------- 1 file changed, 9 insertions(+), 80 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c0f8cb197ac..b6f9c57f536 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -17,6 +17,7 @@ required_jedi_keys = ['rundir', 'exe_src', 'mpi_cmd'] optional_jedi_keys = ['jedi_args', 'jcb_base_yaml', 'jcb_algo', 'jcb_algo_yaml'] + class Jedi: """ Class for initializing and executing JEDI applications @@ -64,7 +65,7 @@ def __init__(self, config: Dict[str, Any]) -> None: for key in optional_jedi_keys: if key not in self.jedi_config: self.jedi_config[key] = None - + # Save a copy of jedi_config self._jedi_config = self.jedi_config.deepcopy() @@ -118,7 +119,7 @@ def execute(self) -> None: exec_cmd = Executable(self.jedi_config.mpi_cmd) exec_cmd.add_default_arg(self.jedi_config.exe) - if not self.jedi_config.jedi_args is None: + if self.jedi_config.jedi_args is not None: for arg in self.jedi_config.jedi_args: exec_cmd.add_default_arg(arg) exec_cmd.add_default_arg(self.jedi_config.yaml) @@ -152,20 +153,20 @@ def render_jcb(self, task_config: AttrDict, algorithm: Optional[str] = None) -> """ # Fill JCB base YAML template and build JCB config dictionary - if not self.jedi_config.jcb_base_yaml is None: + if self.jedi_config.jcb_base_yaml is not None: jcb_config = parse_j2yaml(self.jedi_config.jcb_base_yaml, task_config) else: logger.error(f"FATAL ERROR: JCB base YAML must be specified in order to render YAML using JCB") raise KeyError(f"FATAL ERROR: JCB base YAML must be specified in order to render YAML using JCB") # Add JCB algorithm YAML, if it exists, to JCB config dictionary - if not self.jedi_config.jcb_algo_yaml is None: + if self.jedi_config.jcb_algo_yaml is not None: jcb_config.update(parse_j2yaml(self.jedi_config.jcb_algo_yaml, task_config)) # Set algorithm in JCB config dictionary - if not algorithm is None: + if algorithm is not None: jcb_config['algorithm'] = algorithm - elif not self.jedi_config.jcb_algo is None: + elif self.jedi_config.jcb_algo is not None: jcb_config['algorithm'] = self.jedi_config.jcb_algo elif 'algorithm' in jcb_config: pass @@ -228,7 +229,7 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_block_n for block_name in jedi_config_dict: # yaml_name key is set to name for this block jedi_config_dict[block_name]['yaml_name'] = block_name - + # Make sure all required keys present for key in required_jedi_keys: if key not in jedi_config_dict[block_name]: @@ -237,7 +238,7 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_block_n # Set optional keys to None for key in optional_jedi_keys: - if key not in jedi_config_dict[block_name]: + if key not in jedi_config_dict[block_name]: jedi_config_dict[block_name][key] = None # Construct JEDI object @@ -348,75 +349,3 @@ def extract_tar(tar_file: str) -> None: except tarfile.ExtractError as err: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") - - -# TODO: remove since no longer used -@logit(logger) -def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: - """ - Recursively search through a nested dictionary and return the value for the target key. - This returns the first target key it finds. So if a key exists in a subsequent - nested dictionary, it will not be found. - - Parameters - ---------- - nested_dict : Dict - Dictionary to search - target_key : str - Key to search for - - Returns - ------- - Any - Value of the target key - - Raises - ------ - KeyError - If key is not found in dictionary - - TODO: if this gives issues due to landing on an incorrect key in the nested - dictionary, we will have to implement a more concrete method to search for a key - given a more complete address. See resolved conversations in PR 2387 - - # Example usage: - nested_dict = { - 'a': { - 'b': { - 'c': 1, - 'd': { - 'e': 2, - 'f': 3 - } - }, - 'g': 4 - }, - 'h': { - 'i': 5 - }, - 'j': { - 'k': 6 - } - } - - user_key = input("Enter the key to search for: ") - result = find_value_in_nested_dict(nested_dict, user_key) - """ - - if not isinstance(nested_dict, dict): - raise TypeError(f"Input is not of type(dict)") - - result = nested_dict.get(target_key) - if result is not None: - return result - - for value in nested_dict.values(): - if isinstance(value, dict): - try: - result = find_value_in_nested_dict(value, target_key) - if result is not None: - return result - except KeyError: - pass - - raise KeyError(f"Key '{target_key}' not found in the nested dictionary") From eb6744c6e246cdb74da321fba66c3918497c0a2f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 17:40:17 +0000 Subject: [PATCH 118/142] Fill in missing code for tar archive extraction --- ush/python/pygfs/jedi/jedi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b6f9c57f536..623beff4b9d 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -342,10 +342,11 @@ def extract_tar(tar_file: str) -> None: raise tarfile.FileExistsError(f"FATAL ERROR: {tar_file} does not exist") except tarfile.ReadError as err: if tarfile.is_tarfile(tar_file): - logger.error(f"FATAL ERROR: {tar_file} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") + logger.error(f"FATAL ERROR: tar archive {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: tar archive {tar_file} could not be read") else: - logger.info() + logger.error(f"FATAL ERROR: {tar_file} is not a tar archive") + raise tarfile.ReadError(f"FATAL ERROR: {tar_file} is not a tar archive") except tarfile.ExtractError as err: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") From b35ec83f69c0c87a9a4f851b97e55d1789b93ab4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 17:47:39 +0000 Subject: [PATCH 119/142] pynorms --- parm/gdas/atmanl_jedi_config.yaml.j2 | 2 +- parm/gdas/atmensanl_jedi_config.yaml.j2 | 6 +++--- ush/python/pygfs/jedi/jedi.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/gdas/atmanl_jedi_config.yaml.j2 b/parm/gdas/atmanl_jedi_config.yaml.j2 index 52fe9c55a0b..4046ba09315 100644 --- a/parm/gdas/atmanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmanl_jedi_config.yaml.j2 @@ -1,8 +1,8 @@ atmanlvar: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jedi_args: ['fv3jedi', 'variational'] mpi_cmd: '{{ APRUN_ATMANLVAR }}' + jedi_args: ['fv3jedi', 'variational'] jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}' atmanlfv3inc: diff --git a/parm/gdas/atmensanl_jedi_config.yaml.j2 b/parm/gdas/atmensanl_jedi_config.yaml.j2 index 73ba96efb8b..9ab2ec6ace1 100644 --- a/parm/gdas/atmensanl_jedi_config.yaml.j2 +++ b/parm/gdas/atmensanl_jedi_config.yaml.j2 @@ -1,15 +1,15 @@ atmensanlobs: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jedi_args: ['fv3jedi', 'localensembleda'] mpi_cmd: '{{ APRUN_ATMENSANLOBS }}' + jedi_args: ['fv3jedi', 'localensembleda'] jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}' atmensanlsol: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jedi_args: ['fv3jedi', 'localensembleda'] mpi_cmd: '{{ APRUN_ATMENSANLSOL }}' + jedi_args: ['fv3jedi', 'localensembleda'] jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}' atmensanlfv3inc: @@ -21,7 +21,7 @@ atmensanlfv3inc: atmensanlletkf: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas.x' - jedi_args: ['fv3jedi', 'localensembleda'] mpi_cmd: '{{ APRUN_ATMENSANLLETKF }}' + jedi_args: ['fv3jedi', 'localensembleda'] jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}' diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 623beff4b9d..2806ba4bce9 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -346,7 +346,7 @@ def extract_tar(tar_file: str) -> None: raise tarfile.ReadError(f"FATAL ERROR: tar archive {tar_file} could not be read") else: logger.error(f"FATAL ERROR: {tar_file} is not a tar archive") - raise tarfile.ReadError(f"FATAL ERROR: {tar_file} is not a tar archive") + raise tarfile.ReadError(f"FATAL ERROR: {tar_file} is not a tar archive") except tarfile.ExtractError as err: logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") From 727b615dae7952ba4e8cb468d274acd4485cf50f Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 6 Nov 2024 18:10:54 +0000 Subject: [PATCH 120/142] updates for scripts that run --- parm/config/gfs/config.anlstat | 2 ++ parm/gdas/stat/jcb-base.yaml.j2 | 4 ++++ parm/stat/obspace_stat.yaml | 5 ----- ush/python/pygfs/task/stat_analysis.py | 16 +++++++++------- 4 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 parm/gdas/stat/jcb-base.yaml.j2 delete mode 100644 parm/stat/obspace_stat.yaml diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index 9ad797513d3..d001cda5581 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -8,6 +8,8 @@ echo "BEGIN: config.anlstat" # Get task specific resources source "${EXPDIR}/config.resources" anlstat +export JCB_BASE_YAML="${PARMgfs}/gdas/stat/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-gdas/algorithm/obstats/aero/viirs_n20_template.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x echo "END: config.anlstat" diff --git a/parm/gdas/stat/jcb-base.yaml.j2 b/parm/gdas/stat/jcb-base.yaml.j2 new file mode 100644 index 00000000000..a4db9fbecf9 --- /dev/null +++ b/parm/gdas/stat/jcb-base.yaml.j2 @@ -0,0 +1,4 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero" diff --git a/parm/stat/obspace_stat.yaml b/parm/stat/obspace_stat.yaml deleted file mode 100644 index 7ae55b6770d..00000000000 --- a/parm/stat/obspace_stat.yaml +++ /dev/null @@ -1,5 +0,0 @@ -obs space: - viirs_npp: - path: {FIXgdas}/parm/stat/viirs_npp_template.yaml - viirs_n20: - path: {FIXgdas}/parm/stat/viirs_n20_template.yaml diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 980b9d63f1f..075abd2f2cb 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -64,7 +64,7 @@ def __init__(self, config: Dict[str, Any]): 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'OBSPACE_YAML': "/scratch1/NCEPDEV/da/Kevin.Dougherty/global-workflow/parm/stat/obspace_stat.yaml" + #'OBSPACE_YAML': "/scratch1/NCEPDEV/da/Kevin.Dougherty/global-workflow/parm/stat/obspace_stat.yaml" } ) @@ -81,8 +81,8 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': self.task_config.JCB_ALGO, - 'jcb_algo_yaml': None, #self.task_config.JCB_ALGO_YAML_VAR, + 'jcb_algo': 'viirs_n20_template', + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, #self.task_config.JCB_ALGO_YAML_VAR, 'jedi_args': None #['fv3jedi', 'variational'] } )) @@ -109,6 +109,8 @@ def initialize(self) -> None: logger.info(f"Initializing JEDI variational DA application") self.jedi['statanl'].initialize(self.task_config) + self.jedi['statanl'].render_jcb(self.task_config) + logger.info(f"Copying files to {self.task_config.DATA}/stats") # Copy stat files to DATA path @@ -140,13 +142,13 @@ def initialize(self) -> None: obspace = '_'.join(filename.split('_')[1:3]) # Load g-w obs space intermediate yaml file - with open(self.task_config.OBSPACE_YAML, 'r') as yaml_file: + with open(self.task_config.JCB_ALGO_YAML, 'r') as yaml_file: parsed_yaml_file = yaml.safe_load(yaml_file) - obs_space_template = parsed_yaml_file['obs space'][obspace]['path'] + # obs_space_template = parsed_yaml_file['ob spaces'] # Load specific GDASApp yaml file - with open(obs_space_template, 'r') as obspace_yaml: - parsed_obspace_yaml_file = yaml.safe_load(obspace_yaml) + # with open(obs_space_template, 'r') as obspace_yaml: + # parsed_obspace_yaml_file = yaml.safe_load(obspace_yaml) print("open obspace yaml and do stuff") From cfc8b904c31aa75d0028b8dc457213a23b754ba2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 18:53:50 +0000 Subject: [PATCH 121/142] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 58e31a644f6..e9607fc9d27 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 58e31a644f666ae70353d33d2ca6f41fdd4d6b21 +Subproject commit e9607fc9d27a2d5e9dde9c2f66903d5ec06f4280 From b03967d8336fb9146797eb68301176a5a77e9020 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 19:31:14 +0000 Subject: [PATCH 122/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e9607fc9d27..2f1638f126d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e9607fc9d27a2d5e9dde9c2f66903d5ec06f4280 +Subproject commit 2f1638f126d010ee2f34aa4da5013ce14549d436 From 8d2a9138ac41a576b638b758282c1f9cf7176709 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 20:52:16 +0000 Subject: [PATCH 123/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2f1638f126d..4c9b1d22314 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2f1638f126d010ee2f34aa4da5013ce14549d436 +Subproject commit 4c9b1d22314845bd8cf388cd5b9603b2cbf9ccfd From 62cba2533993a3bda4cfa7a8672fc47d05bedf41 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 20:55:04 +0000 Subject: [PATCH 124/142] Forgot to remove old execute methods --- ush/python/pygfs/task/atmens_analysis.py | 48 ------------------------ 1 file changed, 48 deletions(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index f1c6ecae7e8..81cae238bbe 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -186,54 +186,6 @@ def execute(self, jedi_dict_key: str) -> None: self.jedi_dict[jedi_dict_key].execute() - @logit(logger) - def execute_sol(self) -> None: - """Execute JEDI LETKF application in solver mode - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - self.jedi_dict['atmensanlsol'].execute(self.task_config.APRUN_ATMENSANLSOL) - - @logit(logger) - def execute_fv3inc(self) -> None: - """Execute FV3 increment converter - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - self.jedi_dict['atmensanlfv3inc'].execute(self.task_config.APRUN_ATMENSANLFV3INC) - - @logit(logger) - def execute_letkf(self) -> None: - """Execute full JEDI LETKF application - - Note: This is just for testing. Operationally, we plan to split the LETKF - into observer and solver modes. - - Parameters - ---------- - None - - Returns - ---------- - None - """ - - self.jedi_dict['atmensanlletkf'].execute(self.task_config.APRUN_ATMENSANLLETKF) - @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis From dd6736f8805aa8c270452c06f7992fc9185325d1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 21:28:48 +0000 Subject: [PATCH 125/142] Get rid of execute_vtscales method --- ush/python/pygfs/task/marine_bmat.py | 43 +++++++++------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index c11aa1ae19b..2af42aca3e1 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -146,30 +146,6 @@ def initialize(self: Task) -> None: os.remove(link_name) os.symlink(link_target, link_name) - @logit(logger) - def execute_vtscales(self: Task) -> None: - """Execute vertical diffusion coefficients generator - - This method will execute a Python script which generatres the vertical diffusion coefficients - This includes: - - constructing the executable object - - running the executable object - - Parameters - ---------- - None - - Returns - ---------- - None - """ - # compute the vertical correlation scales based on the MLD - exec_cmd = Executable("python") - exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) - @logit(logger) def execute(self) -> None: """Generate the full B-matrix @@ -187,16 +163,25 @@ def execute(self) -> None: None """ - # TODO: This should be optional in case the geometry file was staged self.jedi_dict['gridgen'].execute() + + # variance partitioning self.jedi_dict['soca_diagb'].execute() - # TODO: Make this optional once we've converged on an acceptable set of scales + + # horizontal diffusion self.jedi_dict['soca_setcorscales'].execute() - # TODO: Make this optional once we've converged on an acceptable set of scales self.jedi_dict['soca_parameters_diffusion_hz'].execute() - self.execute_vtscales() + + # vertical diffusion + exec_cmd = Executable("python") + exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_vtscales.yaml') + mdau.run(exec_cmd) + self.jedi_dict['soca_parameters_diffusion_vt'].execute() - # TODO: refactor this from the old scripts + + # hybrid EnVAR case if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.jedi_dict['soca_ensb'].execute() self.jedi_dict['soca_ensweights'].execute() From a6fd65ad7a954c07774029dc6174e91646f8ad92 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 6 Nov 2024 21:31:22 +0000 Subject: [PATCH 126/142] pynorms --- ush/python/pygfs/task/marine_bmat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 2af42aca3e1..a21699227b2 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -171,13 +171,13 @@ def execute(self) -> None: # horizontal diffusion self.jedi_dict['soca_setcorscales'].execute() self.jedi_dict['soca_parameters_diffusion_hz'].execute() - + # vertical diffusion exec_cmd = Executable("python") exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') - mdau.run(exec_cmd) + mdau.run(exec_cmd) self.jedi_dict['soca_parameters_diffusion_vt'].execute() From 81bbcf99f885f1583820987519b611cec5971f3d Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 7 Nov 2024 14:19:31 +0000 Subject: [PATCH 127/142] pynorms --- ush/python/pygfs/task/stat_analysis.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 075abd2f2cb..2d040064252 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -56,15 +56,12 @@ def __init__(self, config: Dict[str, Any]): 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - # 'npx_anl': _res_anl + 1, - # 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - #'OBSPACE_YAML': "/scratch1/NCEPDEV/da/Kevin.Dougherty/global-workflow/parm/stat/obspace_stat.yaml" + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." } ) @@ -82,8 +79,8 @@ def __init__(self, config: Dict[str, Any]): 'exe_src': self.task_config.JEDIEXE, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, 'jcb_algo': 'viirs_n20_template', - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, #self.task_config.JCB_ALGO_YAML_VAR, - 'jedi_args': None #['fv3jedi', 'variational'] + 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, + 'jedi_args': None } )) @@ -145,10 +142,4 @@ def initialize(self) -> None: with open(self.task_config.JCB_ALGO_YAML, 'r') as yaml_file: parsed_yaml_file = yaml.safe_load(yaml_file) - # obs_space_template = parsed_yaml_file['ob spaces'] - - # Load specific GDASApp yaml file - # with open(obs_space_template, 'r') as obspace_yaml: - # parsed_obspace_yaml_file = yaml.safe_load(obspace_yaml) - print("open obspace yaml and do stuff") From c2d6c7d1dbb4d266e465bab89c8178e052a60f69 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 13 Nov 2024 17:16:47 +0000 Subject: [PATCH 128/142] update scripts to new working g-w --- parm/config/gfs/config.anlstat | 4 ++-- parm/gdas/stat/aero/jcb-base.yaml.j2 | 24 ++++++++++++++++++++++++ parm/gdas/stat/jcb-base.yaml.j2 | 4 ---- ush/python/pygfs/task/stat_analysis.py | 24 +++++++----------------- 4 files changed, 33 insertions(+), 23 deletions(-) create mode 100644 parm/gdas/stat/aero/jcb-base.yaml.j2 delete mode 100644 parm/gdas/stat/jcb-base.yaml.j2 diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index d001cda5581..aff25984a72 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -8,8 +8,8 @@ echo "BEGIN: config.anlstat" # Get task specific resources source "${EXPDIR}/config.resources" anlstat -export JCB_BASE_YAML="${PARMgfs}/gdas/stat/jcb-base.yaml.j2" -export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-gdas/algorithm/obstats/aero/viirs_n20_template.yaml.j2" +export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithm/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x echo "END: config.anlstat" diff --git a/parm/gdas/stat/aero/jcb-base.yaml.j2 b/parm/gdas/stat/aero/jcb-base.yaml.j2 new file mode 100644 index 00000000000..f14a2664fd5 --- /dev/null +++ b/parm/gdas/stat/aero/jcb-base.yaml.j2 @@ -0,0 +1,24 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero" + +window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}" +window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}" +window_length: "{{ STAT_WINDOW_LENGTH }}" + +# Inputted list of ob spaces +# -------------------------- +obspaces: {{ OBSPACES_LIST }} + +# Obspace variable things +# ----------------------- +aero_obsdatain_path: "{{ DATA }}" +aero_obsdatatin_simulated_variables: ['aerosolOpticalDepth'] +aero_obsdatain_observed_variables: ['aerosolOpticalDepth'] + +# Variables +# --------- +aero_variables: ['aerosolOpticalDepth'] +aero_file_groups: ['bkgmob', 'bkgmob1'] +aero_file_qc_groups: ['EffectiveQC0', 'EffectiveQC1'] \ No newline at end of file diff --git a/parm/gdas/stat/jcb-base.yaml.j2 b/parm/gdas/stat/jcb-base.yaml.j2 deleted file mode 100644 index a4db9fbecf9..00000000000 --- a/parm/gdas/stat/jcb-base.yaml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Search path for model and obs for JCB -# ------------------------------------- -algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" -app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero" diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 2d040064252..22413dd5b03 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -57,8 +57,8 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, 'npz_anl': self.task_config.LEVS - 1, - 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'STAT_WINDOW_BEGIN': _window_begin, + 'STAT_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." @@ -78,7 +78,7 @@ def __init__(self, config: Dict[str, Any]): 'rundir': self.task_config.DATA, 'exe_src': self.task_config.JEDIEXE, 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': 'viirs_n20_template', + 'jcb_algo': 'anlstat', 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, 'jedi_args': None } @@ -102,12 +102,6 @@ def initialize(self) -> None: ---------- None """ - # initialize JEDI application - logger.info(f"Initializing JEDI variational DA application") - self.jedi['statanl'].initialize(self.task_config) - - self.jedi['statanl'].render_jcb(self.task_config) - logger.info(f"Copying files to {self.task_config.DATA}/stats") # Copy stat files to DATA path @@ -134,12 +128,8 @@ def initialize(self) -> None: # Get list of .nc4 files obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) - for path in obs_space_paths: - filename = os.path.basename(path) - obspace = '_'.join(filename.split('_')[1:3]) + self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] - # Load g-w obs space intermediate yaml file - with open(self.task_config.JCB_ALGO_YAML, 'r') as yaml_file: - parsed_yaml_file = yaml.safe_load(yaml_file) - - print("open obspace yaml and do stuff") + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi['statanl'].initialize(self.task_config) From 8b87f154acea860533e8534ae938a1483d7944be Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 14 Nov 2024 14:11:13 +0000 Subject: [PATCH 129/142] Try to fix marineanlletkf --- env/HERCULES.env | 9 ++++++++- parm/config/gfs/config.marineanlletkf | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/env/HERCULES.env b/env/HERCULES.env index 15e3928f08c..7ef119a7fcd 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -145,7 +145,7 @@ case ${step} in export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}} [[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task} export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}" -;; + ;; "marineanlchkpt") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" @@ -153,6 +153,13 @@ case ${step} in export NTHREADS_OCNANAL=${NTHREADSmax} export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" ;; + "marineanlletkf") + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF=${APRUN_default} + ;; "anal" | "analcalc") export MKL_NUM_THREADS=4 diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf index 8b84af4eaa5..93f03f80ff3 100644 --- a/parm/config/gfs/config.marineanlletkf +++ b/parm/config/gfs/config.marineanlletkf @@ -14,7 +14,7 @@ export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" -export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" +export GRIDGEN_YAML="${HOMEgfs}/sorc/gdas.cd/parm/jcb-gdas/algorithm/marine/soca_gridgen.yaml.j2" export DIST_HALO_SIZE=500000 echo "END: config.marineanlletkf" From b6f0f98825728f6cecf902db3c96b43e7c1dc38b Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 14 Nov 2024 20:03:43 +0000 Subject: [PATCH 130/142] updated sorc/gdas.cd hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7f342751d0c..fc54039b248 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7f342751d0c51332f8151a5d7670fc0e3658ace4 +Subproject commit fc54039b24821a9d0ac9866494d244ecd4819aa8 From 42904ba032d683add69f75f6c4f148f30b9e278b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 14 Nov 2024 23:22:42 +0000 Subject: [PATCH 131/142] Follow example of ORION.env for HERCULUS.env marineanlletkf --- env/HERCULES.env | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env/HERCULES.env b/env/HERCULES.env index 7ef119a7fcd..fccc2f87a51 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -155,10 +155,8 @@ case ${step} in ;; "marineanlletkf") - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export NTHREADS_MARINEANLLETKF=${NTHREADSmax} - export APRUN_MARINEANLLETKF=${APRUN_default} + export APRUN_MARINEANLLETKF="${APRUN_default}" ;; "anal" | "analcalc") From a9522f698c039d9ca63718b35e9fc2401a7a5d4d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 15 Nov 2024 12:14:04 +0000 Subject: [PATCH 132/142] Fixed bug in orion resource allocation --- parm/config/gfs/config.resources.ORION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION index 461b6f14f7c..fceb8feeb19 100644 --- a/parm/config/gfs/config.resources.ORION +++ b/parm/config/gfs/config.resources.ORION @@ -25,7 +25,8 @@ case ${step} in ;; "atmanlvar") # Run on 8 nodes for memory requirement - export tasks_per_node=8 + export tasks_per_node_gdas=8 + export tasks_per_node_gfs=8 export walltime="00:45:00" ;; "atmensanlobs") From 4a89bf4cb536dfbff29ec8e2187673a8f4b23a23 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 15 Nov 2024 19:33:21 +0000 Subject: [PATCH 133/142] Update APRUN commands in WCOSS2 env --- env/WCOSS2.env | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index fff8f7b096a..a7a2609aada 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -107,17 +107,15 @@ elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" export APRUN_MARINEBMAT="${APRUN_default}" -elif [[ "${step}" = "ocnanalrun" ]]; then +elif [[ "${step}" = "marineanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEANLVAR="${APRUN_default}" - export APRUN_OCNANAL="${APRUN_default}" - -elif [[ "${step}" = "ocnanalchkpt" ]]; then +elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - - export APRUN_OCNANAL="${APRUN_default}" + export APRUN_MARINEANLCHKPT="${APRUN_default}" elif [[ "${step}" = "ocnanalecen" ]]; then @@ -134,13 +132,18 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN_default}" -elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then +elif [[ "${step}" = "analcalc_fv3jedi" ]]; then + + export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax} + export APRUN_ANALCALC_FV3JEDI="${APRUN_default}" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=1G export FI_OFI_RXM_SAR_LIMIT=3145728 - if [[ "${step}" = "analcalc" ]]; then + if [[ "${step}" = "analcalc_gsi" ]]; then export MPICH_MPIIO_HINTS="*:romio_cb_write=disable" fi From 408893c24b13d298b564a9ffd95b11ec6b7145f4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 15 Nov 2024 19:36:42 +0000 Subject: [PATCH 134/142] Update --- env/WCOSS2.env | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index a7a2609aada..75d97e8fdc9 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -131,13 +131,8 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN_default}" - -elif [[ "${step}" = "analcalc_fv3jedi" ]]; then - - export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax} - export APRUN_ANALCALC_FV3JEDI="${APRUN_default}" -elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=1G From 78c989c235fbe4031adb7b7c51b979d9548aa68d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 15 Nov 2024 19:39:34 +0000 Subject: [PATCH 135/142] update (#2) --- env/WCOSS2.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 75d97e8fdc9..27001bebd7a 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -131,14 +131,14 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN_default}" - + elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=1G export FI_OFI_RXM_SAR_LIMIT=3145728 - if [[ "${step}" = "analcalc_gsi" ]]; then + if [[ "${step}" = "analcalc" ]]; then export MPICH_MPIIO_HINTS="*:romio_cb_write=disable" fi From 4695fd7606ee9827e3edf4174ca843e48b9ec1b6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 15 Nov 2024 19:57:35 +0000 Subject: [PATCH 136/142] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 4c9b1d22314..e514b926561 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 4c9b1d22314845bd8cf388cd5b9603b2cbf9ccfd +Subproject commit e514b926561bfc8fa3de741876505aff74255c95 From bb6257b33e983a13f3aac4b380a97f8047fb697b Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:16:25 -0500 Subject: [PATCH 137/142] Update gdas_fv3jedi_ver --- versions/fix.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/fix.ver b/versions/fix.ver index b1757911963..4739ce778ab 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -8,7 +8,7 @@ export cice_ver=20240416 export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20241022 +export gdas_fv3jedi_ver=20241115 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 From ce14a793e7e65492101e37580a90a5138b05e47c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 19 Nov 2024 00:02:14 +0000 Subject: [PATCH 138/142] add updated scripts for new jedi class changes --- parm/config/gfs/config.anlstat | 1 + parm/gdas/statanl_jedi_config.yaml.j2 | 6 +++++ scripts/exglobal_analysis_stats.py | 2 +- ush/python/pygfs/task/stat_analysis.py | 33 ++++++++++++++------------ 4 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 parm/gdas/statanl_jedi_config.yaml.j2 diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index aff25984a72..bfea0cf700b 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -8,6 +8,7 @@ echo "BEGIN: config.anlstat" # Get task specific resources source "${EXPDIR}/config.resources" anlstat +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/statanl_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithm/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x diff --git a/parm/gdas/statanl_jedi_config.yaml.j2 b/parm/gdas/statanl_jedi_config.yaml.j2 new file mode 100644 index 00000000000..64146f22eea --- /dev/null +++ b/parm/gdas/statanl_jedi_config.yaml.j2 @@ -0,0 +1,6 @@ +statanl: + rundir: '{{ DATA }}' + exe_src: '{{ JEDIEXE }}' + jedi_args: None + jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index c9dda5a1cbe..0e3e99058c0 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -25,4 +25,4 @@ StatAnl = StatAnalysis(config) # Initialize JEDI variational analysis - StatAnl.initialize() + StatAnl.jedi_dict['statanl'].initialize() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 22413dd5b03..d17f4b5552a 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -68,21 +68,24 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of JEDI objects - self.jedi = AttrDict() - - # statanlvar - self.jedi['statanl'] = Jedi(AttrDict( - { - 'yaml_name': 'statanl', - 'rundir': self.task_config.DATA, - 'exe_src': self.task_config.JEDIEXE, - 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, - 'jcb_algo': 'anlstat', - 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, - 'jedi_args': None - } - )) + # Create dictionary of Jedi objects + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + + # # Create dictionary of JEDI objects + # self.jedi = AttrDict() + + # # statanlvar + # self.jedi['statanl'] = Jedi(AttrDict( + # { + # 'yaml_name': 'statanl', + # 'rundir': self.task_config.DATA, + # 'exe_src': self.task_config.JEDIEXE, + # 'jcb_base_yaml': self.task_config.JCB_BASE_YAML, + # 'jcb_algo': 'anlstat', + # 'jcb_algo_yaml': self.task_config.JCB_ALGO_YAML, + # 'jedi_args': None + # } + # )) @logit(logger) def initialize(self) -> None: From 0d7cf49442a7caec552f0cc59a33a7e875839315 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 19 Nov 2024 18:37:22 +0000 Subject: [PATCH 139/142] update scripts and fix bugs to new jedi design --- env/HERA.env | 5 +++++ parm/config/gfs/config.anlstat | 2 +- parm/gdas/statanl_jedi_config.yaml.j2 | 6 ++++-- scripts/exglobal_analysis_stats.py | 2 +- ush/python/pygfs/task/stat_analysis.py | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index f10bfcc5378..7c27579db9f 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -105,6 +105,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" +elif [[ "${step}" = "anlstat" ]]; then + + export NTHREADS_ANLSTAT=${NTHREADSmax} + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index bfea0cf700b..1aed963d65d 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -10,7 +10,7 @@ source "${EXPDIR}/config.resources" anlstat export JEDI_CONFIG_YAML="${PARMgfs}/gdas/statanl_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" -export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithm/anlstat.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x echo "END: config.anlstat" diff --git a/parm/gdas/statanl_jedi_config.yaml.j2 b/parm/gdas/statanl_jedi_config.yaml.j2 index 64146f22eea..bbefbaac2cb 100644 --- a/parm/gdas/statanl_jedi_config.yaml.j2 +++ b/parm/gdas/statanl_jedi_config.yaml.j2 @@ -1,6 +1,8 @@ statanl: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' + mpi_cmd: '{{ APRUN_ANLSTAT }}' jedi_args: None - jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/jcb-base.yaml.j2' - jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' \ No newline at end of file + jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' + jcb_algo: 'anlstat' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 0e3e99058c0..c9dda5a1cbe 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -25,4 +25,4 @@ StatAnl = StatAnalysis(config) # Initialize JEDI variational analysis - StatAnl.jedi_dict['statanl'].initialize() + StatAnl.initialize() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index d17f4b5552a..ab363f8e696 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -135,4 +135,4 @@ def initialize(self) -> None: # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") - self.jedi['statanl'].initialize(self.task_config) + self.jedi_dict['statanl'].initialize(self.task_config) From fe62b4d0705215212e9743af8230a26748a4cf5f Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 19 Nov 2024 18:57:39 +0000 Subject: [PATCH 140/142] typo --- workflow/rocoto/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 69619a85824..0ae7c2530a2 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb',= + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', 'snowanl', 'esnowrecen', 'anlstat', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From e5b696dae31d96b8f437958ff441d634ca193019 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 19 Nov 2024 20:02:44 +0000 Subject: [PATCH 141/142] update --- workflow/rocoto/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 0ae7c2530a2..818d502157b 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -18,9 +18,10 @@ class Tasks: 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', - 'snowanl', 'esnowrecen', 'anlstat', + 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', + 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'prepsnowobs', 'snowanl', + 'anlstat', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', From 0ca3071eb95a096c0cf48aa9de6623871d6fb006 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 19 Nov 2024 20:31:57 +0000 Subject: [PATCH 142/142] attempt to fix gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c67479526f0..fc54039b248 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c67479526f0cce9b377d87682c69ea4c3d4b2672 +Subproject commit fc54039b24821a9d0ac9866494d244ecd4819aa8